graylog收集rsyslog实现搜索解析
安装配置
基础
# 按照实际情况放行
systemctl stop firewalld
systemctl disable firewalldsed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
setenforce 0vim /etc/chrony.conf
server ${ntp_server} iburstsystemctl enable --now chronyd
chronyc sources -v
mongodb
vim /etc/yum.repos.d/mongodb-org.repo[mongodb-org-7.0]
name=MongoDB Repository
baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/7.0/x86_64/
gpgcheck=1
enabled=1
gpgkey=https://www.mongodb.org/static/pgp/server-7.0.ascyum install -y mongodb-orgvim /etc/mongod.confnet:port: 27017bindIp: 127.0.0.1systemctl daemon-reload
systemctl enable mongod.service
systemctl start mongod.service
systemctl status mongod.service
daatanode
rpm -Uvh https://packages.graylog2.org/repo/packages/graylog-6.2-repository_latest.rpm
yum install -y graylog-datanode# 修改、查看vm.max_map_count数值
echo 'vm.max_map_count=262144' | sudo tee -a /etc/sysctl.d/99-graylog-datanode.conf
sysctl --system
cat /proc/sys/vm/max_map_count# 生成、修改password_secret值
openssl rand -hex 32
a51ed4638b4030462009c2a32edfc9744fb2abfa25e7a48a9a4e7de67dfeb385
# 生成、修改root_password_sha2值
echo -n H3c#xinhuasan | sha256sum
a0f15f1709c04b0ad5c82ea964b85794954adad992d70dfc6bf7b88952ea3b08vim /etc/graylog/datanode/datanode.confpassword_secret = a51ed4638b4030462009c2a32edfc9744fb2abfa25e7a48a9a4e7de67dfeb385
root_password_sha2 = a0f15f1709c04b0ad5c82ea964b85794954adad992d70dfc6bf7b88952ea3b08
mongodb_uri = mongodb://localhost:27017/graylog
# 手动添加,值为内存的一半
opensearch_heap = 2gsystemctl daemon-reload
systemctl enable graylog-datanode.service
systemctl start graylog-datanode
systemctl status graylog-datanode
graylog-server
rpm -Uvh https://packages.graylog2.org/repo/packages/graylog-6.2-repository_latest.rpm
yum install -y graylog-servervim /etc/graylog/server/server.confpassword_secret = a51ed4638b4030462009c2a32edfc9744fb2abfa25e7a48a9a4e7de67dfeb385
root_password_sha2 = a0f15f1709c04b0ad5c82ea964b85794954adad992d70dfc6bf7b88952ea3b08
# 取消注释
http_bind_address = 0.0.0.0:9000
# 按需调整
message_journal_max_age = 12h
message_journal_max_size = 2gbvim /etc/sysconfig/graylog-server
# 数值为运行时内存的一半,不超过16G
GRAYLOG_SERVER_JAVA_OPTS="-Xms1g -Xmx1g -server -XX:+UseG1GC -XX:-OmitStackTraceInFastThrow"systemctl daemon-reload
systemctl enable graylog-server.service
systemctl start graylog-server.service
systemctl status graylog-server.service
初始化与登录
tail /var/log/graylog-server/server.log
========================================================================================================It seems you are starting Graylog for the first time. To set up a fresh install, a setup interface has
been started. You must log in to it to perform the initial configuration and continue.Initial configuration is accessible at 0.0.0.0:9000, with username 'admin' and password 'dVxEVqPGoK'.
Try clicking on http://admin:dVxEVqPGoK@0.0.0.0:9000========================================================================================================
# 按照提示分配CA后,输入用户admin密码H3c#xinhuasan登录
功能介绍
input
主页菜单 -> system -> inputs -> syslog UDP -> launch new input
Title:名称
port:rsyslog端口
在终端配置好rsyslog,show received messages,则跳转到search
search
语法参考Lucene,注意转义符
# 不区分大小写包含“xxx”
xxx# 包含“xxx”或“yyy”
xxx yyy# 完全符合“ABCDEF”
"ABCDEF"# 字段为“A-B-C.1”
source:A-B-C.1# 包含A-B-C.1或D-E-F.2
source:(A-B-C.1 OR D-E-F.2)# 包含abc
_exists_:abc# 包含A-B-C.1且D-E-F.2
source:A-B-C.1 AND "D-E-F.2"# level小于3,但排除message中包含“IBM”
(level:<=3) AND NOT message:"IBM"
Index Sets
索引集决定日志如何存储到datanode中,system -> indices
在create index set中,
Title:名称
Description:名称
Index prefix:比如:container-docker、server-IBM
streams
将日志实时路由到不同目录,system -> input -> show received messages,拷贝gl2_source_input的值
在streams标签页点击create stream,填写Title、Description、Index Set,勾选Remove matches from ‘All messages’ stream,再保存
manage rules -> Add stream rule
把之前复制的gl2_source_input的value作为stream的匹配规则
回到stream页面,点击start stream,再点击对应的stream,就可以查看路由到该stream的日志
extractors
解析日志:找到一条关注的日志,拷贝id、stored in index
inputs -> manage extractors,填写复制内容
选择regular expression进行解析
一个extractor只能解析一个字符串,按需添加。extractors可以配合其他功能使用。