分类 ELK Stack/logs 下的文章

2018-08-08

2018年08月08日
4,930 阅读
0 评论
0 点赞

2018-08-07

2018年08月07日
3,420 阅读
0 评论
0 点赞

2017-09-13

linuxea:ELK-kibana5.5使用高德地图下载IP地址归类查询库下载地址：https://dev.maxmind.com/zh-hans/geoip/geoip2/geolite2-%E5%BC%80%E6%BA%90%E6%95%B0%E6%8D%AE%E5%BA%93/下载国家的那个解压之后得到这么一个文件：[root@linuxea.com-Node49 /etc/logstash]# ll GeoLite2-City.mmdb -rw-r--r-- 1 logstash logstash 58082983 8月 3 08:57 GeoLite2-City.mmdb应用到配置比如nginx访问日志： geoip { source => "clent_ip" target => "geoip" database => "/etc/logstash/GeoLite2-City.mmdb"database指到地图的位置上在kibana5.5中只需要修改配置文件即可修改kibana在配置文件最后加上一句即可tilemap.url: 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}'

2017年09月13日
6,165 阅读
0 评论
0 点赞

2017-09-11

2017年09月11日
10,982 阅读
5 评论
0 点赞

2017-09-10

linuxea:ELK5.5-redis日志grok处理(filebeat) 收集notice的log日志,日志中有很多信息，常规的收集一些报错信息即可，所以就在收集时候做了控制,只收集错误日志和警告日志include_lines: ["WARNING","ERR"]include_lines 一个正则表达式的列表，以匹配您希望Filebeat包含的行。Filebeat仅导出与列表中正则表达式匹配的行。默认情况下，导出所有行。exclude_files 一个正则表达式的列表，以匹配您想要Filebeat忽略的文件。默认情况下不排除文件参考：https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html多行参考：https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html#multiline日志示例1:M 08 Sep 11:42:43.806 # Server started, Redis version 3.2.9 1:M 08 Sep 11:41:44.806 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 08 Sep 11:12:32.806 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 08 Sep 11:12:32.822 * DB loaded from disk: 0.016 seconds 1:M 08 Sep 11:12:32.822 * The server is now ready to accept connections on port 6379 _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.9 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 08 Sep 11:40:45.806 # ERROR 123最终收集的效果如下：ERRWARNING安装rediscurl -Lks4 https://raw.githubusercontent.com/LinuxEA-Mark/docker-alpine-Redis/master/Sentinel/install_redis.sh|bashredis日志配置loglevel notice logfile "/data/logs/redis_6379.log"配置文件示例：[root@linuxea.com-Node98 /data/rds]# cat /etc/redis/redis.conf bind 0.0.0.0 protected-mode yes port 6379 tcp-backlog 511 timeout 0 tcp-keepalive 300 daemonize no supervised no pidfile "/var/run/redis_6379.pid" loglevel notice logfile "/data/logs/redis_6379.log" databases 8 save 900 1 save 300 10 save 60 10000 stop-writes-on-bgsave-error yes rdbcompression yes rdbchecksum yes dbfilename "dump.rdb" dir "/data/redis" slave-serve-stale-data yes slave-read-only yes repl-diskless-sync no repl-diskless-sync-delay 5 repl-disable-tcp-nodelay no slave-priority 100 appendonly no appendfilename "appendonly.aof" appendfsync everysec no-appendfsync-on-rewrite no auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb aof-load-truncated yes lua-time-limit 5000 slowlog-log-slower-than 100 slowlog-max-len 1000 latency-monitor-threshold 0 notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 list-max-ziplist-size -2 list-compress-depth 0 set-max-intset-entries 512 zset-max-ziplist-entries 128 zset-max-ziplist-value 64 hll-sparse-max-bytes 3000 activerehashing yes client-output-buffer-limit normal 0 0 0 client-output-buffer-limit slave 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 hz 10 aof-rewrite-incremental-fsync yes masterauth "OTdmOWI4ZTM4NTY1M2M4OTZh" requirepass "OTdmOWI4ZTM4NTY1M2M4OTZh" # Generated by CONFIG REWRITE #slaveof 172.25. 6379filebeat配置[root@linuxea.com-Node117 /data/logs]# cat /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log paths: - /data/logs/access_nginx.log document_type: nginx-access-117 - input_type: log paths: - /data/logs/slow_log.CSV document_type: mysql-slow-117 - input_type: log paths: - /data/logs/redis_6379.log document_type: redis-6379-117 include_lines: ["WARNING","ERR"] output.redis: hosts: ["10.10.0.98"] password: "OTdmOWI4ZTM4NTY1M2M4OTZh" key: "default_list" db: 5 timeout: 5 keys: - key: "%{[type]}" mapping: "nginx-access-117": "nginx-access-117" "mysql-slow-117" : "mysql-slow-117" "redis-6379-117" : "redis-6379-117"logstash配置patterns[root@linuxea.com-Node49 /etc/logstash/patterns.d]# cat redis REDISTIMESTAMP %{MONTHDAY} %{MONTH} %{TIME} REDISLOG %{POSINT:pid}\:%{WORD:role} %{REDISTIMESTAMP:timestamp} %{DATA:loglevel} %{GREEDYDATA:msg}logstash配置文件input redis { host => "10.10.0.98" port => "6379" key => "redis-6379-117" data_type => "list" password => "OTdmOWI4ZTM4NTY1M2M4OTZh" threads => "5" db => "5" }filter if [type] == "redis-6379-117" { grok { patterns_dir => "/etc/logstash/patterns.d" match => { "message" => "%{REDISLOG}" } } mutate { gsub => [ "loglevel", "\.", "debug", "loglevel", "\-", "verbose", "loglevel", "\*", "notice", "loglevel", "\#", "warring", "role","X","sentinel", "role","C","RDB/AOF writing child", "role","S","slave", "role","M","master" ] } date { match => [ "timestamp" , "dd MMM HH:mm:ss.SSS" ] target => "@timestamp" remove_field => [ "timestamp" ] } }output if [type] == "redis-6379-117" { elasticsearch { hosts => ["10.0.1.49:9200"] index => "logstash-redis-6379-117-%{+YYYY.MM.dd}" user => "elastic" password => "linuxea" }

2017年09月10日
4,721 阅读
0 评论
0 点赞

2017-09-10

2017年09月10日
5,248 阅读
0 评论
0 点赞

2017-09-09

linuxea:ELK5.5-Tomcat-Access日志grok处理(filebeat) tomcat的访问日志有很多可以调整，这里使用%h %l %u %t [%r] %s [%{Referer}i] [%{User-Agent}i] %b %T，如下日志格式%h 访问的用户IP地址 %l 访问逻辑用户名，通常返回'-' %u 访问验证用户名，通常返回'-' %t 访问日时 %r 访问的方式(post或者是get)，访问的资源和使用的http协议版本 %s 访问返回的http状态 %b 访问资源返回的流量 %T 访问所使用的时间 [%{Referer}i] [%{User-Agent}i]其他可参考：http://tomcat.apache.org/tomcat-8.5-doc/config/valve.html#Access_Logging修改配置文件[root@linuxea.com-Node117 /data/tomcat]# tail -9 conf/server.xml <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs" prefix="access_log" suffix=".log" rotatable="true" resolveHosts="false" pattern="%h %l %u %t [%r] %s [%{Referer}i] [%{User-Agent}i] %b %T" /> </Host> </Engine> </Service> </Server> [root@linuxea.com-Node117 /data/tomcat]# 格式如下： <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs" prefix="access_log" fileDateFormat="yyyy-MM-dd.HH" suffix=".log" rotatable="true" resolveHosts="false" pattern="%h %l %u %t [%r] %s [%{Referer}i] [%{User-Agent}i] %b %T" />那么设置后显示出的日志是这样的：10.10.0.96 - - [04/Sep/2017:19:54:07 +0800] [GET / HTTP/1.1] 200 [-] [Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36] 5 0.104Pattern是这样的：[root@linuxea.com-Node49 /etc/logstash/patterns.d]# cat java JETTYAUDIT %{IP:clent_ip} (?:-|%{USER:logic_user}) (?:-|%{USER:verification_user}) \[%{HTTPDATE:timestamp}\] \[(?:%{WORD:http_verb} %{NOTSPACE:request_url}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\] %{NUMBER:status} \[(?:-|%{NOTSPACE:request_url_2})\] \[%{GREEDYDATA:agent}\] (?:-|%{NUMBER:curl_size}) (?:-|%{NUMBER:responsetime})最终收集的结果图标是这样的：filebeat配置[root@linuxea.com-Node117 /data/tomcat]# cat /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log paths: - /data/logs/access_nginx.log document_type: nginx-access-117 - input_type: log paths: - /data/logs/slow_log.CSV - input_type: log paths: - /data/logs/java.log document_type: java-117 output.redis: hosts: ["10.10.0.98"] password: "OTdmOWI4ZTM4NTY1M2M4OTZh" key: "default_list" db: 5 timeout: 5 keys: - key: "%{[type]}" mapping: "nginx-access-117": "nginx-access-117" "mysql-slow-117" : "mysql-slow-117" "java-117" : "java-117"logstash配置input redis { host => "10.10.0.98" port => "6379" key => "java-117" data_type => "list" password => "OTdmOWI4ZTM4NTY1M2M4OTZh" threads => "5" db => "5" }filter if [type] == "java-117" { grok { patterns_dir => "/etc/logstash/patterns.d" match => { "message" => "%{JETTYAUDIT}" } } useragent { source => "agent" target => "userAgent" } urldecode { all_fields => true } mutate { gsub => ["agent","[\"]",""] #将agent中的 " 换成空 convert => [ "response","integer" ] convert => [ "body_bytes_sent","integer" ] convert => [ "bytes_sent","integer" ] convert => [ "upstream_response_time","float" ] convert => [ "upstream_status","integer" ] convert => [ "request_time","float" ] convert => [ "port","integer" ] } geoip { source => "client_ip" database => "/etc/logstash/GeoLite2-City.mmdb" } if [params] { kv { field_split => ",?" source => "params" } } if [source] =~ /\/API/ { mutate { add_field => { "mode" => "API"} } } else { mutate { add_field => { "mode" => "ENT"} } } date { match => [ "date" , "yyyy-MM-dd HH:mm:ss.SSS" ] } }ouput if [type] == "java-117" { elasticsearch { hosts => ["10.0.1.49:9200"] index => "logstash-java-117-%{+YYYY.MM.dd}" user => "elastic" password => "linuxea" } }完整的配置[root@linuxea.com-Node49 /etc/logstash/patterns.d]# cat ../conf.d/redis-output.yml input { redis { host => "10.10.0.98" port => "6379" key => "nginx-access-117" data_type => "list" password => "OTdmOWI4ZTM4NTY1M2M4OTZh" threads => "5" db => "5" } redis { host => "10.10.0.98" port => "6379" key => "mysql-slow-117" data_type => "list" password => "OTdmOWI4ZTM4NTY1M2M4OTZh" threads => "5" db => "5" } redis { host => "10.10.0.98" port => "6379" key => "java-117" data_type => "list" password => "OTdmOWI4ZTM4NTY1M2M4OTZh" threads => "5" db => "5" } } filter { if [type] == "nginx-access-117" { grok { patterns_dir => [ "/etc/logstash/patterns.d" ] match => { "message" => "%{NGINXACCESS}" } overwrite => [ "message" ] } geoip { source => "clent_ip" target => "geoip" # database => "/etc/logstash/GeoLiteCity.dat" database => "/etc/logstash/GeoLite2-City.mmdb" } useragent { source => "User_Agent" target => "userAgent" } urldecode { all_fields => true } mutate { gsub => ["User_Agent","[\"]",""] #将user_agent中的 " 换成空 convert => [ "response","integer" ] convert => [ "body_bytes_sent","integer" ] convert => [ "bytes_sent","integer" ] convert => [ "upstream_response_time","float" ] convert => [ "upstream_status","integer" ] convert => [ "request_time","float" ] convert => [ "port","integer" ] } date { match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ] } } #########################mysql-slow##################################### if [type] == "mysql-slow-117" { csv { columns => [ "timestamp", "user_host", "query_time", "lock_time", "rows_sent", "rows_examined", "db", "last_insert_id", "insert_id", "server_id", "sql_text", "thread_id", "rows_affected" ] } mutate { convert => { "rows_sent" => "integer" } convert => { "rows_examined" => "integer" } convert => { "last_insert_id" => "integer" } convert => { "insert_id" => "integer" } convert => { "server_id" => "integer" } convert => { "thread_id" => "integer" } convert => { "rows_affected" => "integer" } } date { match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ] remove_field => [ "timestamp" ] } # mutate { remove_field => [ "message" ] } mutate { gsub => [ "query_time", "(.*\.)(\d)(\d)\d+", "\1\2\3", "lock_time", "(.*\.)(\d)(\d)\d+", "\1\2\3" ] } ruby { code => "event.set('query_time' , event.get('query_time') ? event.get('query_time').split(':').inject(0){|a, m| a = a * 60 + m.to_f} : 0)"} ruby { code => "event.set('lock_time' , event.get('lock_time') ? event.get('lock_time').split(':').inject(0){|a, m| a = a * 60 + m.to_f} : 0)" } } #########################java##################################### if [type] == "java-117" { grok { patterns_dir => "/etc/logstash/patterns.d" match => { "message" => "%{JETTYAUDIT}" } } useragent { source => "agent" target => "userAgent" } urldecode { all_fields => true } mutate { gsub => ["agent","[\"]",""] #将agent中的 " 换成空 convert => [ "response","integer" ] convert => [ "body_bytes_sent","integer" ] convert => [ "bytes_sent","integer" ] convert => [ "upstream_response_time","float" ] convert => [ "upstream_status","integer" ] convert => [ "request_time","float" ] convert => [ "port","integer" ] } geoip { source => "client_ip" database => "/etc/logstash/GeoLite2-City.mmdb" } if [params] { kv { field_split => ",?" source => "params" } } if [source] =~ /\/API/ { mutate { add_field => { "mode" => "API"} } } else { mutate { add_field => { "mode" => "ENT"} } } date { match => [ "date" , "yyyy-MM-dd HH:mm:ss.SSS" ] } } #########################java##################################### } output { if "_grokparsefailure" in [tags] { file { path => "/var/log/logstash/grokparsefailure-%{[type]}-%{+YYYY.MM.dd}.log" } } if [type] == "nginx-access-117" { elasticsearch { hosts => ["10.0.1.49:9200"] index => "logstash-nginx-access-117-%{+YYYY.MM.dd}" user => "elastic" password => "linuxea" } } if [type] == "mysql-slow-117" { elasticsearch { hosts => ["10.0.1.49:9200"] index => "logstash-mysql-slow-117-%{+YYYY.MM.dd}" user => "elastic" password => "linuxea" } } if [type] == "java-117" { elasticsearch { hosts => ["10.0.1.49:9200"] index => "logstash-java-117-%{+YYYY.MM.dd}" user => "elastic" password => "linuxea" } } stdout {codec => rubydebug} } [root@linuxea.com-Node49 /etc/logstash/patterns.d]# 最后收集到日志是这样的

2017年09月09日
5,716 阅读
0 评论
0 点赞