logstash-nginx-json-es(6)

安装nginx

yum -y install pcre pcre-devel openss-devel
http://nginx.org/download/nginx-1.6.3.tar.gz
groupadd -r nginx
useradd -g nginx -r nginx
ln -s /usr/local/nginx-1.6.3 /usr/local/nginx

编译

 ./configure \
--prefix=/usr/local/nginx \
--conf-path=/etc/nginx/nginx.conf \
--user=nginx --group=nginx \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx/nginx.pid \
--lock-path=/var/lock/nginx.lock \
--with-http_ssl_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-http_flv_module \
--with-http_mp4_module \
--http-client-body-temp-path=/var/tmp/nginx/client \
--http-proxy-temp-path=/var/tmp/nginx/proxy \
--http-fastcgi-temp-path=/var/tmp/nginx/fastcgi \
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi
make && make install
mkdir -pv /var/tmp/nginx/{client,fastcgi,proxy,uwsgi}
mkdir /usr/local/nginx/logs/
/usr/local/sbin/nginx

编辑nginx配置文件:

vim /etc/nginx/nginx.conf

添加如下字段:

#access_log  logs/access.log  main;
log_format logstash_json '{"@timestamp":"$time_iso8601",'
     '"host": "$server_addr",'
     '"client": "$remote_addr",'
     '"size": $body_bytes_sent,'
     '"responsetime": $request_time,'
     '"domain": "$host",'
     '"url":"$uri",'
     '"referer": "$http_referer",'
     '"agent": "$http_user_agent",'
     '"status":"$status"}';

修改如下:

 access_log  logs/access_json.access.log  logstash_json;

访问后测试:

[root@elk1 logs]# ab -n1000 -c10 http://192.168.1.4:81/

查看日志

[root@elk1 nginx]# cat /usr/local/nginx/logs/access_json.access.log 
{"@timestamp":"2016-03-20T05:46:57-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 612,"responsetime": 0.000,"domain": "192.168.1.4","url":"/index.html","referer": "-","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"200"}
{"@timestamp":"2016-03-20T05:46:57-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 570,"responsetime": 0.000,"domain": "192.168.1.4","url":"/favicon.ico","referer": "http://192.168.1.4:81/","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"404"}
{"@timestamp":"2016-03-20T05:46:59-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 0,"responsetime": 0.000,"domain": "192.168.1.4","url":"/index.html","referer": "-","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"304"}
{"@timestamp":"2016-03-20T05:46:59-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 0,"responsetime": 0.000,"domain": "192.168.1.4","url":"/index.html","referer": "-","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"304"}
[root@elk1 nginx]# 

产生一些日志让logstash收集

[root@elk1 nginx]# ab -n1000 -c10 http://192.168.1.4:81/
[root@elk1 logs]# ll
total 440
-rw-r--r-- 1 root root 449286 Mar 20 05:57 access_json.access.log
[root@elk1 logs]# 

当测试日志可用后,修改logstash配置文件,将access_json.access.log推到redis

[root@elk1 logs]# cat /etc/logstash.conf 
input {
# file {
#   path => "/var/log/messages"
#   type => "system-log"
#   }
 file {
        path => "/usr/local/nginx/logs/access_json.access.log"
    codec => "json"        
}
}
output {
#  redis {
#   host => "192.168.1.6"
#   data_type => "list"
#   key => "system.messages"
#   port => "6379"
#   db => "1"
#}
  redis {
    host => "192.168.1.6"
    data_type => "list"
    key => "nginx-access.log"
    port => "6379"
    db => "2"
}
}
[root@elk1 logs]# 

而后在模拟一些日志

[root@elk1 logs]# ab -n1000 -c10 http://192.168.1.4:81/

而后在redis上查看是否传递到redis

redis 192.168.1.6:6379[2]> select 2
OK
redis 192.168.1.6:6379[2]> keys *
1) "nginx-access.log"
redis 192.168.1.6:6379[2]> llen nginx-access.log
(integer) 1000
redis 192.168.1.6:6379[2]> 

验证数据存在,修改logstash文件传递到es,配置如下:

[root@yum-down ~]# cat /etc/logstash.conf 
input {
#   redis {
#        host => "192.168.1.6"
#        data_type => "list"
#        key => "test.log"
 #       port => "6379"
#        db => "1"
#}
    redis {
        host => "192.168.1.6"
        data_type => "list"
        key => "nginx-access.log"  #key名称和redis保持一致
        port => "6379"
        db => "2"   #db2
}
}
output {
#   elasticsearch {
#    host => ["192.168.1.4:9200","192.168.1.5:9200"]
#    index => "redis-system-messages-%{+YYYY.MM.dd.HH}"
#        protocol => "http"
#        workers => 5
#        template_overwrite => true
#    }
   elasticsearch {
    host => ["192.168.1.4:9200","192.168.1.5:9200"]
    index => "nginx-access-log-%{+YYYY.MM.dd.HH}"  #修改es中日志名称
        protocol => "http"
        workers => 5
        template_overwrite => true
}
}
[root@yum-down ~]# 

1 分享

您可以选择一种方式赞助本站

支付宝扫码赞助

支付宝扫码赞助

日期: 2016-03-21分类: ELK Stack

标签: elk

发表评论