1 收集nginx日志

1.1 配置logstash

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
cat logstash.conf 
input {
  beats {
    port => 5044
    codec => json
  }
}

filter {
  geoip {
    target => "geoip"
    source => "realip"
    add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
    add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
  }
  mutate {
    convert => ["[geoip][coordinates]","float"]
    convert => [ "visit_host","string" ]
    convert => [ "status","integer" ]
    convert => [ "size","integer" ]
    convert => [ "upstreamtime","float" ]
    convert => [ "req_time","float" ]
    remove_field => [ "ecs","agent","host","cloud","@version","input","logs_type" ]
    # 去掉显示 geoip 显示的多余信息
    remove_field => ["[geoip][latitude]", "[geoip][longitude]", "[geoip][country_code]", "[geoip][country_code2]", "[geoip][country_code3]", "[geoip][timezone]", "[geoip][continent_code]", "[geoip][region_code]
"]
  }

  # 根据 http_user_agent来自动处理区分用户客户端系统与版本
  useragent {
    source => "user-agent"
    target => "ua"
    # 过滤useragent没用的字段
    remove_field => [ "[ua][minor]","[ua][major]","[ua][build]","[ua][patch]","[ua][os_minor]","[ua][os_major]" ]
  }
}


output {
  elasticsearch {
    hosts => ["http://192.168.1.37:30092"]
    index => "logstash-nginx-log-domain-access-%{+YYYY.MM}"
    #user => "elastic"
    #password => "changeme"
  }
}

# vim /usr/lib/systemd/system/logstash.service 
[Unit] 
Description=logstash

[Service] 
ExecStart=/usr/local/logstash-7.16.2/bin/logstash -f /usr/local/logstash-7.16.2/config/logstash.conf
ExecReload=/bin/kill -HUP $MAINPID 
KillMode=process 
Restart=on-failure

[Install] 
WantedBy=multi-user.target

systemctl daemon-reload
systemctl restart logstash

18.2 配置filebeat

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
cat filebeat.yml  |grep -v "^#" |grep -v "^$" |grep -v "^  #" |grep -v "^    #"
---
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/domain/*.log
  fields_under_root: true
  tags: "domain-access"

processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - decode_json_fields:
      fields: ['message']
      target: "" 
      overwrite_keys: false 
      process_array: false 
      max_depth: 1
  - drop_fields:
      #删除的多余字段
      fields: ["host", "ecs", "log", "prospector", "agent", "input", "beat", "offset"]

#setup.template.settings:
#  index.number_of_shards: 3
#  index.number_of_replicas: 0
#如果es索引模板已经存在，我们要修改 主副分片数 必须覆盖原来的模块
setup.template.overwrite: true
#设置自定义索引模块
setup.template.name: "nginx-log"
setup.template.pattern: "nginx-log-*"

output.logstash:
  hosts: ["192.168.1.37:5044"]
---
  
/mnt/filebeat-7.16.2-linux-x86_64/filebeat -e -d /mnt/filebeat-7.16.2-linux-x86_64/filebeat.yml 

# vim /usr/lib/systemd/system/filebeat.service 
[Unit] 
Description=filebeat

[Service] 
ExecStart=/usr/local/filebeat-7.16.2/filebeat -e -c /usr/local/filebeat-7.16.2/filebeat.yml
ExecReload=/bin/kill -HUP $MAINPID 
KillMode=process 
Restart=on-failure

[Install] 
WantedBy=multi-user.target

systemctl daemon-reload
systemctl start filebeat
systemctl restart filebeat