Elk-Grafana分析nginx日志

filebeat.inputs:
# 收集nginx日志
- type: log
  enabled: true
  paths:
    - /var/log/nginx/*.log
  fields:                  #
    type: nginx1           # 多日志收集,添加自定义字段
  fields_under_root: true  # 置顶 type字段 这样 logstash里才能根据type的值来过滤
# 日志是json开启这个
  json.keys_under_root: true
  json.overwrite_keys: true
  json.add_error_key: true
          
output:
  logstash:
    hosts: ["172.16.150.75:5044"]

input {
  beats {
    host => '0.0.0.0'
    port => 5044
  }
}

filter {
   if [type] == "nginx1" {
    geoip {
    #multiLang => "zh-CN"
    target => "geoip"
    source => "client_ip"
    database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-City.mmdb"
    add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
    add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
    # 去掉显示 geoip 显示的多余信息
    remove_field => ["[geoip][latitude]", "[geoip][longitude]", "[geoip][country_code]", "[geoip][country_code2]", "[geoip][country_code3]", "[geoip][timezone]", "[geoip][continent_code]", "[geoip][region_code]"]
  }
  mutate {
    convert => [ "size", "integer" ]
    convert => [ "status", "integer" ]
    convert => [ "responsetime", "float" ]
    convert => [ "upstreamtime", "float" ]
    convert => [ "[geoip][coordinates]", "float" ]
    # 过滤 filebeat 没用的字段,这里过滤的字段要考虑好输出到es的，否则过滤了就没法做判断
    remove_field => [ "ecs","agent","host","cloud","@version","input","logs_type" ]
  }
  # 根据http_user_agent来自动处理区分用户客户端系统与版本
  useragent {
    source => "http_user_agent"
    target => "ua"
    # 过滤useragent没用的字段
    remove_field => [ "[ua][minor]","[ua][major]","[ua][build]","[ua][patch]","[ua][os_minor]","[ua][os_major]" ]
  }
  }
  
}
output {
  if [type] == "nginx1" {
  elasticsearch {
    hosts => ["http://172.16.150.75:9200", "http://172.16.150.76:9200", "http://172.16.150.77:9200"]
    index => "logstash-nginx-%{+YYYY.MM.dd}"
  }
  }
}