frostmourne 案例filebeat 采集到ES或logstash

frostmourne 案例filebeat 采集到ES或logstash

#filebeat

- type: log
  tail_files: true
  scan_frequency: 5s
  backoff: 1s
  max_backoff: 10s
  paths:
      - /var/log/nginx/access.log
  fields:
    type: nginx
    ip: 172.16.3.226
  fields_under_root: true

#logstash

input {
        beats {
                host => '0.0.0.0'
                port => 5044 
        }
}
output{
  if [type] == "tomcat" {
    elasticsearch {
      hosts => ["http://172.16.3.225:9200","http://172.16.3.226:9200"，"http://172.16.3.227:9200"]
      index => "tomcat_log-%{ YYYY.MM.dd}"
#      user =>    xxx        # 這裡需要註意的是如果es配置了X-pack那麼就需要在這裡加上用戶密碼
#       password =>  xxx    
    }
    stdout{
      codec=>rubydebug
    }
  }
  else if [type] == "nginx" {
    elasticsearch {
      hosts => ["http://172.16.3.225:9200","http://172.16.3.226:9200"，"http://172.16.3.227:9200"]
      index => "nginx_log_-%{ YYYY.MM.dd}"
    }
    stdout{
      codec=>rubydebug
    }
  }

#logstash 解析

input {
        beats {
                host => '0.0.0.0'
                port => 5044 
        }
}
filter {
 if [type] == "access" {
    grok {
        match => {
            "message" => '(?<clientip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}) - (?<user>\S ) \[(?<timestamp>[^ ]  \ [0-9] )\] "(?<requesttype>[A-Z] ) (?<requesturl>[^
 ] ) HTTP/\d.\d" (?<status>\d ) (?<bodysize>\d ) "(?<url>\S )" "[^"] "'}
#移除不需要的字段
       remove_field => ["message","@version","path"] 
    }
    date {
        match => ["requesttime", "dd/MMM/yyyy:HH:mm:ss Z"]
        target => "@timestamp"
    }
  }
}
output{
  if [type] == "nginx" {
    elasticsearch {
      hosts => ["http://172.16.3.225:9200","http://172.16.3.226:9200"，"http://172.16.3.227:9200"]
      index => "nginx_log-%{ YYYY.MM.dd}"
    }
  }
  else if [type] == "tomcat" {
    elasticsearch {
      hosts => ["http://172.16.3.225:9200","http://172.16.3.226:9200"，"http://172.16.3.227:9200"]
      index => "tomcat_log-%{ YYYY.MM.dd}"
    }
  }
  else if [type] == "access" {
    elasticsearch {
      hosts => ["http://172.16.3.225:9200","http://172.16.3.226:9200"，"http://172.16.3.227:9200"]
      index => "access-%{ YYYY.MM.dd}"
    }
  }
    stdout{
      codec=>rubydebug
    }
}