Logstash - 转换日志

• 

  简述

  Logstash 提供了各种插件来转换已解析的日志。这些插件可以Add, Delete,和Update日志中的字段，以便在输出系统中更好地理解和查询。

  我们正在使用Mutate Plugin在输入日志的每一行中添加一个字段名称用户。
• 

  安装变异过滤器插件

  安装变异过滤器插件；我们可以使用以下命令。

  
  >Logstash-plugin install Logstash-filter-mutate
  

  logstash.conf

  在此配置文件中，在 Aggregate Plugin 之后添加 Mutate Plugin 以添加新字段。

  
  input {
     file {
        path => "C:/tpwork/logstash/bin/log/input.log"
     }
  } 
  filter {
     grok {
        match => [ "message", "%{LOGLEVEL:loglevel} -
           %{NOTSPACE:taskid} - %{NOTSPACE:logger} -
           %{WORD:label}( - %{INT:duration:int})?" ]
     }
     if [logger] == "TRANSACTION_START" {
        aggregate {
           task_id => "%{taskid}"
           code => "map['sql_duration'] = 0"
           map_action => "create"
        }
     }
     if [logger] == "SQL" {
        aggregate {
           task_id => "%{taskid}"
           code => "map['sql_duration'] ||= 0 ; 
              map['sql_duration'] += event.get('duration')"
        }
     }
     if [logger] == "TRANSACTION_END" {
        aggregate {
           task_id => "%{taskid}"
           code => "event.set('sql_duration', map['sql_duration'])"
           end_of_task => true
           timeout => 120
        }
     }
     mutate {
        add_field => {"user" => "cainiaoya.com"}
     }
  }
  output {
     file {
        path => "C:/tpwork/logstash/bin/log/output.log"
     }
  }
  

  运行 Logstash

  我们可以使用以下命令运行 Logstash。

  
  >logstash –f logstash.conf
  

  输入日志

  以下代码块显示了输入日志数据。

  
  INFO - 48566 - TRANSACTION_START - start
  INFO - 48566 - SQL - transaction1 - 320
  INFO - 48566 - SQL - transaction1 - 200
  INFO - 48566 - TRANSACTION_END - end
  

  输出日志

  您可以看到输出事件中有一个名为“user”的新字段。

  
  {
     "path":"C:/tpwork/logstash/bin/log/input.log",
     "@timestamp":"2016-12-25T19:55:37.383Z",
     "@version":"1",
     "host":"wcnlab-PC",
     "message":"NFO - 48566 - TRANSACTION_START - start\r",
     "user":"cainiaoya.com","tags":["_grokparsefailure"]
  }
  {
     "duration":320,"path":"C:/tpwork/logstash/bin/log/input.log",
     "@timestamp":"2016-12-25T19:55:37.383Z","loglevel":"INFO","logger":"SQL",
     "@version":"1","host":"wcnlab-PC","label":"transaction1",
     "message":" INFO - 48566 - SQL - transaction1 - 320\r",
     "user":"cainiaoya.com","taskid":"48566","tags":[]
  }
  {
     "duration":200,"path":"C:/tpwork/logstash/bin/log/input.log",
     "@timestamp":"2016-12-25T19:55:37.399Z","loglevel":"INFO",
     "logger":"SQL","@version":"1","host":"wcnlab-PC","label":"transaction1",
     "message":" INFO - 48566 - SQL - transaction1 - 200\r",
     "user":"cainiaoya.com","taskid":"48566","tags":[]
  }
  {
     "sql_duration":520,"path":"C:/tpwork/logstash/bin/log/input.log",
     "@timestamp":"2016-12-25T19:55:37.399Z","loglevel":"INFO",
     "logger":"TRANSACTION_END","@version":"1","host":"wcnlab-PC","label":"end",
     "message":" INFO - 48566 - TRANSACTION_END - end\r",
     "user":"cainiaoya.com","taskid":"48566","tags":[]
  }