input {
  file {
    path => "/etc/logstash/conf.d/logstash/elasticsearch_logs/elasticsearch.log"
    type => "elasticsearch"
    start_position => "beginning" 
    sincedb_path => "/dev/null"
    codec => multiline {
      pattern => "^\["
      negate => true
      what => "previous"
    }
  }
}

filter {
  if [type] == "elasticsearch" {
    grok {
      match => [ "message", "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{DATA:severity}%{SPACE}\]\[%{DATA:source}%{SPACE}\]%{SPACE}(?<message>(.|\r|\n)*)" ]
      overwrite => [ "message" ]
    }

    if "_grokparsefailure" not in [tags] {
      grok {  
        match => [
          "message", "^\[%{DATA:node}\] %{SPACE}\[%{DATA:index}\]%{SPACE}(?<short_message>(.|\r|\n)*)",
          "message", "^\[%{DATA:node}\]%{SPACE}(?<short_message>(.|\r|\n)*)" ]
        tag_on_failure => []
      }
    }
  }
}

output {
  elasticsearch {
            hosts => [ "localhost:9200"]
            index => "es-test-logs"
        }
  stdout { codec => rubydebug }
}