##############
#  INPUT 
##############
input {
  syslog { tags => ['syslog'] }
}



##############
#  FILTER
##############
filter {

  if "syslog" in [tags] {

      # convert any unicode 2028 back into new line, supports multiline exceptions
      # cannot get gsub to work, at least on logstash 2.x
      #mutate {
      #  gsub => [ "[@message]", "\\u2028", "\n"]
      #}

      # ruby replacement of unicode working under logstash 2.x
      ruby {
        code => "
        event.get('message').gsub!(/u2028/,%Q[\n])
        "
      }

    # https://docs.cloudfoundry.org/devguide/deploy-apps/streaming-logs.html
    # Look for major CF type: API,STG,RTR,LGR,APP,SSH,CELL
    grok {
      match => { "message" => "(?:%{INT:syslog_length} )?%{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(?:%{NOTSPACE:syslog5424_app}|-) +(?:%{NOTSPACE:syslog5424_proc}|-) +(?:%{NOTSPACE}|-) +(?:%{NOTSPACE}|-) +%{GREEDYDATA:syslog5424_msg}" }
      break_on_match => false
      add_field => [ "[@metadata][found_cf_match]", "true" ]
    }

    # look for cloud foundry log type, instance
    grok {
      match => { "syslog5424_proc" => "^\[%{DATA:cf_log_type}((/%{DATA})?)(%{INT:cf_instance})?\]$" }
      remove_tag => ["_grokparsefailure"]
      break_on_match => false
    }


    # it is from CF, look for major type
    if [@metadata][found_cf_match] == "true" {


      # from go router
      if [cf_log_type] == "RTR" {


        grok {
          match => { "syslog5424_msg" => "\[%{TIMESTAMP_ISO8601:rtr_ts}\] (\\)?\"%{DATA:HTTPMETHOD} %{DATA:URL} %{DATA:HTTPVER}(\\)?\" %{INT:HTTPCODE} %{INT:bytesreq} %{INT:bytessent} (\\)?\"%{DATA}(\\)?\" (\\)?\"%{DATA:useragent}(\\)?\" \"?%{DATA:IP}:%{NUMBER:PORT}\"? %{GREEDYDATA:message}" }
          remove_field => [ "syslog5424_msg" ]
          overwrite => [ "message" ]
          add_field => [ "use_index", "RTR" ]
          add_field => [ "[@metadata][processed]", "true" ]
          remove_tag => ["_grokparsefailure", "_grokparsefailure_sysloginput" ]
          break_on_match => false
        }

      } # RTR



      # from CF application
      else if [cf_log_type] == "APP" {


        # look for enhanced format from logback with multiline exceptions
        grok {
          match => { "syslog5424_msg" => "%{TIMESTAMP_ISO8601:app_ts} %{DATA:log_level} %{DATA:threadname} %{DATA:classname}:%{INT:linenumber} \[%{DATA:kvpairs}\] %{DATA:message}( MULTIEXCEPTION %{GREEDYDATA:multiexception})" }
          remove_field => [ "syslog5424_msg" ]
          overwrite => [ "message" ]
          add_field => [ "use_index", "APP_ENHANCED" ]
          add_field => [ "[@metadata][processed]", "true" ]
          remove_tag => ["_grokparsefailure", "_grokparsefailure_sysloginput" ]
          break_on_match => false
        }

        # extract key/value pairs
        if "_grokparsefailure" not in [tags] {

          # pull out known params from between initial brackets
          kv {
            source => "kvpairs"
            prefix => "kv_"
            # 'trim' in 2.x
            # became 'trim_value' in 5.3
            # removing so we can stay generic for this example
            #trim_value => " "
            include_keys => [ "app_name","app_version","instance_id" ]
            field_split => ";"
            value_split => "="
           } # kv
           mutate {
             remove_field => [ "kvpairs" ]
           }

        }

      } # if from APP


    } # if CF match



    # if not processed by any other means, then capture
    if [@metadata][processed] != "true" {

      # debug
      #ruby { code => "File.open('/tmp/logstashdebug.log','a') { |f| f.puts '!!!APP!!!' + event.get('syslog5424_msg') }" }

      grok {
        match => { "syslog5424_msg" => "%{GREEDYDATA:message}" }
        remove_field => [ "syslog5424_msg" ]
        add_field => [ "use_index", "SYSLOG" ]
        add_field => [ "[@metadata][processed]", "true" ]
        remove_tag => ["_grokparsefailure", "_grokparsefailure_sysloginput" ]
        overwrite => ["message"]
      }

    }


  } # is syslog

} # filter

##############
#  OUTPUT
##############
output {
  stdout { codec => rubydebug }

# sends to ElasticSearch daily index: RTR-* or APP_ENHANCED-* or SYSLOG-*
#  elasticsearch { 
#    hosts => "127.0.0.1:9200"
#    index => "%{use_index}-%{+YYYY.MM.dd}"
#  }

}