input {
  file {
    mode => "read"
    ignore_older => "604800"
    path => "/apps/datalake/env/data/inbound/ALPHAPROJECT_007/ELK/insight_forge/reportArchive/*.csv"
    db_path => "/apps/datalake/env/data/inbound/ALPHAPROJECT_007/ELK/insight_forge/reportArchive/db_dir/db.log"
    db_clean_after => "365"
    file_completed_action => "log"
    file_completed_log_path => "/apps/datalake/env/data/inbound/ALPHAPROJECT_007/ELK/insight_forge/reportArchive/db_dir/completed_files.log"
  }
}
filter {
  csv {
     separator => ","
      columns => ["FORGE_ID", "INSIGHT_TYPE", "DATA_SOURCE", "TREND_ANALYSIS", "METRIC_VALUE", "TIME_STAMP", "CONFIDENCE_SCORE", "IMPACT_FACTOR"]
  }

mutate {
      convert =>{"METRIC_VALUE" => "integer"}
      convert =>{"IMPACT_FACTOR" => "integer"}
      convert =>{"CONFIDENCE_SCORE" => "integer"}
      remove_field => ["path","host","message"]
   }

}
output {
   elasticsearch {
     user => "ALPHAPROJECT_007-elastic"
     password =>"secretcypher23"
     hosts => "https://sgdev2301.sg.dev.net:7340"
     index => "insight_forge_data"
     document_id => "%{FORGE_ID}-%{INSIGHT_TYPE}"
     ssl => true
     ssl_certificate_verification => false
     keystore => "/apps/ELK/logstash/cert/cacerts"
     keystore_password => "keystorepass"
     truststore => "/apps/ELK/logstash/cert/cacerts"
     truststore_password => "keystorepass"
  }
stdout {}
}