---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-config
namespace: monitoring
labels:
k8s-app: fluentd-logzio
data:
fluent.conf: |
@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
@include kubernetes.conf
@include system.conf
@include conf.d/*.conf
@type logzio_buffered
@id out_logzio
endpoint_url "#{ENV['LOGZIO_LOG_LISTENER']}?token=#{ENV['LOGZIO_LOG_SHIPPING_TOKEN']}"
output_include_time true
output_include_tags true
# Set the buffer type to file to improve the reliability and reduce the memory consumption
@type "#{ENV['LOGZIO_BUFFER_TYPE']}"
path "#{ENV['LOGZIO_BUFFER_PATH']}"
# Set queue_full action to block because we want to pause gracefully
# in case of the off-the-limits load instead of throwing an exception
overflow_action "#{ENV['LOGZIO_OVERFLOW_ACTION']}"
# Set the chunk limit conservatively to avoid exceeding the GCL limit
# of 10MiB per write request.
chunk_limit_size "#{ENV['LOGZIO_CHUNK_LIMIT_SIZE']}"
# Cap the combined memory usage of this buffer and the one below to
# 2MiB/chunk * (6 + 2) chunks = 16 MiB
queue_limit_length "#{ENV['LOGZIO_QUEUE_LIMIT_LENGTH']}"
# Never wait more than 5 seconds before flushing logs in the non-error case.
flush_interval "#{ENV['LOGZIO_FLUSH_INTERVAL']}"
# Never wait longer than 30 seconds between retries.
retry_max_interval "#{ENV['LOGZIO_RETRY_MAX_INTERVAL']}"
# Disable the limit on the number of retries (retry forever).
retry_forever "#{ENV['LOGZIO_RETRY_FOREVER']}"
# Use multiple threads for processing.
flush_thread_count "#{ENV['LOGZIO_FLUSH_THREAD_COUNT']}"
kubernetes.conf: |
@include "#{ENV['AUDIT_LOG_FORMAT'] || 'audit'}.conf"
# This handles multiline exceptions automatically: https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions
@type detect_exceptions
remove_tag_prefix logzio
message log
languages all
multiline_flush_interval 0.1
@include "partial-#{ENV['CRI']}.conf"
# This adds type to the log && change key log to message
@type record_modifier
type k8s
message ${record["log"]}
remove_keys log
@type kubernetes_metadata
@id filter_kube_metadata
kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
@type dedot
de_dot true
de_dot_separator _
de_dot_nested true
system.conf: |
log_level "#{ENV['LOGZIO_LOG_LEVEL']}"
systemd.conf: |
# Logs from systemd-journal for interesting services.
# Logs from docker-systemd
# Logs from systemd-journal for interesting services.
audit.conf: |
# Example:
# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods"
# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
audit-json.conf: |
partial-docker.conf: |
# Concat docker cri partial log
# https://github.com/fluent-plugins-nursery/fluent-plugin-concat
# https://github.com/moby/moby/issues/34620#issuecomment-619369707
@type concat
key log
use_first_timestamp true
multiline_end_regexp /\n$/
separator ""
partial-containerd.conf: |
# Concat containerd cri partial log
# https://github.com/fluent/fluentd-kubernetes-daemonset/issues/412#issuecomment-636536767
@type concat
key log
use_first_timestamp true
partial_key logtag
partial_value P
separator ""