apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-config
  namespace: kube-system
data:
  fluent.conf: |
    <source>
      @type tail
      path /mnt/fluentd-logs/**/*.log
      pos_file /mnt/posfiles/kubernetes.pos
      read_from_head true
      tag kubernetes.*
      <parse>
        @type json
        time_type string
        time_format %iso8601
        keep_time_key true
      </parse>
    </source>

    <source>
      @type prometheus
      bind 0.0.0.0
      port 9110
      metrics_path /metrics
    </source>

    <source>
      @type prometheus_output_monitor
      interval 60
      <labels>
        hostname ${hostname}
      </labels>
    </source>

    <filter kubernetes.**>
      @type age
    </filter>

    <filter kubernetes.**>
      @type record_transformer
      <record>
        # s3_path is: ${application}/${component}/${namespace}/${environment}/${version}/${cont_name}
        s3_path     ${tag_parts[4]}/${tag_parts[5]}/${tag_parts[6]}/${tag_parts[7]}/${tag_parts[8]}/${tag_parts[9]}
        application ${tag_parts[4]}
        version     ${tag_parts[8]}
        cont_name   ${tag_parts[9]}
        pod         ${tag_parts[10]}
      </record>
    </filter>

    <filter kubernetes.**>
      @type prometheus
      <metric>
        name fluentd_num_records_total
        type counter
        desc The total number of incoming records
        <labels>
          application ${application}
          version     ${version}
          cont_name   ${cont_name}
          hostname    ${hostname}
        </labels>
      </metric>
    </filter>

    <match kubernetes.**>
      @type s3
      s3_bucket {{ index .ConfigItems "logging_s3_bucket" }}
      s3_region eu-central-1
      auto_create_bucket false
      check_bucket false
      <instance_profile_credentials>
        retries 10
      </instance_profile_credentials>
      s3_object_key_format kubernetes/${s3_path}/%Y/%m/%d/%H/${pod}/%Y%m%dT%H%M_%{index}.%{file_extension}
      <buffer time,s3_path,pod>
        @type file
        path /mnt/buffer/kubernetes/
        timekey 5m
        timekey_wait 1m
        timekey_use_utc true
        chunk_limit_size 4MB
        total_limit_size 1GB
        queued_chunks_limit_size 100
        flush_at_shutdown true
      </buffer>
      <format>
        @type csv
        fields time,log
      </format>
    </match>
