apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-config
  namespace: zcp-system
  labels:
    app: fluentd
data:
  fluentd-aggregator.conf: |
    # Receive from the forwarder
    @include forward-input.conf
    @include kubernetes-filter.conf
    # Send to storage
    @include output.conf
  fluentd.conf: |
    @include "#{ENV['FLUENTD_CONFIG'] || 'fluentd-standalone.conf'}"
  forward-input.conf: |
    <system>
      # equal to -qq option
      log_level info
    </system>
    <source>
      @type forward
      port 24224
      bind 0.0.0.0
    </source>
  kubernetes-filter.conf: |
    <filter kube.**>
      @type record_transformer
      enable_ruby true
      <record>
        new_index ${record["kubernetes"]["namespace_name"]}-${time.strftime('%Y.%m.%d')}
      </record>
    </filter>
    <filter kube.**>
      @type concat
      key log
      multiline_start_regexp /(?<apptime>^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}|^\d{2}:\d{2}:\d{2}.\d{3}|^\d{4}\d{2}\d{2}\s\d{2}:\d{2}:\d{2}.\d{3})/
      flush_interval 1
      timeout_label @PLATFORM
    </filter>
    <match kube.**>
      @type relabel
      @label @PLATFORM
    </match>
    <filter app.**>
      @type record_transformer
      enable_ruby true
      <record>
        new_index ${record["kubernetes"]["kubernetes.namespace_name"]}-${time.strftime('%Y.%m.%d')}
      </record>
    </filter>
    <filter app.**>
      @type concat
      key log
      multiline_start_regexp /(?<apptime>^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}|^\d{2}:\d{2}:\d{2}.\d{3}|^\d{4}\d{2}\d{2}\s\d{2}:\d{2}:\d{2}.\d{3})/
      flush_interval 1
      timeout_label @APP
    </filter>
    <match app.**>
      @type relabel
      @label @APP
    </match>
    <match ingress.**>
      @type rewrite_tag_filter
      <rule>
        key     log
        pattern /^\s*$/
        tag clear
      </rule>
      <rule>
        key     log
        pattern /{\".*}/
        tag     access.${tag}
      </rule>
      <rule>
        key     log
        pattern /.*/
        tag     info.${tag}
      </rule>
    </match>
    <match clear>
      @type null
    </match>
    <filter access.ingress.**>
      @type parser
      key_name log
      <parse>
        @type json
      </parse>
      reserve_data true
    </filter>
    <filter info.ingress.**>
      @type concat
      key log
      multiline_start_regexp /((?<level>^[IE])?(?<apptime>(\d{4} \d{2}:\d{2}:\d{2}.\d{6})|(\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2})))/
      flush_interval 1
      timeout_label @INGRESS
    </filter>
    <filter *.ingress.**>
      @type record_transformer
      <record>
        log_type ${tag_prefix[0]}
      </record>
    </filter>
    <match *.ingress.**>
      @type relabel
      @label @INGRESS
    </match>
  output.conf: |
    <label @PLATFORM>
      <match kube.**>
        @type elasticsearch
        @log_level info
        include_tag_key true
        # Replace with the host/port to your Elasticsearch cluster.
        # This assumes a service `elasticsearch` exists in the default namespace
        host elasticsearch
        port 9200
        scheme http
        ssl_verify false
        logstash_format true
        # logstash_prefix fluentd-kubernetes_namespace_name
        template_file /fluentd/etc/application-log.json
        template_name application-log.json
        target_index_key new_index
        buffer_chunk_limit 2M
        buffer_queue_limit 32
        flush_interval 10s
        max_retry_wait 30
        disable_retry_limit
        num_threads 8
      </match>
    </label>
    <label @APP>
      <match **>
        @type elasticsearch
        @log_level info
        include_tag_key true
        # Replace with the host/port to your Elasticsearch cluster.
        # This assumes a service `elasticsearch` exists in the default namespace
        host elasticsearch
        port 9200
        scheme http
        ssl_verify false
        logstash_format true
        logstash_prefix application
        template_file /fluentd/etc/application-log.json
        template_name application-log.json
    target_index_key new_index
        buffer_chunk_limit 2M
        buffer_queue_limit 32
        flush_interval 2s
        max_retry_wait 30
        disable_retry_limit
        num_threads 8
      </match>
    </label>
    <match kubemeta.**>
      @type elasticsearch
      @log_level info
      include_tag_key true
      # Replace with the host/port to your Elasticsearch cluster.
      # This assumes a service `elasticsearch` exists in the default namespace
      host elasticsearch
      port 9200
      scheme http
      ssl_verify false
      logstash_format true
      logstash_prefix kubemeta
      template_file /fluentd/etc/application-log.json
      template_name application-log.json
      buffer_chunk_limit 2M
      buffer_queue_limit 32
      flush_interval 2s
      max_retry_wait 30
      disable_retry_limit
      num_threads 8
    </match>
    <label @INGRESS>
      <match *.ingress.**>
        @type elasticsearch
        @log_level info
        include_tag_key true
        # Replace with the host/port to your Elasticsearch cluster.
        # This assumes a service `elasticsearch` exists in the default namespace
        host elasticsearch
        port 9200
        scheme http
        ssl_verify false
        logstash_format true
        logstash_prefix ingress
        template_file /fluentd/etc/application-log.json
        template_name application-log.json
        buffer_chunk_limit 2M
        buffer_queue_limit 32
        flush_interval 2s
        max_retry_wait 30
        disable_retry_limit
        num_threads 8
      </match>
    </label>
  application-log.json: |
    {
      "template" : "*",
      "version" : 1,
      "order" : 10000,
      "settings" : {
        "index.refresh_interval" : "5s",
        "number_of_shards": 5,
        "number_of_replicas": 1
      },
      "mappings" : {
        "_default_" : {
          "dynamic_templates" : [ {
            "message_field" : {
              "path_match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "text",
                "norms" : false
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "text", "norms" : false,
                "fields" : {
                  "keyword" : {
                    "type": "keyword",
                    "ignore_above": 8191
                  }
                }
              }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date" },
            "@version": { "type": "keyword" },
            "geoip"  : {
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip" },
                "location" : { "type" : "geo_point" },
                "latitude" : { "type" : "half_float" },
                "longitude" : { "type" : "half_float" }
              }
            }
          }
        }
      }
    }