---
kind: ConfigMap
apiVersion: v1
metadata:
  name: fluentd-es-config
  namespace: kube-system
  labels:
    k8s-app: fluentd-es
    version: v1.0
    addonmanager.kubernetes.io/mode: Reconcile
data:
  0.input.containers.conf: |-
    # CRI Log Example:
    # 2016-02-17T00:04:05.931087621Z stdout [info:2016-02-16T16:04:05.930-08:00] Some log text here
    <source>
      @type tail
      @id in_tail_containers_logs
      path /var/log/containers/*.log
      pos_file /var/log/es-containers.log.pos
      tag kubernetes.*
      read_from_head true
      <parse>
        @type multi_format
        <pattern>
          format json
          time_key time
          time_format %Y-%m-%dT%H:%M:%S.%NZ
        </pattern>
        <pattern>
          format /^(?<time>.+)\b(?<stream>stdout|stderr)\b(?<log>.*)$/
          time_format %Y-%m-%dT%H:%M:%S.%N%:z
        </pattern>
      </parse>
    </source>
  0.input.system.conf: |-
    # Example:
    # 2015-12-21 23:17:22,066 [salt.state       ][INFO    ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
    <source>
      @type tail
      @id in_tail_minion
      path /var/log/salt/minion
      pos_file /var/log/es-salt.pos
      tag salt
      <parse>
        @type regexp
        expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
        time_format %Y-%m-%d %H:%M:%S
      </parse>
    </source>

    # Example:
    # Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
    <source>
      @type tail
      @id in_tail_startupscript
      path /var/log/startupscript.log
      pos_file /var/log/es-startupscript.log.pos
      tag startupscript
      <parse>
        @type syslog
      </parse>
    </source>

    # Examples:
    # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
    # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
    <source>
      @type tail
      @id in_tail_docker
      path /var/log/docker.log
      pos_file /var/log/es-docker.log.pos
      tag docker
      <parse>
        @type regexp
        expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
      </parse>
    </source>

    # Example:
    # 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
    <source>
      @type tail
      @id in_tail_etcd
      # Not parsing this, because it doesn't have anything particularly useful to
      # parse out of it (like severities).
      path /var/log/etcd.log
      pos_file /var/log/es-etcd.log.pos
      tag etcd
      <parse>
        @type none
      </parse>
    </source>

    # Multi-line parsing is required for all the kube logs because very large log
    # statements, such as those that include entire object bodies, get split into
    # multiple lines by glog.

    # Example:
    # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
    <source>
      @type tail
      @id in_tail_kubelet
      multiline_flush_interval 5s
      path /var/log/kubelet.log
      pos_file /var/log/es-kubelet.log.pos
      tag kubelet
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
    <source>
      @type tail
      @id in_tail_kube_proxy
      multiline_flush_interval 5s
      path /var/log/kube-proxy.log
      pos_file /var/log/es-kube-proxy.log.pos
      tag kube-proxy
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
    <source>
      @type tail
      @id in_tail_kube_apiserver
      multiline_flush_interval 5s
      path /var/log/kube-apiserver.log
      pos_file /var/log/es-kube-apiserver.log.pos
      tag kube-apiserver
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
    <source>
      @type tail
      @id in_tail_kube_controller_manager
      multiline_flush_interval 5s
      path /var/log/kube-controller-manager.log
      pos_file /var/log/es-kube-controller-manager.log.pos
      tag kube-controller-manager
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
    <source>
      @type tail
      @id in_tail_kube_scheduler
      multiline_flush_interval 5s
      path /var/log/kube-scheduler.log
      pos_file /var/log/es-kube-scheduler.log.pos
      tag kube-scheduler
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # I1104 10:36:20.242766       5 rescheduler.go:73] Running Rescheduler
    <source>
      @type tail
      @id in_tail_rescheduler
      multiline_flush_interval 5s
      path /var/log/rescheduler.log
      pos_file /var/log/es-rescheduler.log.pos
      tag rescheduler
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
    <source>
      @type tail
      @id in_tail_glbc
      multiline_flush_interval 5s
      path /var/log/glbc.log
      pos_file /var/log/es-glbc.log.pos
      tag glbc
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
    <source>
      @type tail
      @id in_tail_cluster_autoscaler
      multiline_flush_interval 5s
      path /var/log/cluster-autoscaler.log
      pos_file /var/log/es-cluster-autoscaler.log.pos
      tag cluster-autoscaler
      <parse>
        @type multiline
        format_firstline /^\w\d{4}/
        format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
        time_format %m%d %H:%M:%S.%N
      </parse>
    </source>

    # Example:
    # 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
    # 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
    <source>
      @type tail
      @id in_tail_kube_apiserver_audit
      multiline_flush_interval 5s
      path /var/log/kubernetes/kube-apiserver-audit.log
      pos_file /var/log/kube-apiserver-audit.log.pos
      tag kube-apiserver-audit
      <parse>
        @type multiline
        format_firstline /^\S+\s+AUDIT:/
        # Fields must be explicitly captured by name to be parsed into the record.
        # Fields may not always be present, and order may change, so this just looks
        # for a list of key="\"quoted\" value" pairs separated by spaces.
        # Unknown fields are ignored.
        # Note: We can't separate query/response lines as format1/format2 because
        #       they don't always come one after the other for a given query.
        format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
        time_format %Y-%m-%dT%T.%L%Z
      </parse>
    </source>

  0.input.systemd.conf: |-
    # Logs from systemd-journal for interesting services.
    <source>
      @type systemd
      @id in_systemd_docker
      matches [{ "_SYSTEMD_UNIT": "docker.service" }]
      <storage>
        @type local
        persistent true
        path /var/log/fluentd-journald-docker.pos
      </storage>
      read_from_head true
      tag docker
    </source>

    <source>
      @type systemd
      @id in_systemd_kubelet
      matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
      <storage>
        @type local
        persistent true
        path /var/log/fluentd-journald-kubelet.pos
      </storage>
      read_from_head true
      tag kubelet
    </source>

    <source>
      @type systemd
      @id in_systemd_node_problem_detector
      matches [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }]
      <storage>
        @type local
        persistent true
        path /var/log/fluentd-journald-node-problem-detector.pos
      </storage>
      read_from_head true
      tag node-problem-detector
    </source>
  0.input.forward.conf: |-
    # Takes the messages sent over TCP
    <source>
      @type forward
    </source>
  0.input.monitoring.conf: |-
    # Prometheus Exporter Plugin
    # input plugin that exports metrics
    <source>
      @type prometheus
    </source>

    <source>
      @type monitor_agent
    </source>

    # input plugin that collects metrics from MonitorAgent
    <source>
      @type prometheus_monitor
      <labels>
        host ${hostname}
      </labels>
    </source>

    # input plugin that collects metrics for output plugin
    <source>
      @type prometheus_output_monitor
      <labels>
        host ${hostname}
      </labels>
    </source>

    # input plugin that collects metrics for in_tail plugin
    <source>
      @type prometheus_tail_monitor
      <labels>
        host ${hostname}
      </labels>
    </source>
  1.filter.kubernetes.conf: |-
    <filter kubernetes.**>
      @type parser
      key_name log
      #emit_invalid_record_to_error false
      <parse>
        @type multi_format
        <pattern>
          # Nginx Access Log Format:
          # log_format main '[$time_local] '
          #                 'remote_addr=$remote_addr '
          #                 'https=$https '
          #                 'host=$host '
          #                 'request="$request" '
          #                 'content_length=$content_length '
          #                 'request_id=$request_id '
          #                 'request_time=$request_time '
          #                 'referer=$http_referer '
          #                 'user_agent="$http_user_agent" '
          #                 'x_forwarded_for="$http_x_forwarded_for" '
          #                 'status=$status '
          #                 'bytes_sent=$bytes_sent '
          #                 'body_bytes_sent=$body_bytes_sent '
          #                 'upstream_addr=$upstream_addr '
          #                 'upstream_status=$upstream_status '
          #                 'upstream_response_time=$upstream_response_time '
          #                 'upstream_connect_time=$upstream_connect_time '
          #                 'upstream_header_time=$upstream_header_time';
          # Example:
          # remote_addr=10.244.0.1 https= host=10.244.0.170 request="GET /lvsheartbeat/heartbeat.gif HTTP/1.1" content_length=- request_id=b069864f2b9b4b8ecd45f6280ac31a20 request_time=0.001 referer=- user_agent="kube-probe/1.13" x_forwarded_for="-" status=200 bytes_sent=318 body_bytes_sent=35 upstream_addr=10.99.158.207:80 upstream_status=200 upstream_response_time=0.004 upstream_connect_time=0.000 upstream_header_time=0.004
          format /^\[(?<time>[^\]]+)\] remote_addr=(?<remote_addr>[^\s]+) https=(?<https>\w*) host=(?<host>[^\s]+) request="(?<method>\w+) (?<path>[^\s]+) (?<proto>[^\s]+)" content_length=(?<content_length>(\d+|-)) request_id=(?<request_id>[^\s]+) request_time=(?<request_time>(\d|\.)+) referer=(?<referer>[^\s]+) user_agent="(?<user_agent>[^"]+)" x_forwarded_for="(?<x_forwarded_for>[^"]+)" status=(?<status>\d{3}) bytes_sent=(?<bytes_sent>\d+) body_bytes_sent=(?<body_bytes_sent>\d+) upstream_addr=(?<upstream_addr>[^\s]+) upstream_status=(?<upstream_status>(\d{3}|-)) upstream_response_time=(?<upstream_response_time>((\d|\.)+|-)) upstream_connect_time=(?<upstream_connect_time>((\d|\.)+|-)) upstream_header_time=(?<upstream_header_time>((\d|\.)+|-))$/
          types content_length:integer,request_time:float,status:integer,bytes_sent:integer,body_bytes_sent:integer,upstream_status:integer,upstream_response_time:float,upstream_connect_time:float,upstream_header_time:float
          time_format %d/%b/%Y:%H:%M:%S %z
        </pattern>
        <pattern>
          # Nginx Error Log Example
          format /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)/
          types pid:integer,tid:integer
          time_format %Y/%m/%d %H:%M:%S
        </pattern>
        <pattern>
          format json
          time_key time
          time_format %Y/%m/%d %H:%M:%S.%L %Z
        </pattern>
        <pattern>
          format none
        </pattern>
      </parse>
    </filter>

    # Enriches records with Kubernetes metadata
    <filter kubernetes.**>
      @type kubernetes_metadata
    </filter>

  2.output.es.conf: |-
    <match **>
      @type elasticsearch
      @id out_es
      @log_level info
      include_tag_key true
      host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
      port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
      path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
      scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
      ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
      ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
      user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
      password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}"
      reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
      reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'false'}"
      logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
      logstash_format true
      type_name fluentd
      <buffer>
        flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
        flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
        chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
        queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
        retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
        retry_forever true
      </buffer>
    </match>
