apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  labels:
    k8s-app: fluent-bit
data:
  # Configuration files: server, input, filters and output
  # ======================================================
  fluent-bit.conf: |
    [SERVICE]
        Flush         1
        Log_Level     info
        Daemon        off
        Parsers_File  parsers.conf
        HTTP_Server   On
        HTTP_Listen   0.0.0.0
        HTTP_Port     2020

    @INCLUDE input-kubernetes.conf
    @INCLUDE filter-kubernetes.conf
    @INCLUDE output-elasticsearch.conf

  input-kubernetes.conf: |
    [INPUT]
        Name              tail
        Tag               app.*
        Path              /var/log/containers/app-*.log
        Parser            docker
        DB                /var/log/flb_tail_app.db
        Mem_Buf_Limit     5MB
        Skip_Long_Lines   On
        Refresh_Interval  60
        Ignore_Older      7d

    [INPUT]
        Name              tail
        Tag               images.*
        Path              /var/log/containers/images-*.log
        Parser            docker
        DB                /var/log/flb_tail_images.db
        Mem_Buf_Limit     5MB
        Skip_Long_Lines   On
        Refresh_Interval  60
        Ignore_Older      7d

  filter-kubernetes.conf: |
    [FILTER]
        Name                kubernetes
        Match               app.*_log-app-container-*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     app.var.log.containers.
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off
        Merge_Parser        app_log_crio

    [FILTER]
        Name                kubernetes
        Match               app.*_log-*-apache-access-*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     app.var.log.containers.
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off
        Merge_Parser        apache2_crio

    [FILTER]
        Name                kubernetes
        Match               app.*_log-*-apache-error-*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     app.var.log.containers.
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off
        Merge_Parser        apache_error_crio


    [FILTER]
        Name                kubernetes
        Match               images.*_log-*-apache-access-*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     images.var.log.containers.
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off
        Merge_Parser        apache2_crio

    [FILTER]
        Name                kubernetes
        Match               images.*_log-*-apache-error-*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     images.var.log.containers.
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off
        Merge_Parser        apache_error_crio

  output-elasticsearch.conf: |
    #[OUTPUT]
    #    Name            file
    #    Match           images.*_www-timecard-ru_*
    #    Format          out_file
    #    Path            /logoutput

    [OUTPUT]
        Name            es
        Match           app.*_log-app-container-*
        Host            ${FLUENT_ELASTICSEARCH_HOST}
        Port            ${FLUENT_ELASTICSEARCH_PORT}
        Logstash_Format On
        Logstash_Prefix app
        Replace_Dots    On
        Retry_Limit     1
        Trace_Error     On

    [OUTPUT]
        Name            es
        Match           *-apache-*
        Host            ${FLUENT_ELASTICSEARCH_HOST}
        Port            ${FLUENT_ELASTICSEARCH_PORT}
        Logstash_Format On
        Logstash_Prefix apache
        Replace_Dots    On
        Retry_Limit     1
        Trace_Error     On

  parsers.conf: |
    [PARSER]
        Name   apache
        Format regex
        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   apache2_docker
        Format regex
        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   apache2_crio
        Format regex
        Regex  ^(?<crio_log_time>.+) (?<crio_log_stream>stdout|stderr) (?<crio_log_tag>F|P) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?
        Time_Keep   On
        Time_Key    crio_log_time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

    [PARSER]
        Name   apache_error_docker
        Format regex
        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<apache_error_pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$

    [PARSER]
        Name   apache_error_crio
        Format regex
        # Regex  ^(?<crio_log_time>.+) (?<crio_log_stream>stdout|stderr) (?<crio_log_tag>F|P) \[(?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[\d]*)\:tid (?<tid>[\d]*)\])? (?<message>.*)
        Regex  ^(?<crio_log_time>.+) (?<crio_log_stream>stdout|stderr) (?<crio_log_tag>F|P) \[(?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<apache_error_pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
        Time_Keep   On
        Time_Key    crio_log_time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

    [PARSER]
        Name   nginx
        Format regex
        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   json
        Format json
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On

    [PARSER]
        # http://rubular.com/r/tjUt3Awgg4
        Name cri
        Format regex
        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

    [PARSER]
        Name        syslog
        Format      regex
        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
        Time_Key    time
        Time_Format %b %d %H:%M:%S

    [PARSER]
        Name        app_log_docker
        Format      regex
        # Regex       ^(?<datetime>[^\[]*)\[(?<pid>\d+)\] (?<log_severity>(DEBUG|WARNING|ERROR|PANIC)):(?<pretty_function>[^\[]+)\[(?<line_number>\d+)\] *(?<log_message>.*)$
        Regex       ^(?<datetime>[^\[]*)\[(?<pid>\d+)\] (?<log_severity>(DEBUG|INFO|WARNING|ERROR|PANIC)):(?<pretty_function>[^\[]+)\[(?<line_number>\d+)\] *(?<log_message>(?>chrono duration = (?<chrono_duration>\d+) microseconds|.*))$
        Types       chrono_duration:integer line_number:integer pid:integer
        Time_Key    time
        Time_Format %b %d %Y %H:%M:%S
        Time_Keep   On

    [PARSER]
        Name        app_log_crio
        Format      regex
        # Regex       ^(?<datetime>[^\[]*)\[(?<pid>\d+)\] (?<log_severity>(DEBUG|WARNING|ERROR|PANIC)):(?<pretty_function>[^\[]+)\[(?<line_number>\d+)\] *(?<log_message>.*)$
        Regex       ^(?<crio_log_time>.+) (?<crio_log_stream>stdout|stderr) (?<crio_log_tag>F|P) (?<datetime>[^\[]*)\[(?<pid>\d+)\] (?<log_severity>(DEBUG|INFO|WARNING|ERROR|PANIC)):(?<pretty_function>[^\[]+)\[(?<line_number>\d+)\] *(?<log_message>(?>chrono duration = (?<chrono_duration>\d+) microseconds|.*))$
        Types       chrono_duration:integer line_number:integer pid:integer
        Time_Keep   On
        Time_Key    crio_log_time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
