# NR Parser 2019.11.08 1735
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: newrelic
  labels:
    k8s-app: newrelic-logging
data:
  # Configuration files: server, input, filters and output
  # ======================================================
  fluent-bit.conf: |
    [SERVICE]
        Flush         1
        Log_Level     ${LOG_LEVEL}
        Daemon        off
        Parsers_File  parsers.conf
        HTTP_Server   On
        HTTP_Listen   0.0.0.0
        HTTP_Port     2020

    @INCLUDE input-kubernetes.conf
    @INCLUDE filter-kubernetes.conf
    @INCLUDE filter-nginx-custom.conf
    @INCLUDE filter-metadata.conf
    @INCLUDE output-newrelic.conf


  filter-kubernetes.conf: |
    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc.cluster.local:443
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off
        Labels              On
        Merge_Log           On
        Annotations         On
        Kube_Tag_Prefix     kube.
        Regex_Parser        k8s-custom-tag


  filter-nginx-custom.conf: |
    [FILTER]
        Name                parser
        Match               kube.${custom_nginx_container}.*
        Key_Name            log
        Parser              nginx-custom
        Reserve_Data        On 
        Preserve_Key        On

    [FILTER]
        Name                lua
        Match               kube.${custom_nginx_container}.*
        Script              nginx.lua
        Call                cb_nginx

  filter-nginx-custom-b.conf: |
    [FILTER]
        Name                parser
        Match               kube.${custom_nginx_container}.*
        Key_Name            nginx_log
        Parser              json-notime
        Reserve_Data        On 



  filter-metadata.conf: |
    [FILTER]
        Name   modify
        Match  kube.*
        Add    NR_CATEGORY  ${nr_category}
        Add    NR_SOURCE   ${nr_source} 
        Add    CLUSTER_NAME  ${clusterName}
        Copy   HOST       HOSTNAME


  input-kubernetes.conf: |
    [INPUT]
        Name              tail
        Path              ${PATH}
        Parser            docker-new
        DB                /var/log/flb_${nr_namespace}_kube.db
        Mem_Buf_Limit     70MB
        Skip_Long_Lines   On
        #Path_key          filename
        Refresh_Interval  5
        Tag_Regex         (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
        Tag               kube.<container_name>.<namespace_name>.<pod_name>.<docker_id>


  output-newrelic.conf: |
    [OUTPUT]
        Name  newrelic
        Match *
        licenseKey ${LICENSE_KEY}
        endpoint ${ENDPOINT}
        maxBufferSize ${BUFFER_SIZE}
        maxRecords ${MAX_RECORDS}

  parsers.conf: |
    [PARSER]
        Name   json-notime
        Format json

    [PARSER]
        Name   json
        Format json
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name        docker-new
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On
        Decode_Field_As   escaped_utf8 log 
        #Decode_Field_As      json log

    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On
        Decode_Field_As   escaped    log

    [PARSER]
        Name    k8s-custom-tag
        Format  regex
        Regex   (?<tag>[^.]+)?\.(?<container_name>.+)\.(?<namespace_name>[^_]+)\.(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?<docker_id>[a-z0-9]{64})$

    [PARSER]
        Name   nginx
        Format regex
        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<gzip>[^ ]*)")

    [PARSER]
        Name    kube-custom
        Format  regex
        Regex   (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$

    [PARSER]
        Name      nginx-custom
        Format    regex
        #Regex     ^([^\]]*) \[(?<nginx_meta>[^\]]*)] (?<nginx_log>.*)$
        #Regex     (|LOG_LINE|) (?<nginx_time>[^\]]*) \[(?<nginx_meta>[^\]]*)] (?<nginx_log>.*)$
        #Regex      (|LOG_LINE|) (?<nginx_time>[^\]]*) \[(?<nginx_meta_origin>[^:]*):(?<nginx_meta_session_cookie_id>[^:]*):REQUEST ID=(?<nginx_meta_request_id>[^\]]*)\](?<nginx_log>.*)$
        Regex      (|LOG_LINE|) (?<nginx_time>[^\]]*) \[(?<nginx_meta>[^\]]*)\](?<nginx_log>.*)$
        #Regex      (?<ng_header>|LOG_LINE|) (?<ng_time>[^\]]*) \[(?<nginx_origin>[^:]*):(?<session_cookie_id>[^:]*):REQUEST ID=(?<nginx_request_id>[^\]]*)\] { host: \"(?<ng_host>[^\"]*)\", remote_addr: \"(?<remote_addr>[^\"]*)\", request: \"(?<request_method>\w*) (?<request_url>\S*) (?<request_proto>\S*)\", status: (?<status>[^,]*), referer: \"(?<referer>[^\"]*)\", request_time: (?<request_time>[^\,]*), connection: \"(?<connection>[^\"]*)\", connection_requests: \"(?<connection_request>[^\"]*)\", location: \"(?<location>[^\"]*)\", upstream_location_response: \"(?<upstream_location_response>[^\"]*)\", bytes_sent: \"(?<bytes_sent>[^\"]*)\", client_ip: \"(?<client_ip>[^\"]*)\", x_dynaTrace: \"(?<x_dynaTrace>[^\"]*)\" }$
        #Decode_Field_As   json    ng_log



    [PARSER]
        Name   apache
        Format regex
        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   apache2
        Format regex
        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   apache_error
        Format regex
        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$

    [PARSER]
        Name        k8s-nginx-ingress
        Format      regex
        Regex       ^(?<host>[^ ]*) - \[(?<real_ip>[^ ]*)\] - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<last>[^$]*)
        Time_Key    time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name        docker-daemon
        Format      regex
        Regex       time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<msg>[^ ].*)"
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On

    [PARSER]
        Name        syslog-rfc5424
        Format      regex
        Regex       ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On

    [PARSER]
        Name        syslog-rfc3164-local
        Format      regex
        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
        Time_Key    time
        Time_Format %b %d %H:%M:%S
        Time_Keep   On

    [PARSER]
        Name        syslog-rfc3164
        Format      regex
        Regex       /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
        Time_Key    time
        Time_Format %b %d %H:%M:%S
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On

    [PARSER]
        Name    mongodb
        Format  regex
        Regex   ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On
        Time_Key time

    [PARSER]
        # http://rubular.com/r/tjUt3Awgg4
        Name cri
        Format regex
        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

    [PARSER] 
        Name tomcat-access
        Format regex
        Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>\d*) (?<time_taken>\d*) (?<bytes_sent>\d+)$
        Time_Key   time
        Time_Format %d/%b/%Y:%H:%M:%S %z

        
  nginx.lua: |
        -- Print record to the standard output
        function cb_print(tag, timestamp, record)
           output = tag .. ":  [" .. string.format("%f", timestamp) .. ", { "

           for key, val in pairs(record) do
              output = output .. string.format(" %s => %s,", key, val)
           end

           output = string.sub(output,1,-2) .. " }]"
           print(output)

           -- Record not modified so 'code' return value is 0 (first parameter)
           return 0, 0, 0
        end

        function trim(s)
        end

        function cb_nginx(tag, timestamp, record)
           new_record = record
           new_record["nr_tag"] = tag
           log_source = record["stream"] 
           -- if log_source then print("stream ->" .. log_source) end
           if log_source and log_source:find("stdout") then 
            log_line = record["nginx_log"]
            if log_line then log_line = log_line:gsub("^[%s{]*(.-)[}%s]*$","%1") end

            if log_line then
               -- print (log_line)
               ram = {}
               local pattern = string.format("([^%s]+)", ",")
               local pattern_b =  "([^:]+):(.*)"
               log_line:gsub(pattern, function(c)
                   --print("c:" .. c)
                   local key,value = c:match(pattern_b)
                   if key then
                     key = key:match("^%s*(.-)%s*$")
                     -- if value then  value = value:gsub("\"*([^\"]-)\"*$","%1") end
                     if value then  value = value:gsub("^[\"%s]*(.-)[\"%s]*$","%1") end
                     if not value then  value = "-" end
                     ram[key] =  value
                   end
               end)
               -- remove the old value
               new_record["nginx_log"] = nil
               new_record["nginx"] = ram
            end
           end
           return 1, timestamp, new_record
        end

        -- Drop the record
        function cb_drop(tag, timestamp, record)
           return -1, 0, 0
        end

        -- Compose a new JSON map and report it
        function cb_replace(tag, timestamp, record)
           -- Record modified, so 'code' return value (first parameter) is 1
           new_record = {}
           new_record["new"] = 12345
           new_record["old"] = record
           return 1, timestamp, new_record
        end


