apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-log-collector-config-all
  namespace: prophecis
  labels:
    k8s-app: fluent-bit
data:
  # Configuration files: server, input, filters and output
  # ======================================================
  fluent-bit.conf: |
    [SERVICE]
        Flush         1   
        Log_Level     info
        Daemon        off
        Parsers_File  parsers.conf  
        HTTP_Server   On
        HTTP_Listen   0.0.0.0
        HTTP_Port     2020
    @INCLUDE training-log.conf
    @INCLUDE record_modifier.conf
    @INCLUDE output-elasticsearch.conf
  training-log.conf: |
    [INPUT]
        Name              tail
        Tag               training.*    
        Path              /job/*.log 
        Parser            syslog    
        DB                /job/training-log.db    
        Mem_Buf_Limit     5MB
        Skip_Long_Lines   On
        Refresh_Interval  10
        Key               line    
  record_modifier.conf: |
    [FILTER]
        Name    lua
        Match   training.*
        script  training.lua   
        call    cb_print    
        Type_int_key rindex time    
        Match training.*
    [FILTER]
        Name nest  
        Match training.*
        Operation nest
        Wildcard time
        Wildcard rindex
        Wildcard training_id 
        Nest_under meta  
  training.lua: |
    start_line_num = 1
    function cb_print(tag, timestamp, record)
       fname_strat_num = string.find(tag,".training")+1
       fname_end_num = string.find(tag,".",-1)
       fname =  string.sub(tag,fname_strat_num,-5)
   
       fline_name = "/job/"..fname.."-line.pos"
       print(fline_name) 
       -- check if file exists
       local file = io.open(fline_name, "r")
       -- if file not exists, create file and set default value
       if(file == nil)
       then
         print("file not exists")
         file = io.open(fline_name, "w+")
         file:write(start_line_num)
       end
       file:close()
       -- read current line num from line.pos
       file = io.open(fline_name, "r")
       local current = file:read()
       file:close()
       -- object which will be returned
       new_record = {}
       new_record["training_id"] = fname
       -- line num
       new_record["rindex"] = tonumber(current)
   
       fname_strat_num = string.find(tag,".training")+1
       fname_end_num = string.find(tag,".",-1)
       fname =  string.sub(tag,fname_strat_num,-5)
       new_record["training_id"] = fname

       -- current timestamp
       origin_time = string.format("%d", timestamp*1000)
       -- new_record["time"] = string.sub(origin_time,1,-8)
       new_record["time"] = tonumber(origin_time)
       -- other key from input
       for key, val in pairs(record) do
         new_record[key] = val
       end
       -- save back in line.pos
       file = io.open(fline_name, "w+")
       current = current + 1
       file:write(current)
       file:close()
       return 1, timestamp, new_record
    end
  output-elasticsearch.conf: |
    [OUTPUT]
        Name            es
        Match           *
        Host            ${FLUENT_ELASTICSEARCH_HOST}
        Port            ${FLUENT_ELASTICSEARCH_PORT}
        HTTP_User       ${FLUENT_ELASTICSEARCH_USER}
        HTTP_Passwd     ${FLUENT_ELASTICSEARCH_PASSWD}
        Index           dlaas_learner_data   
        #Logstash_Format On
        Replace_Dots    On
        Retry_Limit     False
        Type            logline    
        # Time_Key        time    

    [OUTPUT]
        Name  stdout
        Match *


  parsers.conf: |
    [PARSER]
        Name   apache
        Format regex
        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   apache2
        Format regex
        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   apache_error
        Format regex
        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$

    [PARSER]
        Name   nginx
        Format regex
        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name   json
        Format json
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z

    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On

    [PARSER]
        Name        syslog
        Format      regex
        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
        # Time_Format %b %d %H:%M:%S
        Time_Format %S%L
        Time_Keep   On
