Skip to content

Instantly share code, notes, and snippets.

@liemle3893

liemle3893/application-log.conf

Last active April 28, 2020 02:55
Show Gist options
  • Save liemle3893/e29f26cee7558a04a00b27622ecfef7e to your computer and use it in GitHub Desktop.
Save liemle3893/e29f26cee7558a04a00b27622ecfef7e to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. liemle3893 revised this gist Apr 28, 2020. 3 changed files with 36 additions and 0 deletions.
    29 changes: 29 additions & 0 deletions application-log.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,29 @@
    [SERVICE]
    Flush 1
    Log_Level info
    Parsers_File parsers.conf

    [INPUT]
    Name tail
    # Log file will be placed at: /tmp/fluent-bit/logs/<app_name>/<file_name>.log
    Path /tmp/fluent-bit/logs/*/*.log
    Path_Key LogSource
    Tag stdout

    [FILTER]
    Name parser
    Match *
    Key_Name LogSource
    Parser app_name
    Reserve_Data true
    # Preserve_Key true

    [OUTPUT]
    Name file
    Match *
    Format plain
    Path output.txt

    [OUTPUT]
    Name stdout
    Match *
    5 changes: 5 additions & 0 deletions parsers.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -13,3 +13,8 @@
    Name dummy_test
    Format regex
    Regex ^(?<__attrs__NOMAD_DC>[^ ]+)/(?<__attrs__NOMAD_REGION>[^ ]+)/(?<__attrs__NOMAD_NAMESPACE>[^ ]+)/(?<__attrs__NOMAD_JOB_NAME>[^ ]+)/(?<__attrs__NOMAD_GROUP_NAME>[^ ]+)/(?<__attrs__NOMAD_TASK_NAME>[^ ]+)/(?<__attrs__NOMAD_ALLOC_ID>[^ ]+)/(?<__attrs__IP>.+)$

    [PARSER]
    Name app_name
    Format regex
    Regex ^.*/(?<APP_NAME>[^ ]+)/[^ ]+.log$
    2 changes: 2 additions & 0 deletions regex-filter.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -12,6 +12,8 @@
    Rename stream __stream__
    Rename time __time__
    Rename attrs __attrs__



    [FILTER]
    Name nest
  2. liemle3893 revised this gist Apr 22, 2020. 1 changed file with 46 additions and 0 deletions.
    46 changes: 46 additions & 0 deletions regex-filter.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,46 @@
    [SERVICE]
    Parsers_File parsers.conf

    [INPUT]
    Name dummy
    Tag dummy.data
    Dummy {"log":"level=info msg=ahii","stream":"stdout","attrs":{"tag":"dc1/global/dev-namespace/my-job/my-job-group/my-job-task/abcfe13/10.30.30.10"},"time":"2020-04-16T10:54:28.236573284Z"}

    [FILTER]
    Name modify
    Match dummy.*
    Rename stream __stream__
    Rename time __time__
    Rename attrs __attrs__

    [FILTER]
    Name nest
    Operation lift
    Match *
    Nested_under __attrs__
    Add_prefix __attrs__

    [FILTER]
    Name parser
    Match dummy.*
    Key_Name __attrs__tag
    Parser dummy_test

    [FILTER]
    Name rewrite_tag
    Match dummy.*
    Rule $__attrs__NOMAD_ALLOC_ID ^(.*)$ nomad true
    Emitter_Name nomad_reemitted
    Emitter_Storage.type memory

    [FILTER]
    Name nest
    Match *
    Operation nest
    Wildcard __attrs__*
    Remove_prefix __attrs__
    Nest_under __attrs__

    [OUTPUT]
    Name stdout
    Match *
  3. liemle3893 revised this gist Apr 22, 2020. 1 changed file with 5 additions and 0 deletions.
    5 changes: 5 additions & 0 deletions parsers.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -8,3 +8,8 @@
    # =============|==================|=================
    Decode_Field_As escaped_utf8 log do_next
    Decode_Field_As json log

    [PARSER]
    Name dummy_test
    Format regex
    Regex ^(?<__attrs__NOMAD_DC>[^ ]+)/(?<__attrs__NOMAD_REGION>[^ ]+)/(?<__attrs__NOMAD_NAMESPACE>[^ ]+)/(?<__attrs__NOMAD_JOB_NAME>[^ ]+)/(?<__attrs__NOMAD_GROUP_NAME>[^ ]+)/(?<__attrs__NOMAD_TASK_NAME>[^ ]+)/(?<__attrs__NOMAD_ALLOC_ID>[^ ]+)/(?<__attrs__IP>.+)$
  4. liemle3893 revised this gist Apr 22, 2020. 1 changed file with 10 additions and 0 deletions.
    10 changes: 10 additions & 0 deletions parsers.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,10 @@
    [PARSER]
    Name docker
    Format json
    Time_Key time
    Time_Format %Y-%m-%dT%H:%M:%S.%L
    Time_Keep On
    # Command | Decoder | Field | Optional Action
    # =============|==================|=================
    Decode_Field_As escaped_utf8 log do_next
    Decode_Field_As json log
  5. liemle3893 created this gist Apr 22, 2020.
    66 changes: 66 additions & 0 deletions fluentbit-example.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,66 @@
    [SERVICE]
    Flush 1
    Log_Level info
    Parsers_File parsers.conf

    [INPUT]
    NAME dummy
    Dummy {"log":"{\"ClientAddr\":\"10.30.83.2:51764\",\"ClientHost\":\"10.30.83.2\",\"ClientPort\":\"51764\",\"ClientUsername\":\"-\",\"DownstreamContentSize\":2,\"DownstreamStatus\":200,\"Duration\":271946,\"OriginContentSize\":2,\"OriginDuration\":44897,\"OriginStatus\":200,\"Overhead\":227049,\"RequestAddr\":\"10.30.83.2:38931\",\"RequestContentSize\":0,\"RequestCount\":76,\"RequestHost\":\"10.30.83.2\",\"RequestMethod\":\"GET\",\"RequestPath\":\"/ping\",\"RequestPort\":\"38931\",\"RequestProtocol\":\"HTTP/1.1\",\"RequestScheme\":\"http\",\"RetryAttempts\":0,\"RouterName\":\"ping@internal\",\"StartLocal\":\"2020-04-16T17:54:28.235942534+07:00\",\"StartUTC\":\"2020-04-16T10:54:28.235942534Z\",\"entryPointName\":\"traefik\",\"level\":\"info\",\"msg\":\"\",\"time\":\"2020-04-16T17:54:28+07:00\"}\n","stream":"stdout","attrs":{"NOMAD_ALLOC_ID":"458b35bb-e8d0-5584-e1bb-e86277cc06e7","NOMAD_DC":"QTSC","NOMAD_GROUP_NAME":"traefik","NOMAD_JOB_NAME":"ingress","NOMAD_NAMESPACE":"default","NOMAD_TASK_NAME":"traefik"},"time":"2020-04-16T10:54:28.236573284Z"}
    Tag docker
    Parser docker

    [FILTER]
    Name modify
    Match docker
    Rename stream __stream__
    Rename time __time__
    Rename attrs __attrs__

    [FILTER]
    Name nest
    Match docker
    Operation lift
    Nested_under log

    [FILTER]
    Name modify
    Match docker
    Rename __time__ time

    [FILTER]
    Name modify
    Match docker
    Rename time @timestamp

    # Traefik/HTTP endpoint specific. Exclude Request about ping/health check
    [FILTER]
    Name grep
    Match docker
    Exclude RequestPath /(ping|metrics|health(z?))

    # Rewrite logs tagged with docker to Nomad
    [FILTER]
    Name nest
    Operation lift
    Match *
    Nested_under __attrs__
    Add_prefix __attrs__

    [FILTER]
    Name rewrite_tag
    Match docker
    Rule $__attrs__NOMAD_ALLOC_ID ^(.*)$ nomad false
    Emitter_Name nomad_reemitted
    Emitter_Storage.type memory

    [FILTER]
    Name nest
    Match nomad
    Operation nest
    Wildcard __attrs__*
    Remove_prefix __attrs__
    Nest_under __attrs__

    [OUTPUT]
    Name stdout
    Match *