Skip to content

Instantly share code, notes, and snippets.

@NiceGuyIT

NiceGuyIT/README.md

Last active October 28, 2024 08:16
Show Gist options
  • Save NiceGuyIT/58dd4d553fe3017cbfc3f98c2fbdbc93 to your computer and use it in GitHub Desktop.
Save NiceGuyIT/58dd4d553fe3017cbfc3f98c2fbdbc93 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. NiceGuyIT renamed this gist Jan 27, 2018. 1 changed file with 0 additions and 0 deletions.
    0 nginx-json-elk.md → README.md
    View file
    File renamed without changes.
  2. NiceGuyIT revised this gist Jan 27, 2018. 3 changed files with 123 additions and 92 deletions.
    67 changes: 67 additions & 0 deletions filter-nginx.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,67 @@
    # Process JSON documents output by nginx.
    filter {

    if "nginx" in [tags] {
    # nginx doesn't log the http version, only the protocol.
    # i.e. HTTP/1.1, HTTP/2
    grok {
    match => {
    "[nginx][access][http_protocol]" => "HTTP/%{NUMBER:[nginx][access][http_version]}"
    }
    }

    # @timestamp is when filebeat reads the event.
    mutate {
    add_field => {
    "[nginx][read_timestamp]" => "%{@timestamp}"
    }
    }

    # msec has millisecond resolution.
    date {
    match => [
    "[nginx][time][msec]",
    "UNIX"
    ]
    target => "@timestamp"
    }

    # Requires geoip plugin
    geoip {
    source => "[nginx][access][remote_ip]"
    target => "[nginx][access][geoip]"
    }

    # Requires user-agent plugin
    useragent {
    source => "[nginx][access][agent]"
    target => "[nginx][access][user_agent]"
    }

    } else if [source] =~ '\/nginx\/error' {

    # nginx error_log
    grok {
    match => {
    "message" => "%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"
    }
    }

    # @timestamp is when filebeat reads the event.
    mutate {
    add_field => {
    "[nginx][read_timestamp]" => "%{@timestamp}"
    }
    }

    # For error logs
    date {
    match => [
    "[nginx][error][time]",
    "YYYY/MM/dd H:m:s"
    ]
    target => "@timestamp"
    }

    }
    }
    52 changes: 52 additions & 0 deletions log-json.conf
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,52 @@
    # Match the format for Elasticsearch
    log_format json escape=json
    '{ '
    '"fileset": { '
    '"module": "nginx", '
    '"name": "access" '
    '}, '
    '"nginx": { '
    '"access": { '
    '"remote_ip": "$remote_addr", '
    '"user_name": "$remote_user", '
    '"time": "$time_local", '
    '"method": "$request_method", '
    '"host": "$host", '
    '"url": "$request_uri", '
    '"http_protocol": "$server_protocol", '
    '"response_code": "$status", '
    '"body_sent": { '
    '"bytes": "$body_bytes_sent" '
    '}, '
    '"referrer": "$http_referer", '
    '"agent": "$http_user_agent" '
    '}, '
    '"request": "$request", '
    '"connection": "$connection", '
    '"pipe": "$pipe", '
    '"connection_requests": "$connection_requests", '
    '"time": { '
    '"iso8601": "$time_iso8601", '
    '"msec": "$msec", '
    '"request": "$request_time" '
    '}, '
    '"bytes": { '
    '"request_length": "$request_length", '
    '"body_sent": "$body_bytes_sent", '
    '"sent": "$bytes_sent" '
    '}, '
    '"http": { '
    '"x_forwarded_for": "$http_x_forwarded_for", '
    '"x_forwarded_proto": "$http_x_forwarded_proto", '
    '"x_real_ip": "$http_x_real_ip", '
    '"x_scheme": "$http_x_scheme" '
    '}, '
    '"upstream": { '
    '"addr": "$upstream_addr", '
    '"status": "$upstream_status", '
    '"response_time": "$upstream_response_time", '
    '"connect_time": "$upstream_connect_time", '
    '"header_time": "$upstream_header_time" '
    '} '
    '} '
    '}';
    96 changes: 4 additions & 92 deletions nginx-json-elk.md
    View file
    Original file line number Diff line number Diff line change
    @@ -45,7 +45,7 @@ This [article](https://www.elastic.co/blog/should-i-use-logstash-or-elasticsearc

    # Filebeat setup

    This will load the templates and pipelines into Elasticsearch and the dashboards into Kibana.
    This will load the templates and fields into Elasticsearch and the dashboards into Kibana. The ingest pipelines are loaded with `filebeat --setup` which proceeds to run filebeat after the setup. `filebeat setup` does not load the pipelines. I assume this is because the pipelines are relevent only when filebeat is connected directly to Elasticsearch.

    ```bash
    filebeat setup -e \
    @@ -57,61 +57,10 @@ filebeat setup -e \

    # nginx

    Everything is wrapped in a single "nginx" element/object to match ES's "nginx" mapping.
    Everything is wrapped in a single "nginx" element/object to match ES's "nginx" mapping. Put `log-json.conf` in the nginx directory.

    ```
    # Match the format for Elasticsearch
    log_format json escape=json
    '{ '
    '"fileset": { '
    '"module": "nginx", '
    '"name": "access" '
    '}, '
    '"nginx": { '
    '"access": { '
    '"remote_ip": "$remote_addr", '
    '"user_name": "$remote_user", '
    '"time": "$time_local", '
    '"method": "$request_method", '
    '"url": "$request_uri", '
    '"http_protocol": "$server_protocol", '
    '"response_code": "$status", '
    '"body_sent": { '
    '"bytes": "$body_bytes_sent"'
    ' }, '
    '"referrer": "$http_referer", '
    '"agent": "$http_user_agent" '
    ' }, '
    '"request": "$request", '
    '"connection": "$connection", '
    '"pipe": "$pipe", '
    '"connection_requests": "$connection_requests", '
    '"time": { '
    '"iso8601": "$time_iso8601", '
    '"msec": "$msec", '
    '"request": "$request_time"'
    ' }, '
    '"bytes": { '
    '"request_length": "$request_length", '
    '"body_sent": "$body_bytes_sent", '
    '"sent": "$bytes_sent"'
    ' }, '
    '"http": { '
    '"x_forwarded_for": "$http_x_forwarded_for", '
    '"x_forwarded_proto": "$http_x_forwarded_proto", '
    '"x_real_ip": "$http_x_real_ip", '
    '"x_scheme": "$http_x_scheme"'
    ' }, '
    '"upstream": { '
    '"addr": "$upstream_addr", '
    '"status": "$upstream_status", '
    '"response_time": "$upstream_response_time", '
    '"connect_time": "$upstream_connect_time", '
    '"header_time": "$upstream_header_time" '
    '} '
    '} '
    '}';
    include log-json.conf
    access_log /var/log/nginx/access.json json;
    ```

    @@ -132,41 +81,4 @@ filebeat.prospectors:
    # Logstash
    You have `input` and `output` defined in another file in `/etc/logstash/conf.d/`. Right? Add this to `/etc/logstash/conf.d/nginx-json-filter.conf` or similar.

    ```
    # Process JSON documents output by nginx.
    filter {
    if "nginx" in [tags] {
    grok {
    match => {
    # nginx doesn't log the http version, only the protocol.
    # i.e. HTTP/1.1, HTTP/2
    "[nginx][access][http_protocol]" => "HTTP/%{NUMBER:[nginx][access][http_version]}"
    }
    }

    # msec has millisecond resolution
    date {
    match => [
    "[nginx][time][msec]",
    "UNIX"
    ]
    target => "@timestamp"
    }

    # logstash-filter-geoip plugin is installed by default
    geoip {
    source => "[nginx][access][remote_ip]"
    target => "[nginx][access][geoip]"
    }

    # logstash-filter-useragent plugin is installed by default
    useragent {
    source => "[nginx][access][agent]"
    target => "[nginx][access][user_agent]"
    }

    }
    }
    ```
    You have `input` and `output` defined in another file in `/etc/logstash/conf.d/`. Right? Add `filter-nginx.conf` to the mix. This processes `error_log` files as well.
  3. NiceGuyIT revised this gist Jan 24, 2018. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions nginx-json-elk.md
    View file
    Original file line number Diff line number Diff line change
    @@ -63,9 +63,9 @@ Everything is wrapped in a single "nginx" element/object to match ES's "nginx" m
    # Match the format for Elasticsearch
    log_format json escape=json
    '{ '
    '"fileset": {'
    '"fileset": { '
    '"module": "nginx", '
    '"name": "access", '
    '"name": "access" '
    '}, '
    '"nginx": { '
    '"access": { '
  4. NiceGuyIT revised this gist Jan 24, 2018. 1 changed file with 83 additions and 53 deletions.
    136 changes: 83 additions & 53 deletions nginx-json-elk.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,11 +1,40 @@
    # Intro

    This is an example configuration to have nginx output JSON logs to make it easier for Logstash processing. I was trying to get nginx > Filebeat > Logstash > ES working and it wasn't until I connected Filebeat directly to Elasticsearch that I saw the expected data. Google led me to [`ingest-convert.sh`](https://www.elastic.co/guide/en/logstash/current/ingest-converter.html) and I realized `filebeat setup` works for Filebeat > ES but not Filebeat > Logstash > ES. It seems ingest pipelines are mutually exclusive with Logstash. Actually, they exist but are not used for data coming from Logstash.
    This is an example configuration to have nginx output JSON logs to make it easier for Logstash processing. I was trying to get nginx > Filebeat > Logstash > ES working and it wasn't until I connected Filebeat directly to Elasticsearch that I saw the expected data. Google led me to [`ingest-convert.sh`](https://www.elastic.co/guide/en/logstash/current/ingest-converter.html) and I realized `filebeat setup` works for Filebeat > ES but not Filebeat > Logstash > ES. This is because Logstash does not use ingest pipelines by default. You have to enable them in the `elasticsearch` output block.

    Having nginx log JSON in the format required for Elasticsearch means there's very little processing (i.e. `grok`) to be done in Logstash. nginx can only output JSON for access logs; the `error_log` format cannot be changed.

    Extra fields are output and not used by the Kibana dashboards. I included them in case they might be useful. Since they are not declared in the `filebeat setup`, their default is "string" when you refresh the field list. This might limit their usefulness for aggregations.

    # Why Logstash?

    * Don't use Logstash? Point your beats to ES and be done with it.
    * On the fence about Logstash? Don't. Point your beats to ES and be done with it.
    * Do you have to use Logstash? Then this _might_ be useful.
    * Do you have a specific need to use Logstash? Then this _might_ be useful.

    ## Logstash pipelines vs Elasticsearch ingest nodes
    This [article](https://www.elastic.co/blog/should-i-use-logstash-or-elasticsearch-ingest-nodes) has some good information about why you would choose Logstash over ingest nodes. While there are some benefits to using Logstash, it adds complexity because you have to maintain the pipeline processors outside ES OR use conditionals in the `output` block to specify the ingest node to use for each type of document. I'm still learning the internals but here are some things I found when trying to use multiple ingest nodes in ES.

    1. Only 1 pipeline can be specified inside an `elasticsearch` block. Multiple pipelines are supported by files, which means you'll need to pull them from the *beat install or extract them from ES.
    1. Conditionals need to be outside the `elasticsearch` block, not inside. This means multiple `elasticsearch` outputs, one for each beat.
    1. Variables can be used for the pipeline name but I don't know of a way to check if the pipeline exists before expanding the variables. If the pipeline doesn't exist, tons of warnings are logged.

    # Pros

    1. Simplifies the processing for nginx access logs.
    2. Provides millisecond resolution.
    3. Provides other fields if you're interested in them.

    # Cons

    1. Logstash does not process nginx error logs by default. You need to set [pipeline](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-pipeline) in the output block.
    2. Maintaining pipelines in Logstash instead of using the already managed ingest processors provided by `filebeat setup`. Although once you understand that you need to [convert](https://www.elastic.co/guide/en/logstash/current/ingest-converter.html) the ingest pipeline to Logstash `.conf` format, this could be a benefit.

    # Issues

    1. ES uses `remote_ip` while nginx uses `$http_x_real_ip` for servers behind proxies. I believe this is best fixed with `proxy_set_header` in the nginx config but haven't had time to hash it out.

    # Requirements

    1. A working ELK stack.
    @@ -31,53 +60,59 @@ filebeat setup -e \
    Everything is wrapped in a single "nginx" element/object to match ES's "nginx" mapping.

    ```
    # Match the format for Elasticsearch
    log_format json escape=json
    '{ nginx { '
    '"access": { '
    '"remote_ip": "$remote_addr", '
    '"user_name": "$remote_user", '
    '"time": "$time_local", '
    '"method": "$request_method", '
    '"url": "$request_uri", '
    '"http_protocol": "$server_protocol", '
    '"response_code": "$status", '
    '"body_sent": { '
    '"bytes": "$body_bytes_sent"'
    ' }, '
    '"referrer": "$http_referer", '
    '"agent": "$http_user_agent" '
    ' }, '
    '"request": "$request", '
    '"connection": "$connection", '
    '"pipe": "$pipe", '
    '"connection_requests": "$connection_requests", '
    '"time": { '
    '"iso8601": "$time_iso8601", '
    '"msec": "$msec", '
    '"request": "$request_time"'
    ' }, '
    '"bytes": { '
    '"request_length": "$request_length", '
    '"body_sent": "$body_bytes_sent", '
    '"sent": "$bytes_sent"'
    ' }, '
    '"http": { '
    '"x_forwarded_for": "$http_x_forwarded_for", '
    '"x_forwarded_proto": "$http_x_forwarded_proto", '
    '"x_real_ip": "$http_x_real_ip", '
    '"x_scheme": "$http_x_scheme"'
    ' }, '
    '"upstream": { '
    '"addr": "$upstream_addr", '
    '"status": "$upstream_status", '
    '"response_time": "$upstream_response_time", '
    '"connect_time": "$upstream_connect_time", '
    '"header_time": "$upstream_header_time" '
    ' }'
    ' } }';
    access_log /var/log/nginx/access.json json;
    # Match the format for Elasticsearch
    log_format json escape=json
    '{ '
    '"fileset": {'
    '"module": "nginx", '
    '"name": "access", '
    '}, '
    '"nginx": { '
    '"access": { '
    '"remote_ip": "$remote_addr", '
    '"user_name": "$remote_user", '
    '"time": "$time_local", '
    '"method": "$request_method", '
    '"url": "$request_uri", '
    '"http_protocol": "$server_protocol", '
    '"response_code": "$status", '
    '"body_sent": { '
    '"bytes": "$body_bytes_sent"'
    ' }, '
    '"referrer": "$http_referer", '
    '"agent": "$http_user_agent" '
    ' }, '
    '"request": "$request", '
    '"connection": "$connection", '
    '"pipe": "$pipe", '
    '"connection_requests": "$connection_requests", '
    '"time": { '
    '"iso8601": "$time_iso8601", '
    '"msec": "$msec", '
    '"request": "$request_time"'
    ' }, '
    '"bytes": { '
    '"request_length": "$request_length", '
    '"body_sent": "$body_bytes_sent", '
    '"sent": "$bytes_sent"'
    ' }, '
    '"http": { '
    '"x_forwarded_for": "$http_x_forwarded_for", '
    '"x_forwarded_proto": "$http_x_forwarded_proto", '
    '"x_real_ip": "$http_x_real_ip", '
    '"x_scheme": "$http_x_scheme"'
    ' }, '
    '"upstream": { '
    '"addr": "$upstream_addr", '
    '"status": "$upstream_status", '
    '"response_time": "$upstream_response_time", '
    '"connect_time": "$upstream_connect_time", '
    '"header_time": "$upstream_header_time" '
    '} '
    '} '
    '}';
    access_log /var/log/nginx/access.json json;
    ```

    # filebeat.yml
    @@ -135,8 +170,3 @@ filter {
    }
    }
    ```
    # Issues
    1. ES uses `remote_ip` while nginx uses `$http_x_real_ip` for servers behind proxies. I believe this is best fixed with `proxy_set_header` in the nginx config but haven't had time to hash it out.
    2. Visual Builder dashboards in Kibana don't work due to `fileset.module:nginx AND fileset.name:access` in the Panel Filter (under Panel Options). Need to add `fileset.module` and `fileset.name` and possibly others to the nginx output.
  5. NiceGuyIT revised this gist Jan 23, 2018. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions nginx-json-elk.md
    View file
    Original file line number Diff line number Diff line change
    @@ -9,8 +9,8 @@ Extra fields are output and not used by the Kibana dashboards. I included them i
    # Requirements

    1. A working ELK stack.
    1. \*.example.com is used to demonstrate different servers.
    2. Tested with 6.1.2 on openSUSE. YMMV
    1. \*.example.com is used to demonstrate different servers.
    2. Tested with 6.1.2 on openSUSE. YMMV
    2. [nginx 1.11.8](http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) or higher.
    3. Half a brain. You're expected to know administration. Well, maybe three quarters of a brain.

  6. NiceGuyIT created this gist Jan 23, 2018.
    142 changes: 142 additions & 0 deletions nginx-json-elk.md
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,142 @@
    # Intro

    This is an example configuration to have nginx output JSON logs to make it easier for Logstash processing. I was trying to get nginx > Filebeat > Logstash > ES working and it wasn't until I connected Filebeat directly to Elasticsearch that I saw the expected data. Google led me to [`ingest-convert.sh`](https://www.elastic.co/guide/en/logstash/current/ingest-converter.html) and I realized `filebeat setup` works for Filebeat > ES but not Filebeat > Logstash > ES. It seems ingest pipelines are mutually exclusive with Logstash. Actually, they exist but are not used for data coming from Logstash.

    Having nginx log JSON in the format required for Elasticsearch means there's very little processing (i.e. `grok`) to be done in Logstash. nginx can only output JSON for access logs; the `error_log` format cannot be changed.

    Extra fields are output and not used by the Kibana dashboards. I included them in case they might be useful. Since they are not declared in the `filebeat setup`, their default is "string" when you refresh the field list. This might limit their usefulness for aggregations.

    # Requirements

    1. A working ELK stack.
    1. \*.example.com is used to demonstrate different servers.
    2. Tested with 6.1.2 on openSUSE. YMMV
    2. [nginx 1.11.8](http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) or higher.
    3. Half a brain. You're expected to know administration. Well, maybe three quarters of a brain.

    # Filebeat setup

    This will load the templates and pipelines into Elasticsearch and the dashboards into Kibana.

    ```bash
    filebeat setup -e \
    -E 'setup.template.overwrite=true' \
    -E 'setup.kibana.host="kibana.example.com:5601"' \
    -E 'output.logstash.enabled=false' \
    -E 'output.elasticsearch.hosts=["elasticsearch.example.com:9200"]'
    ```

    # nginx

    Everything is wrapped in a single "nginx" element/object to match ES's "nginx" mapping.

    ```
    # Match the format for Elasticsearch
    log_format json escape=json
    '{ nginx { '
    '"access": { '
    '"remote_ip": "$remote_addr", '
    '"user_name": "$remote_user", '
    '"time": "$time_local", '
    '"method": "$request_method", '
    '"url": "$request_uri", '
    '"http_protocol": "$server_protocol", '
    '"response_code": "$status", '
    '"body_sent": { '
    '"bytes": "$body_bytes_sent"'
    ' }, '
    '"referrer": "$http_referer", '
    '"agent": "$http_user_agent" '
    ' }, '
    '"request": "$request", '
    '"connection": "$connection", '
    '"pipe": "$pipe", '
    '"connection_requests": "$connection_requests", '
    '"time": { '
    '"iso8601": "$time_iso8601", '
    '"msec": "$msec", '
    '"request": "$request_time"'
    ' }, '
    '"bytes": { '
    '"request_length": "$request_length", '
    '"body_sent": "$body_bytes_sent", '
    '"sent": "$bytes_sent"'
    ' }, '
    '"http": { '
    '"x_forwarded_for": "$http_x_forwarded_for", '
    '"x_forwarded_proto": "$http_x_forwarded_proto", '
    '"x_real_ip": "$http_x_real_ip", '
    '"x_scheme": "$http_x_scheme"'
    ' }, '
    '"upstream": { '
    '"addr": "$upstream_addr", '
    '"status": "$upstream_status", '
    '"response_time": "$upstream_response_time", '
    '"connect_time": "$upstream_connect_time", '
    '"header_time": "$upstream_header_time" '
    ' }'
    ' } }';
    access_log /var/log/nginx/access.json json;
    ```

    # filebeat.yml

    `keys_under_root=true` will output fields `nginx.*` which is what we want, while `keys_under_root=false` will output `json.nginx.*`.

    ```yaml
    filebeat.prospectors:
    - type: log
    paths:
    - /var/log/nginx/*.json
    tags: ["nginx", "json"]
    json:
    keys_under_root: true
    add_error_key: true
    ```
    # Logstash
    You have `input` and `output` defined in another file in `/etc/logstash/conf.d/`. Right? Add this to `/etc/logstash/conf.d/nginx-json-filter.conf` or similar.

    ```
    # Process JSON documents output by nginx.
    filter {
    if "nginx" in [tags] {
    grok {
    match => {
    # nginx doesn't log the http version, only the protocol.
    # i.e. HTTP/1.1, HTTP/2
    "[nginx][access][http_protocol]" => "HTTP/%{NUMBER:[nginx][access][http_version]}"
    }
    }

    # msec has millisecond resolution
    date {
    match => [
    "[nginx][time][msec]",
    "UNIX"
    ]
    target => "@timestamp"
    }

    # logstash-filter-geoip plugin is installed by default
    geoip {
    source => "[nginx][access][remote_ip]"
    target => "[nginx][access][geoip]"
    }

    # logstash-filter-useragent plugin is installed by default
    useragent {
    source => "[nginx][access][agent]"
    target => "[nginx][access][user_agent]"
    }

    }
    }
    ```
    # Issues
    1. ES uses `remote_ip` while nginx uses `$http_x_real_ip` for servers behind proxies. I believe this is best fixed with `proxy_set_header` in the nginx config but haven't had time to hash it out.
    2. Visual Builder dashboards in Kibana don't work due to `fileset.module:nginx AND fileset.name:access` in the Panel Filter (under Panel Options). Need to add `fileset.module` and `fileset.name` and possibly others to the nginx output.