Skip to content

Instantly share code, notes, and snippets.

@danielmotaleite

danielmotaleite/gist:81a9b05959c10c2c8bc1

Created May 18, 2015 18:45
Show Gist options
  • Save danielmotaleite/81a9b05959c10c2c8bc1 to your computer and use it in GitHub Desktop.
Save danielmotaleite/81a9b05959c10c2c8bc1 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. danielmotaleite created this gist May 18, 2015.
    94 changes: 94 additions & 0 deletions gistfile1.txt
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,94 @@
    input {
    lumberjack {
    port => 17000
    #type => "logs"
    ssl_certificate => "/etc/logstash/logstash.crt"
    ssl_key => "/etc/logstash/logstash.key"
    }
    }
    filter {
    if [service] == "nginx" and [type] == "access" {
    grok {
    match => [ "message", "%{IP:client_ip} %{QS:proxy_ip} %{IP:downstream_ip} (?:%{USERNAME:auth}|-) \[%{HTTPDATE:timestamp}\] - %{NOTSPACE:cf_id} %{URIPROTO:proto} %{URIHOST:site}(:%{INT:port})? \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response} (?:%{NUMBER:bytes:int}|-) (?:%{NUMBER:request_time:float}|-) (?:%{NUMBER:upstream_request_time:float}|-) %{QS:referrer} %{QS:agent}" ]
    }
    date {
    match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
    add_tag => ["timestamp_changed"]
    }
    }
    if [service] == "nginx" and [type] == "error" {
    grok {
    match => [
    "message",
    "%{DATESTAMP:timestamp} \[%{DATA:severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{GREEDYDATA:error} client: %{IP:client_ip}, server: %{URIHOST:site}, request: \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\"(, upstream: %{QS:upstream},)?, host: \"%{URIHOST:site2}\"(, referrer: %{QS:referrer})?",
    "%{DATESTAMP:timestamp} \[%{DATA:severity}\] %{GREEDYDATA:error}",
    "%{DATESTAMP:timestamp} %{GREEDYDATA:error}"
    ]
    }
    date {
    match => [ "timestamp", "yyyy/MM/dd HH:mm:ss" ]
    add_tag => ["timestamp_changed"]
    }
    }
    if [client_ip] =~ /:[0-9a-f]*:/ {
    mutate {
    add_tag => [ "ipv6" ]
    }
    } else {
    mutate {
    add_tag => [ "ipv4" ]
    }
    }

    if [client_ip] {
    geoip {
    source => ["client_ip"]
    target => "geoip"
    }
    }
    }
    filter {
    if [application] == "alice" and [type] == "exception" {
    multiline {
    pattern => "^\s*([0-9]{4}-[0-9]{2}-[0-9]{2})"
    negate => true
    what => previous
    }
    grok {
    match => [ "message", "%{TIMESTAMP_ISO8601:time}\s+(?<error>([\S\s]+))\s+\((?<file_error>(.*))\)(?<stack>(\s*#.*)*)" ]
    }
    date {
    match => [ "time", "YYYY-MM-dd HH:mm:ss"]
    add_tag => ["timestamp_changed"]
    }
    }
    }
    filter {
    if [application] == "bob" and [type] == "exception" {
    multiline {
    pattern => "Exception ID"
    negate => true
    what => previous
    }
    grok {
    match => [ "message", "^%{TIMESTAMP_ISO8601:timestamp}\s+.*\s+\{Exception ID:\s+(?<exception_id>(.*))\}\s+(?<exception>(.*))\s*\|\s*%{GREEDYDATA:error}\s*(?<stack>([\s,\S]*))" ]
    }
    date {
    match => [ "timestamp", "YYYY-MM-dd HH:mm:ss"]
    add_tag => ["timestamp_changed"]
    }
    }
    }
    output {
    elasticsearch {
    host => maghreb60
    port => 11000
    protocol => 'transport'
    index => "logstash-%{+YYYY.MM.dd}-%{service}-%{type}-%{environment}"
    }
    stdout { codec => 'rubydebug' }
    }


    log:
    2015/05/18 08:27:36 [error] 14678#0: *7010174 FastCGI sent in stderr: "Primary script unknown" while reading response header from upstream, client: 122.183.234.125, server: www.xxxxx.tz, request: "GET /wp-admin/admin-ajax.php?action=revslider_ajax_action HTTP/1.0", upstream: "fastcgi://unix:/var/run/php5-fpm/default.sock:", host: "www.xxx.tz"