Skip to content

Instantly share code, notes, and snippets.

@ruanbekker

ruanbekker/analyze_nginx_access_logs.sh

Last active April 30, 2024 15:42
Show Gist options
  • Save ruanbekker/d6110ef0f93e640bf8466193db9c23c1 to your computer and use it in GitHub Desktop.
Save ruanbekker/d6110ef0f93e640bf8466193db9c23c1 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. ruanbekker revised this gist Aug 26, 2017. No changes.
  2. ruanbekker created this gist Aug 26, 2017.
    129 changes: 129 additions & 0 deletions analyze_nginx_access_logs.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,129 @@
    #!/bin/bash

    # variables
    LOGFILE="/var/log/nginx/access.log"
    LOGFILE_GZ="/var/log/nginx/access.log.*"
    RESPONSE_CODE="200"

    # functions
    filters(){
    grep $RESPONSE_CODE \
    | grep -v "\/rss\/" \
    | grep -v robots.txt \
    | grep -v "\.css" \
    | grep -v "\.jss*" \
    | grep -v "\.png" \
    | grep -v "\.ico"
    }

    filters_404(){
    grep "404"
    }

    request_ips(){
    awk '{print $1}'
    }

    request_method(){
    awk '{print $6}' \
    | cut -d'"' -f2
    }

    request_pages(){
    awk '{print $7}'
    }

    wordcount(){
    sort \
    | uniq -c
    }

    sort_desc(){
    sort -rn
    }

    return_kv(){
    awk '{print $1, $2}'
    }

    request_pages(){
    awk '{print $7}'
    }

    return_top_ten(){
    head -10
    }

    ## actions
    get_request_ips(){
    echo ""
    echo "Top 10 Request IP's:"
    echo "===================="

    cat $LOGFILE \
    | filters \
    | request_ips \
    | wordcount \
    | sort_desc \
    | return_kv \
    | return_top_ten
    echo ""
    }

    get_request_methods(){
    echo "Top Request Methods:"
    echo "===================="
    cat $LOGFILE \
    | filters \
    | request_method \
    | wordcount \
    | return_kv
    echo ""
    }

    get_request_pages_404(){
    echo "Top 10: 404 Page Responses:"
    echo "==========================="
    zgrep '-' $LOGFILE $LOGFILE_GZ\
    | filters_404 \
    | request_pages \
    | wordcount \
    | sort_desc \
    | return_kv \
    | return_top_ten
    echo ""
    }


    get_request_pages(){
    echo "Top 10 Request Pages:"
    echo "====================="
    cat $LOGFILE \
    | filters \
    | request_pages \
    | wordcount \
    | sort_desc \
    | return_kv \
    | return_top_ten
    echo ""
    }

    get_request_pages_all(){
    echo "Top 10 Request Pages from All Logs:"
    echo "==================================="
    zgrep '-' --no-filename $LOGFILE $LOGFILE_GZ \
    | filters \
    | request_pages \
    | wordcount \
    | sort_desc \
    | return_kv \
    | return_top_ten
    echo ""
    }

    # executing
    get_request_ips
    get_request_methods
    get_request_pages
    get_request_pages_all
    get_request_pages_404