Skip to content

Instantly share code, notes, and snippets.

@bodnar1212

bodnar1212/web server log analysis cheat sheet

Forked from hvelarde/web server log analysis cheat sheet
Created June 23, 2023 20:24
Show Gist options
  • Save bodnar1212/c2022ca61df1b6647b3015cc834799a7 to your computer and use it in GitHub Desktop.
Save bodnar1212/c2022ca61df1b6647b3015cc834799a7 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @hvelarde hvelarde revised this gist Oct 26, 2018. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion web server log analysis cheat sheet
    View file
    Original file line number Diff line number Diff line change
    @@ -20,7 +20,7 @@ awk '$9 ~ /404/ {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -r
    awk '$9 ~ /404/' /var/log/nginx/access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head

    # get top IP addresses causing backend errors
    awk '$0 ~ /\[error\]/ && match($0, /(client: )(.*)(, server)/, arr) {print arr[2]}' /var/log/nginx/error.log | sort | uniq -c | sort -rn
    awk '$0 ~ /\[error\]/ && match($0, /(client: )(.*)(, server)/, arr) {print arr[2]}' /var/log/nginx/error.log | sort | uniq -c | sort -rn | awk -v OFS='\t' '{"host " $2 | getline ip; print $0, ip}'

    # get all request of last 10 minutes
    awk -v date=$(date +[%d/%b/%Y:%H:%M --date="-10 minutes") '$4 > date' /var/log/nginx/access.log
  2. @hvelarde hvelarde revised this gist Oct 26, 2018. 1 changed file with 11 additions and 2 deletions.
    13 changes: 11 additions & 2 deletions web server log analysis cheat sheet
    View file
    Original file line number Diff line number Diff line change
    @@ -2,7 +2,7 @@
    awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

    # get top requesters by IP
    awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head | awk -v OFS='\t' '{"host " $2 | getline ip; print $0, ip}'

    # get top requesters by user agent
    awk -F'"' '{print $6}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    @@ -11,7 +11,7 @@ awk -F'"' '{print $6}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | h
    awk '{print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top IP addresses requesting non-existent content
    awk '$9 ~ /404/ {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    awk '$9 ~ /404/ {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head | awk -v OFS='\t' '{"host " $2 | getline ip; print $0, ip}'

    # get top URL returning 404 Not Found
    awk '$9 ~ /404/ {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    @@ -22,5 +22,14 @@ awk '$9 ~ /404/' /var/log/nginx/access.log | awk -F'"' '{print $6}' | sort | uni
    # get top IP addresses causing backend errors
    awk '$0 ~ /\[error\]/ && match($0, /(client: )(.*)(, server)/, arr) {print arr[2]}' /var/log/nginx/error.log | sort | uniq -c | sort -rn

    # get all request of last 10 minutes
    awk -v date=$(date +[%d/%b/%Y:%H:%M --date="-10 minutes") '$4 > date' /var/log/nginx/access.log

    # get frontend request statistics (total count, max time, min time, mean time, median time, and standard deviation)
    awk 'match($0, /( rt=)(.*)( ua=)/, arr) {print arr[2]}' /var/log/nginx/access.log | datamash count 1 max 1 min 1 mean 1 median 1 pstdev 1

    # get backend request statistics (total count, max time, min time, mean time, median time, and standard deviation)
    awk 'match($0, /( ut=")([0-9]+\.[0-9]{3})(.*)(" ul=)/, arr) {print arr[2]}' /var/log/nginx/access.log | datamash count 1 max 1 min 1 mean 1 median 1 pstdev 1

    # get slower requests by URL (ignoring requests using POST method)
    awk -F'rt=' '$0 !~ /POST/ && substr($2,0,5) > 5' /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head
  3. @hvelarde hvelarde revised this gist Aug 3, 2018. 1 changed file with 3 additions and 0 deletions.
    3 changes: 3 additions & 0 deletions web server log analysis cheat sheet
    View file
    Original file line number Diff line number Diff line change
    @@ -21,3 +21,6 @@ awk '$9 ~ /404/' /var/log/nginx/access.log | awk -F'"' '{print $6}' | sort | uni

    # get top IP addresses causing backend errors
    awk '$0 ~ /\[error\]/ && match($0, /(client: )(.*)(, server)/, arr) {print arr[2]}' /var/log/nginx/error.log | sort | uniq -c | sort -rn

    # get slower requests by URL (ignoring requests using POST method)
    awk -F'rt=' '$0 !~ /POST/ && substr($2,0,5) > 5' /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head
  4. @hvelarde hvelarde revised this gist May 11, 2018. 1 changed file with 6 additions and 3 deletions.
    9 changes: 6 additions & 3 deletions web server log analysis cheat sheet
    View file
    Original file line number Diff line number Diff line change
    @@ -11,10 +11,13 @@ awk -F'"' '{print $6}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | h
    awk '{print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top IP addresses requesting non-existent content
    awk '($9 ~ /404/) {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    awk '$9 ~ /404/ {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top URL returning 404 Not Found
    awk '($9 ~ /404/) {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    awk '$9 ~ /404/ {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top user agents requesting non-existent content
    awk '($9 ~ /404/)' /var/log/nginx/access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head
    awk '$9 ~ /404/' /var/log/nginx/access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head

    # get top IP addresses causing backend errors
    awk '$0 ~ /\[error\]/ && match($0, /(client: )(.*)(, server)/, arr) {print arr[2]}' /var/log/nginx/error.log | sort | uniq -c | sort -rn
  5. @hvelarde hvelarde revised this gist Jul 14, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion web server log analysis cheat sheet
    View file
    Original file line number Diff line number Diff line change
    @@ -17,4 +17,4 @@ awk '($9 ~ /404/) {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort
    awk '($9 ~ /404/) {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top user agents requesting non-existent content
    awk '($9 ~ /404/)' /var/log//nginx/access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head
    awk '($9 ~ /404/)' /var/log/nginx/access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head
  6. @hvelarde hvelarde revised this gist Jun 7, 2017. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions web server log analysis cheat sheet
    View file
    Original file line number Diff line number Diff line change
    @@ -5,7 +5,7 @@ awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn
    awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top requesters by user agent
    cut -d' ' -f12- /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    awk -F'"' '{print $6}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top requests by URL
    awk '{print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
    @@ -17,4 +17,4 @@ awk '($9 ~ /404/) {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort
    awk '($9 ~ /404/) {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top user agents requesting non-existent content
    awk '($9 ~ /404/)' /var/log//nginx/access.log | cut -d' ' -f12- | sort | uniq -c | sort -rn | head
    awk '($9 ~ /404/)' /var/log//nginx/access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head
  7. @hvelarde hvelarde created this gist May 24, 2017.
    20 changes: 20 additions & 0 deletions web server log analysis cheat sheet
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,20 @@
    # get total requests by status code
    awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

    # get top requesters by IP
    awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top requesters by user agent
    cut -d' ' -f12- /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top requests by URL
    awk '{print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top IP addresses requesting non-existent content
    awk '($9 ~ /404/) {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top URL returning 404 Not Found
    awk '($9 ~ /404/) {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

    # get top user agents requesting non-existent content
    awk '($9 ~ /404/)' /var/log//nginx/access.log | cut -d' ' -f12- | sort | uniq -c | sort -rn | head