Parse logs between times

sed -n '/31\/Mar\/2020:09:00/,/31\/Mar\/2020:13:00/ p' domain_com.log

grep "01/APR/2020:16:3[5-9]" domain_com.log

Grep IPs causing 403s

grep -v DBM domain_com.log | grep "Access denied with code 403" | awk -F\" '{print $2}' | sort | uniq -c  | sort -n

Find redirects – parse to file

awk '($7 == "/" && $9 ~ "30")' domain_com.access.log > domain-redirects-onfrontpage.txt

zGrep and (re)Grep

zgrep "Uptime" domain_com.access.log | grep '" 503'

Display top 10 IP addresses

zcat /domain_com.access.log.gz | awk -F"-" '{print $1}' | sort | uniq -dc | sort -nr | head -10

Display top 10 URLs – how many times it appeared on log and also ignoring query string

zcat /domain_com.access.log.gz | awk -F'"' '{print $2}' | awk '{print $2}' | awk -F'?' '{print $1}' | sort | uniq -c | sort -nr | head -10

Display top 10 user-agents

zcat /domain_com.access.log.gz | awk -F'"' '{print $6}' | sort | uniq -dc | sort -nr | head -10

Display top 10 referrer

zcat /domain_com.access.log.gz | awk '{print $11}' | sort | uniq -c | sort -nr | head -10

Display top 10 response code

zcat /domain_com.access.log.gz | awk '{print $9}' | sort | uniq -c | sort -nr | head -10

Display top 10 http methods

zcat /domain_com.access.log.gz | awk -F'"' '{print $2}' | awk '{print $1}' | sort | uniq -dc | sort -nr | head -10

Tail listing events by IP

tail -n 50000 /domain_com.access.log | awk '{print $7}' | sort | uniq -c | sort -nr
  50000 /

Tail listing events by Class C subnet

tail -n 100000 /domain_com.access.log | awk -F\.  '{print $1"."$2"."$3}' | sort  | uniq -c | sort -nr | awk '{print $2,"\t",$1}'

ZGrep IP per minute and per hour

zgrep "X.X.X.X" domain.log.gz | awk {'print $4'} | sed 's/^.//' | cut -d: -f1-3 | uniq -c

zgrep "X.X.X.X" domain.log.gz |  awk {'print $4'} | sed 's/^.//' | cut -d: -f1-2 | uniq -c