Bot Block

From James Dooley's Wiki
Jump to: navigation, search

Overview

Script

Get Top User Agents

Get a list of top user agents (bot nets will only have a few and there will be tons of hits) 

tail -1000 <DOMLOGFILE> | cut -d '"' -f6 | sort | uniq -c | sort -n

or 

cat <DOMLOGFILE> | cut -d '"' -f6 | sort | uniq -c | sort -n

Block By User Agent

Block those user agents 

#!/bin/bash
loglines=1000
logperiod=`tail -$loglines <DOMLOG WITH FULL PATH>`
logrotate=10000
logdelete=100
useragents[0]='Opera/9.02 (Windows NT 5.1; U; ru)'
useragents[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'
useragents[2]='Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1'

function plog {
        echo "[ `date` ] $1" >> /var/log/botblock
        if [ "`wc -l /var/log/botblock | awk '{print $1}'`" -gt "$logrotate" ]
        then
                sed -i -e "`echo $logdelete`d" /var/log/botblock
        fi
        return
}

function botblock {
        touch /var/run/.botblock
        echo $$ > /var/run/.botblock



        agentlen=${#useragents[@]}

        for (( u=0; u<${agentlen}; u++));
        do
                for i in $(echo "$logperiod" | grep "GET /" | grep "${useragents[$u]}" | cut -d " " -f1 | sort | uniq | sort)
                do
                        plog "Attempting to block $i ${useragents[$u]}"
                        plog "`/usr/local/sbin/apf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"   ########APF FIREWALL
                        #plog "`/usr/sbin/csf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"        ########CSF FIREWALL
                done
        done
        rm -f /var/run/.botblock
}

function checkbots {
        if [ ! -e "/var/run/.botblock" ]
        then
                botblock
        else
                plog "Lock file found, script may be already running"
                opid=`cat /var/run/.botblock`
                if [ ! "`ps ax | grep $opid | grep ${0##*/}`" ]
                then
                        plog "PID not active or not owned by script, clearing pid file"
                        rm -f /var/run/.botblock
                        botblock
                else
                        plog "Script already running, PID active"
                fi
        fi
}

function enablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		echo "Cron already enabled, use change to set new time"
	else
		echo "What time would you like to set the cron to"
		echo "[IE: * * * * * ]"
		read crontime;
		if [ ! $crontime ] 
		then
			crontime="* * * * *"
		fi
		echo "SHELL=/bin/bash" > /etc/cron.d/botblock.sh
		echo "$crontime root $(readlink -f $0)" >> /etc/cron.d/botblock.sh
		chmod 0644 /etc/cron.d/botblock.sh
		echo "Cron enabled [$crontime root $(readlink -f $0)]"
		return
	fi
	return
}
 
function disablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		rm -f /etc/cron.d/botblock.sh
		echo "Cron disabled"
	else
		echo "Cron not enabled"
	fi
 	return
}
 
function scanlog {
        if [ -e "$2" ]
        then
                tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        elif [ -e "/usr/local/apache/domlogs/$2" ]
        then
                tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        else
                echo "$2 - Log file not found"
        fi
return
}

function blockcount {
        grep Botnet /etc/apf/deny_hosts.rules | cut -d "(" -f2-3 | sort | uniq -c | sort -n
return
}

function findcommon {
	if [ -e "$2" ]
	then
		for i in $(tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n); do echo "$i"; done | sort | uniq -c | sort -n | tail -40
	elif [ -e "/usr/local/apache/domlogs/$2" ]
	then
		for i in $(tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n); do echo "$i"; done | sort | uniq -c | sort -n | tail -40
	else
		echo "$2 - Log file not found"
	fi
}
 
case $1 in
        --cron)
                case $2 in
			on)
				enablecron
				;;
			off)
				disablecron
				;;
			change)
				disablecron
				enablecron
				;;
		esac
                ;;
        --scanlog)
                scanlog
                ;;
        --blockcount)
                blockcount
                ;;
        --findcommon)
                findcommon
                ;;
        --help)
                echo "Check for bots based on user agent and block in firewall:"
		echo " --cron [on, change, off]"
		echo "	on: Turns on cron job and asks for time"
		echo "	change: Changes the cron time"
		echo "	off: Turns off the cron job"
                echo ""
                echo " --scanlog [domlog to scan]"
                echo "  Scan a log file and return user agents"
                echo " --blockcount"
                echo "  Return number of current blocks per user agent"
                echo " --findcommon [domlog to scan]"
                echo "  Find common keywords for blocking based on user agents"
                ;;
        *)
                checkbots
                ;;
  esac

Block By Agent Length

#!/bin/bash
loglines=1000
logperiod=`tail -$loglines /usr/local/apache/domlogs/abzocke.derblog.net`
logrotate=10000
logdelete=100
#useragents[0]='Opera/9.02 (Windows NT 5.1; U; ru)'
#useragents[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'
#useragents[2]='Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1'

function plog {
        echo "[ `date` ] $1" >> /var/log/botblock
        if [ "`wc -l /var/log/botblock | awk '{print $1}'`" -gt "$logrotate" ]
        then
                sed -i -e "`echo $logdelete`d" /var/log/botblock
        fi
        return
}

function botblock {
        touch /var/run/.botblock
        echo $$ > /var/run/.botblock



        agentlen=${#useragents[@]}

#        for (( u=0; u<${agentlen}; u++));
#        do
#                for i in $(echo "$logperiod" | grep "GET /" | grep "${useragents[$u]}" | cut -d " " -f1 | sort | uniq | sort)
#                do
#                        plog "Attempting to block $i ${useragents[$u]}"
#                        plog "`/usr/local/sbin/apf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"
#                done
#        done
	for i in $(echo "$logperiod" | grep "GET / HTTP/1.1" | grep -E '"[a-zA-Z0-9]{10}"$' | awk '{ print $1 }')
	do
		plog "Attempting to block $i"
		#plog "`/usr/local/sbin/apf -d $i Botnet 2>&1`"
		plog "`/usr/sbin/csf -d $i Botnet 2>&1`"
	done

        rm -f /var/run/.botblock
}

function checkbots {
        if [ ! -e "/var/run/.botblock" ]
        then
                botblock
        else
                plog "Lock file found, script may be already running"
                opid=`cat /var/run/.botblock`
                if [ ! "`ps ax | grep $opid | grep ${0##*/}`" ]
                then
                        plog "PID not active or not owned by script, clearing pid file"
                        rm -f /var/run/.botblock
                        botblock
                else
                        plog "Script already running, PID active"
                fi
        fi
}

function enablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		echo "Cron already enabled, use change to set new time"
	else
		echo "What time would you like to set the cron to"
		echo "[IE: * * * * * ]"
		read crontime;
		if [ ! $crontime ] 
		then
			crontime="* * * * *"
		fi
		echo "SHELL=/bin/bash" > /etc/cron.d/botblock.sh
		echo "$crontime root $(readlink -f $0)" >> /etc/cron.d/botblock.sh
		chmod 0644 /etc/cron.d/botblock.sh
		echo "Cron enabled [$crontime root $(readlink -f $0)]"
		return
	fi
	return
}
 
function disablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		rm -f /etc/cron.d/botblock.sh
		echo "Cron disabled"
	else
		echo "Cron not enabled"
	fi
 	return
}
 
function scanlog {
        if [ -e "$2" ]
        then
                tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        elif [ -e "/usr/local/apache/domlogs/$2" ]
	then
                tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        else
                echo "$2 - Log file not found"
        fi
return
}

function blockcount {
        grep Botnet /etc/apf/deny_hosts.rules | cut -d "(" -f2-3 | sort | uniq -c | sort -n
return
}
 
case $1 in
        --cron)
                case $2 in
			on)
				enablecron
				;;
			off)
				disablecron
				;;
			change)
				disablecron
				enablecron
				;;
		esac
                ;;
        --scanlog)
                scanlog
                ;;
        --blockcount)
                blockcount
		;;
        --help)
                echo "Check for bots based on user agent and block in firewall:"
		echo " --cron [on, change, off]"
		echo "	on: Turns on cron job and asks for time"
		echo "	change: Changes the cron time"
		echo "	off: Turns off the cron job"
                echo ""
                echo " --scanlog [domlog to scan]"
                echo "  Scan a log file and return user agents"
                echo " --blockcount"
                echo "  Return number of current blocks per user agent"
                ;;
        *)
                checkbots
                ;;
  esac

What to change

Retrieved from "http://wiki.jamesdooley.us/index.php?title=Bot_Block&oldid=574"