Difference between revisions of "Bot Block"

From James Dooley's Wiki
Jump to: navigation, search
(→‎Script)
 
(11 intermediate revisions by the same user not shown)
Line 1: Line 1:
 +
[[Category:Scripts]]
 
==Overview==
 
==Overview==
  
  
 
==Script==
 
==Script==
−
 
+
===Get Top User Agents===
 
Get a list of top user agents (bot nets will only have a few and there will be tons of hits)
 
Get a list of top user agents (bot nets will only have a few and there will be tons of hits)
−
<code>[bash,n]
+
<source lang='bash'>
 
tail -1000 <DOMLOGFILE> | cut -d '"' -f6 | sort | uniq -c | sort -n
 
tail -1000 <DOMLOGFILE> | cut -d '"' -f6 | sort | uniq -c | sort -n
−
</code>
+
</source>
 +
or
 +
<source lang='bash'>
 +
cat <DOMLOGFILE> | cut -d '"' -f6 | sort | uniq -c | sort -n
 +
</source>
 +
===Block By User Agent===
 +
Block those user agents
 +
<source lang='bash'>
  
−
Block those user agents
 
−
<code>[bash,n]
 
 
#!/bin/bash
 
#!/bin/bash
−
logperiod=`tail -1000 <FULL DOMLOG FILE>`
+
loglines=1000
 +
logperiod=`tail -$loglines <DOMLOG WITH FULL PATH>`
 
logrotate=10000
 
logrotate=10000
 +
logdelete=100
 
useragents[0]='Opera/9.02 (Windows NT 5.1; U; ru)'
 
useragents[0]='Opera/9.02 (Windows NT 5.1; U; ru)'
 
useragents[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'
 
useragents[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'
Line 19: Line 27:
  
 
function plog {
 
function plog {
−
         echo "[ `date` ] $1" >> /var/log/rubotblock
+
         echo "[ `date` ] $1" >> /var/log/botblock
−
         if [ "`wc -l /var/log/rubotblock | awk '{print $1}'`" -gt "$logrotate" ]
+
         if [ "`wc -l /var/log/botblock | awk '{print $1}'`" -gt "$logrotate" ]
 +
        then
 +
                sed -i -e "`echo $logdelete`d" /var/log/botblock
 +
        fi
 +
        return
 +
}
 +
 
 +
function botblock {
 +
        touch /var/run/.botblock
 +
        echo $$ > /var/run/.botblock
 +
 
 +
 
 +
 
 +
        agentlen=${#useragents[@]}
 +
 
 +
        for (( u=0; u<${agentlen}; u++));
 +
        do
 +
                for i in $(echo "$logperiod" | grep "GET /" | grep "${useragents[$u]}" | cut -d " " -f1 | sort | uniq | sort)
 +
                do
 +
                        plog "Attempting to block $i ${useragents[$u]}"
 +
                        plog "`/usr/local/sbin/apf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"  ########APF FIREWALL
 +
                        #plog "`/usr/sbin/csf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"        ########CSF FIREWALL
 +
                done
 +
        done
 +
        rm -f /var/run/.botblock
 +
}
 +
 
 +
function checkbots {
 +
        if [ ! -e "/var/run/.botblock" ]
 +
        then
 +
                botblock
 +
        else
 +
                plog "Lock file found, script may be already running"
 +
                opid=`cat /var/run/.botblock`
 +
                if [ ! "`ps ax | grep $opid | grep ${0##*/}`" ]
 +
                then
 +
                        plog "PID not active or not owned by script, clearing pid file"
 +
                        rm -f /var/run/.botblock
 +
                        botblock
 +
                else
 +
                        plog "Script already running, PID active"
 +
                fi
 +
        fi
 +
}
 +
 
 +
function enablecron {
 +
if [ -e "/etc/cron.d/botblock.sh" ]
 +
then
 +
echo "Cron already enabled, use change to set new time"
 +
else
 +
echo "What time would you like to set the cron to"
 +
echo "[IE: * * * * * ]"
 +
read crontime;
 +
if [ ! $crontime ]
 +
then
 +
crontime="* * * * *"
 +
fi
 +
echo "SHELL=/bin/bash" > /etc/cron.d/botblock.sh
 +
echo "$crontime root $(readlink -f $0)" >> /etc/cron.d/botblock.sh
 +
chmod 0644 /etc/cron.d/botblock.sh
 +
echo "Cron enabled [$crontime root $(readlink -f $0)]"
 +
return
 +
fi
 +
return
 +
}
 +
 +
function disablecron {
 +
if [ -e "/etc/cron.d/botblock.sh" ]
 +
then
 +
rm -f /etc/cron.d/botblock.sh
 +
echo "Cron disabled"
 +
else
 +
echo "Cron not enabled"
 +
fi
 +
return
 +
}
 +
 +
function scanlog {
 +
        if [ -e "$2" ]
 +
        then
 +
                tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n
 +
        elif [ -e "/usr/local/apache/domlogs/$2" ]
 +
        then
 +
                tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n
 +
        else
 +
                echo "$2 - Log file not found"
 +
        fi
 +
return
 +
}
 +
 
 +
function blockcount {
 +
        grep Botnet /etc/apf/deny_hosts.rules | cut -d "(" -f2-3 | sort | uniq -c | sort -n
 +
return
 +
}
 +
 
 +
function findcommon {
 +
if [ -e "$2" ]
 +
then
 +
for i in $(tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n); do echo "$i"; done | sort | uniq -c | sort -n | tail -40
 +
elif [ -e "/usr/local/apache/domlogs/$2" ]
 +
then
 +
for i in $(tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n); do echo "$i"; done | sort | uniq -c | sort -n | tail -40
 +
else
 +
echo "$2 - Log file not found"
 +
fi
 +
}
 +
 +
case $1 in
 +
        --cron)
 +
                case $2 in
 +
on)
 +
enablecron
 +
;;
 +
off)
 +
disablecron
 +
;;
 +
change)
 +
disablecron
 +
enablecron
 +
;;
 +
esac
 +
                ;;
 +
        --scanlog)
 +
                scanlog
 +
                ;;
 +
        --blockcount)
 +
                blockcount
 +
                ;;
 +
        --findcommon)
 +
                findcommon
 +
                ;;
 +
        --help)
 +
                echo "Check for bots based on user agent and block in firewall:"
 +
echo " --cron [on, change, off]"
 +
echo " on: Turns on cron job and asks for time"
 +
echo " change: Changes the cron time"
 +
echo " off: Turns off the cron job"
 +
                echo ""
 +
                echo " --scanlog [domlog to scan]"
 +
                echo "  Scan a log file and return user agents"
 +
                echo " --blockcount"
 +
                echo "  Return number of current blocks per user agent"
 +
                echo " --findcommon [domlog to scan]"
 +
                echo "  Find common keywords for blocking based on user agents"
 +
                ;;
 +
        *)
 +
                checkbots
 +
                ;;
 +
  esac
 +
 
 +
</source>
 +
 
 +
===Block By Agent Length===
 +
<source lang='bash'>
 +
#!/bin/bash
 +
loglines=1000
 +
logperiod=`tail -$loglines /usr/local/apache/domlogs/abzocke.derblog.net`
 +
logrotate=10000
 +
logdelete=100
 +
#useragents[0]='Opera/9.02 (Windows NT 5.1; U; ru)'
 +
#useragents[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'
 +
#useragents[2]='Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1'
 +
 
 +
function plog {
 +
        echo "[ `date` ] $1" >> /var/log/botblock
 +
        if [ "`wc -l /var/log/botblock | awk '{print $1}'`" -gt "$logrotate" ]
 
         then
 
         then
−
                 sed -i -e "1d" /var/log/rubotblock
+
                 sed -i -e "`echo $logdelete`d" /var/log/botblock
 
         fi
 
         fi
 
         return
 
         return
Line 28: Line 201:
  
 
function botblock {
 
function botblock {
−
agentlen=${#useragents[@]}
+
        touch /var/run/.botblock
 +
        echo $$ > /var/run/.botblock
  
−
for (( u=0; u<${agentlen}; u++));
+
 
−
do
+
 
−
for i in $(echo "$logperiod" | grep "GET /" | grep "${useragents[$u]}" | cut -d " " -f1 | sort | uniq | sort)
+
        agentlen=${#useragents[@]}
 +
 
 +
#        for (( u=0; u<${agentlen}; u++));
 +
#        do
 +
#                for i in $(echo "$logperiod" | grep "GET /" | grep "${useragents[$u]}" | cut -d " " -f1 | sort | uniq | sort)
 +
#                do
 +
#                        plog "Attempting to block $i ${useragents[$u]}"
 +
#                        plog "`/usr/local/sbin/apf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"
 +
#                done
 +
#        done
 +
for i in $(echo "$logperiod" | grep "GET / HTTP/1.1" | grep -E '"[a-zA-Z0-9]{10}"$' | awk '{ print $1 }')
 
do
 
do
−
        plog "Attempting to block $i ${useragents[$u]}"
+
plog "Attempting to block $i"
−
        plog "`/usr/local/sbin/apf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"
+
#plog "`/usr/local/sbin/apf -d $i Botnet 2>&1`"
 +
plog "`/usr/sbin/csf -d $i Botnet 2>&1`"
 
done
 
done
−
done
 
  
 +
        rm -f /var/run/.botblock
 
}
 
}
  
−
if [ ! -e "/var/run/.rubotblock" ]
+
function checkbots {
−
then
+
        if [ ! -e "/var/run/.botblock" ]
−
        botblock
 
−
else   
 
−
        plog "Lock file found, may be already running"
 
−
        opid=`cat /var/run/.rubotblock`
 
−
        if [ ! "`ps ax | grep $opid | grep ${0##*/}`" ]
 
 
         then
 
         then
−
                plog "PID not active or not owned by script, removing lock file"
 
−
                rm -f /var/run/.rubotblock
 
 
                 botblock
 
                 botblock
 
         else
 
         else
−
                 plog "Script already running, pid is not stale"
+
                 plog "Lock file found, script may be already running"
 +
                opid=`cat /var/run/.botblock`
 +
                if [ ! "`ps ax | grep $opid | grep ${0##*/}`" ]
 +
                then
 +
                        plog "PID not active or not owned by script, clearing pid file"
 +
                        rm -f /var/run/.botblock
 +
                        botblock
 +
                else
 +
                        plog "Script already running, PID active"
 +
                fi
 +
        fi
 +
}
 +
 
 +
function enablecron {
 +
if [ -e "/etc/cron.d/botblock.sh" ]
 +
then
 +
echo "Cron already enabled, use change to set new time"
 +
else
 +
echo "What time would you like to set the cron to"
 +
echo "[IE: * * * * * ]"
 +
read crontime;
 +
if [ ! $crontime ]
 +
then
 +
crontime="* * * * *"
 +
fi
 +
echo "SHELL=/bin/bash" > /etc/cron.d/botblock.sh
 +
echo "$crontime root $(readlink -f $0)" >> /etc/cron.d/botblock.sh
 +
chmod 0644 /etc/cron.d/botblock.sh
 +
echo "Cron enabled [$crontime root $(readlink -f $0)]"
 +
return
 +
fi
 +
return
 +
}
 +
 +
function disablecron {
 +
if [ -e "/etc/cron.d/botblock.sh" ]
 +
then
 +
rm -f /etc/cron.d/botblock.sh
 +
echo "Cron disabled"
 +
else
 +
echo "Cron not enabled"
 +
fi
 +
return
 +
}
 +
 +
function scanlog {
 +
        if [ -e "$2" ]
 +
        then
 +
                tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n
 +
        elif [ -e "/usr/local/apache/domlogs/$2" ]
 +
then
 +
                tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n
 +
        else
 +
                echo "$2 - Log file not found"
 
         fi
 
         fi
−
fi
+
return
 +
}
  
−
</code>
+
function blockcount {
 +
        grep Botnet /etc/apf/deny_hosts.rules | cut -d "(" -f2-3 | sort | uniq -c | sort -n
 +
return
 +
}
 +
 +
case $1 in
 +
        --cron)
 +
                case $2 in
 +
on)
 +
enablecron
 +
;;
 +
off)
 +
disablecron
 +
;;
 +
change)
 +
disablecron
 +
enablecron
 +
;;
 +
esac
 +
                ;;
 +
        --scanlog)
 +
                scanlog
 +
                ;;
 +
        --blockcount)
 +
                blockcount
 +
;;
 +
        --help)
 +
                echo "Check for bots based on user agent and block in firewall:"
 +
echo " --cron [on, change, off]"
 +
echo " on: Turns on cron job and asks for time"
 +
echo " change: Changes the cron time"
 +
echo " off: Turns off the cron job"
 +
                echo ""
 +
                echo " --scanlog [domlog to scan]"
 +
                echo "  Scan a log file and return user agents"
 +
                echo " --blockcount"
 +
                echo "  Return number of current blocks per user agent"
 +
                ;;
 +
        *)
 +
                checkbots
 +
                ;;
 +
  esac
 +
</source>
  
 
==What to change==
 
==What to change==

Latest revision as of 14:31, 25 March 2014

Overview

Script

Get Top User Agents

Get a list of top user agents (bot nets will only have a few and there will be tons of hits) 

tail -1000 <DOMLOGFILE> | cut -d '"' -f6 | sort | uniq -c | sort -n

or 

cat <DOMLOGFILE> | cut -d '"' -f6 | sort | uniq -c | sort -n

Block By User Agent

Block those user agents 

#!/bin/bash
loglines=1000
logperiod=`tail -$loglines <DOMLOG WITH FULL PATH>`
logrotate=10000
logdelete=100
useragents[0]='Opera/9.02 (Windows NT 5.1; U; ru)'
useragents[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'
useragents[2]='Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1'

function plog {
        echo "[ `date` ] $1" >> /var/log/botblock
        if [ "`wc -l /var/log/botblock | awk '{print $1}'`" -gt "$logrotate" ]
        then
                sed -i -e "`echo $logdelete`d" /var/log/botblock
        fi
        return
}

function botblock {
        touch /var/run/.botblock
        echo $$ > /var/run/.botblock



        agentlen=${#useragents[@]}

        for (( u=0; u<${agentlen}; u++));
        do
                for i in $(echo "$logperiod" | grep "GET /" | grep "${useragents[$u]}" | cut -d " " -f1 | sort | uniq | sort)
                do
                        plog "Attempting to block $i ${useragents[$u]}"
                        plog "`/usr/local/sbin/apf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"   ########APF FIREWALL
                        #plog "`/usr/sbin/csf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"        ########CSF FIREWALL
                done
        done
        rm -f /var/run/.botblock
}

function checkbots {
        if [ ! -e "/var/run/.botblock" ]
        then
                botblock
        else
                plog "Lock file found, script may be already running"
                opid=`cat /var/run/.botblock`
                if [ ! "`ps ax | grep $opid | grep ${0##*/}`" ]
                then
                        plog "PID not active or not owned by script, clearing pid file"
                        rm -f /var/run/.botblock
                        botblock
                else
                        plog "Script already running, PID active"
                fi
        fi
}

function enablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		echo "Cron already enabled, use change to set new time"
	else
		echo "What time would you like to set the cron to"
		echo "[IE: * * * * * ]"
		read crontime;
		if [ ! $crontime ] 
		then
			crontime="* * * * *"
		fi
		echo "SHELL=/bin/bash" > /etc/cron.d/botblock.sh
		echo "$crontime root $(readlink -f $0)" >> /etc/cron.d/botblock.sh
		chmod 0644 /etc/cron.d/botblock.sh
		echo "Cron enabled [$crontime root $(readlink -f $0)]"
		return
	fi
	return
}
 
function disablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		rm -f /etc/cron.d/botblock.sh
		echo "Cron disabled"
	else
		echo "Cron not enabled"
	fi
 	return
}
 
function scanlog {
        if [ -e "$2" ]
        then
                tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        elif [ -e "/usr/local/apache/domlogs/$2" ]
        then
                tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        else
                echo "$2 - Log file not found"
        fi
return
}

function blockcount {
        grep Botnet /etc/apf/deny_hosts.rules | cut -d "(" -f2-3 | sort | uniq -c | sort -n
return
}

function findcommon {
	if [ -e "$2" ]
	then
		for i in $(tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n); do echo "$i"; done | sort | uniq -c | sort -n | tail -40
	elif [ -e "/usr/local/apache/domlogs/$2" ]
	then
		for i in $(tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n); do echo "$i"; done | sort | uniq -c | sort -n | tail -40
	else
		echo "$2 - Log file not found"
	fi
}
 
case $1 in
        --cron)
                case $2 in
			on)
				enablecron
				;;
			off)
				disablecron
				;;
			change)
				disablecron
				enablecron
				;;
		esac
                ;;
        --scanlog)
                scanlog
                ;;
        --blockcount)
                blockcount
                ;;
        --findcommon)
                findcommon
                ;;
        --help)
                echo "Check for bots based on user agent and block in firewall:"
		echo " --cron [on, change, off]"
		echo "	on: Turns on cron job and asks for time"
		echo "	change: Changes the cron time"
		echo "	off: Turns off the cron job"
                echo ""
                echo " --scanlog [domlog to scan]"
                echo "  Scan a log file and return user agents"
                echo " --blockcount"
                echo "  Return number of current blocks per user agent"
                echo " --findcommon [domlog to scan]"
                echo "  Find common keywords for blocking based on user agents"
                ;;
        *)
                checkbots
                ;;
  esac

Block By Agent Length

#!/bin/bash
loglines=1000
logperiod=`tail -$loglines /usr/local/apache/domlogs/abzocke.derblog.net`
logrotate=10000
logdelete=100
#useragents[0]='Opera/9.02 (Windows NT 5.1; U; ru)'
#useragents[1]='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'
#useragents[2]='Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1'

function plog {
        echo "[ `date` ] $1" >> /var/log/botblock
        if [ "`wc -l /var/log/botblock | awk '{print $1}'`" -gt "$logrotate" ]
        then
                sed -i -e "`echo $logdelete`d" /var/log/botblock
        fi
        return
}

function botblock {
        touch /var/run/.botblock
        echo $$ > /var/run/.botblock



        agentlen=${#useragents[@]}

#        for (( u=0; u<${agentlen}; u++));
#        do
#                for i in $(echo "$logperiod" | grep "GET /" | grep "${useragents[$u]}" | cut -d " " -f1 | sort | uniq | sort)
#                do
#                        plog "Attempting to block $i ${useragents[$u]}"
#                        plog "`/usr/local/sbin/apf -d $i "RU Botnet IP (${useragents[$u]})" 2>&1`"
#                done
#        done
	for i in $(echo "$logperiod" | grep "GET / HTTP/1.1" | grep -E '"[a-zA-Z0-9]{10}"$' | awk '{ print $1 }')
	do
		plog "Attempting to block $i"
		#plog "`/usr/local/sbin/apf -d $i Botnet 2>&1`"
		plog "`/usr/sbin/csf -d $i Botnet 2>&1`"
	done

        rm -f /var/run/.botblock
}

function checkbots {
        if [ ! -e "/var/run/.botblock" ]
        then
                botblock
        else
                plog "Lock file found, script may be already running"
                opid=`cat /var/run/.botblock`
                if [ ! "`ps ax | grep $opid | grep ${0##*/}`" ]
                then
                        plog "PID not active or not owned by script, clearing pid file"
                        rm -f /var/run/.botblock
                        botblock
                else
                        plog "Script already running, PID active"
                fi
        fi
}

function enablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		echo "Cron already enabled, use change to set new time"
	else
		echo "What time would you like to set the cron to"
		echo "[IE: * * * * * ]"
		read crontime;
		if [ ! $crontime ] 
		then
			crontime="* * * * *"
		fi
		echo "SHELL=/bin/bash" > /etc/cron.d/botblock.sh
		echo "$crontime root $(readlink -f $0)" >> /etc/cron.d/botblock.sh
		chmod 0644 /etc/cron.d/botblock.sh
		echo "Cron enabled [$crontime root $(readlink -f $0)]"
		return
	fi
	return
}
 
function disablecron {
	if [ -e "/etc/cron.d/botblock.sh" ]
	then
		rm -f /etc/cron.d/botblock.sh
		echo "Cron disabled"
	else
		echo "Cron not enabled"
	fi
 	return
}
 
function scanlog {
        if [ -e "$2" ]
        then
                tail -$loglines $2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        elif [ -e "/usr/local/apache/domlogs/$2" ]
	then
                tail -$loglines /usr/local/apache/domlogs/$2 | cut -d '"' -f6 | sort | uniq -c | sort -n 
        else
                echo "$2 - Log file not found"
        fi
return
}

function blockcount {
        grep Botnet /etc/apf/deny_hosts.rules | cut -d "(" -f2-3 | sort | uniq -c | sort -n
return
}
 
case $1 in
        --cron)
                case $2 in
			on)
				enablecron
				;;
			off)
				disablecron
				;;
			change)
				disablecron
				enablecron
				;;
		esac
                ;;
        --scanlog)
                scanlog
                ;;
        --blockcount)
                blockcount
		;;
        --help)
                echo "Check for bots based on user agent and block in firewall:"
		echo " --cron [on, change, off]"
		echo "	on: Turns on cron job and asks for time"
		echo "	change: Changes the cron time"
		echo "	off: Turns off the cron job"
                echo ""
                echo " --scanlog [domlog to scan]"
                echo "  Scan a log file and return user agents"
                echo " --blockcount"
                echo "  Return number of current blocks per user agent"
                ;;
        *)
                checkbots
                ;;
  esac

What to change

Retrieved from "http://wiki.jamesdooley.us/index.php?title=Bot_Block&oldid=574"