sad/firehol
1
0
Fork 0
mirror of https://github.com/firehol/firehol.git synced 2024-06-16 03:58:22 +00:00
Code Issues Packages Projects Releases Wiki Activity
master
firehol/contrib/dnsbl-ipset.sh
Dimitris Apostolou 8e3735fc2a
Fix typos
2022-02-06 10:48:52 +02:00

1141 lines
34 KiB
Bash
Executable File
Raw Permalink Blame History

#!/bin/bash
#
# Version
# $Id: 3468803a2dcaa0fd6de525779bdcb1ee87d4eb9e $
#
# FireHOL - A firewall for humans...
#
# Copyright
#
# Copyright (C) 2003-2015 Costa Tsaousis <costa@tsaousis.gr>
# Copyright (C) 2012-2015 Phil Whineray <phil@sanewall.org>
#
# License
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# See the file COPYING for details.
#
# -----------------------------------------------------------------------------
# OVERVIEW
#
# This program will tail the iptables kernel log, extract SRC/DST IP addresses
# from it, do DNSBL lookups for them, score them and according to this score
# add them to an ipset. This ipset may be used by a firewall rule to block
# further access.
#
# So, initially an attacker will get access, but after a few seconds, his IP
# might be blocked.
#
# This program itself just manipulates ipsets.
# It does not alter your firewall - it does not generate or execute iptables
# statements.
#
#
# -----------------------------------------------------------------------------
# HOW IT WORKS
#
# The configuration file is /etc/firehol/dnsbl-ipset.conf
# It will be generated with defaults, on first run.
#
# 1. It tails the iptables log file given by IPTABLES_LOG.
#
# 2. It greps for lines containing ULOG_MATCH, it ignores all other.
# Normally, you should log some packets using iptables, and then
# set the iptables log text to ULOG_MATCH, to process these lines only.
#
# 3. If the line contains SRC=<IPv4 IP> and DST=<IPv4 IP> it extracts these
# IPs from the log file.
# It is safe to tail the system log or a log file containing more than
# just iptables logs, the program will process only what it understands.
#
# 4. For each IP extracted (both SRC and DST), it first checks if they
# appear in EXCLUSION_IPSETS, BLACKLIST_IPSET, CACHE_IPSET
# If an IP is found in any of the above, it is ignored.
#
# 5. For each IP not found on any of the ipsets above, it does the following:
#
# 6. Adds the IP to the CACHE_IPSET with options set by CACHE_IPSET_OPTIONS.
# These options allow setting a timeout, after which the IP will be
# re-checked.
#
# 7. It generates DNSBL hostnames to be resolved, for all DNSBLs given in the
# configuration.
#
# 8. These hostnames are resolved with 'adnshost' an excellent asynchronous
# client resolver with machine readable output. adnshost is part of
# adns-tools.
#
# 9. Each successful DNS resolution is scored according to the configuration.
# The configuration allows you to control scoring DNS resolutions to
# suit your needs.
# The default scoring is suitable for servers servicing users, it favours
# dynamic IPs and penalizes server IPs.
#
# 10. The score of each successful DNS lookup is added to the total score
# for each IP.
#
# 11. When the DNS resolution for an IP completes for all DNSBLs configured,
# it takes a decision:
#
# 12. If the total score for an IP is above BLACKLIST_SCORE, it adds the IP
# to the BLACKLIST_IPSET, otherwise, it ignores it.
#
#
# -----------------------------------------------------------------------------
# OTHER FEATURES
#
# a. The speed of DNSBL resolution is controlled by DELAY_BETWEEN_CHECKS.
# The default is 0.2 which means 5 DNS lookups per second per DNSBL.
#
# b. If the DNS resolution is slow, the program will start throttling.
# This is controlled with THROTTLE_THRESHOLD which controls how many IPs
# should be in the queue (currently being resolved) for the program to
# work without any delays.
#
# c. It saves 4 log files (in the dir set by LOG_DIR):
# 1. ips.log - all the IPs checked (one line per IP)
# 2. matches.log - all the DNSBLs that responded (one line per resolution)
# 3. clean.log - all the IPs with low score (one line per IP)
# 4. blacklisted.log - all the IPs with high score (one line per IP)
#
# d. The blacklisted ipset will include a comment so that you will know
# why each IP was blacklisted (which DNSBLs matched it, with which score)
#
# e. The BLACKLIST_IPSET has to exist prior to running this script.
# The program will give you the command to add in firehol.conf
#
PROGRAM_FILE="${0}"
# single line flock, from man flock
[ "${DNSBL_IPSET_LOCKER}" != "${0}" ] && exec env DNSBL_IPSET_LOCKER="$0" flock -en "/var/run/dnsbl-ipset.lock" "${0}" "${@}" || :
LC_ALL=C
umask 077
if [ ! "${UID}" = 0 ]
then
echo >&2 "Only root can run this program."
exit 1
fi
renice 10 $$ >/dev/null 2>/dev/null
require_cmd() {
local cmd= block=1
if [ "a${1}" = "a-n" ]
then
block=0
shift
fi
unalias ${1} >/dev/null 2>&1
cmd=`which ${1} 2>/dev/null | head -n 1`
if [ $? -gt 0 -o ! -x "${cmd}" ]
then
if [ ${block} -eq 1 ]
then
echo >&2 "ERROR: Command '${1}' not found in the system path."
exit 1
fi
return 1
fi
eval "${1^^}_CMD=${cmd}"
return 0
}
require_cmd adnshost
require_cmd ipset
require_cmd sed
require_cmd grep
require_cmd tail
RUNNING_ON_TERMINAL=0
if [ "z$1" = "z-nc" ]
then
shift
else
test -t 2 && RUNNING_ON_TERMINAL=1
if [ -t 2 -a $[$(tput colors 2>/dev/null)] -ge 8 ]
then
# Enable colors
COLOR_RESET="\e[0m"
COLOR_BLACK="\e[30m"
COLOR_RED="\e[31m"
COLOR_GREEN="\e[32m"
COLOR_YELLOW="\e[33m"
COLOR_BLUE="\e[34m"
COLOR_PURPLE="\e[35m"
COLOR_CYAN="\e[36m"
COLOR_WHITE="\e[37m"
COLOR_BGBLACK="\e[40m"
COLOR_BGRED="\e[41m"
COLOR_BGGREEN="\e[42m"
COLOR_BGYELLOW="\e[43m"
COLOR_BGBLUE="\e[44m"
COLOR_BGPURPLE="\e[45m"
COLOR_BGCYAN="\e[46m"
COLOR_BGWHITE="\e[47m"
COLOR_BOLD="\e[1m"
COLOR_DIM="\e[2m"
COLOR_UNDERLINED="\e[4m"
COLOR_BLINK="\e[5m"
COLOR_INVERTED="\e[7m"
fi
fi
# -----------------------------------------------------------------------------
# the console spinner
PROGRAM_SPINNER_SPACES=' '
PROGRAM_SPINNER_BACKSPACES='\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b'
PROGRAM_SPINNER_LAST=0
PROGRAM_SPINNER='|/-\'
PROGRAM_SPINNER_RUNNING=0
PROGRAM_SPINNER_PREFIX="Queue"
spinner()
{
local t="${PROGRAM_SPINNER_PREFIX} ${1}"
printf >&2 "${PROGRAM_SPINNER_BACKSPACES:0:$PROGRAM_SPINNER_LAST}"
PROGRAM_SPINNER_LAST=$(( (${#t} + 5) * 2 ))
local temp=${PROGRAM_SPINNER#?}
printf >&2 "[${t} %c] " "${PROGRAM_SPINNER}"
PROGRAM_SPINNER=$temp${PROGRAM_SPINNER%"$temp"}
PROGRAM_SPINNER_RUNNING=1
}
spinner_end() {
local last=$((PROGRAM_SPINNER_LAST / 2))
printf >&2 "${PROGRAM_SPINNER_BACKSPACES:0:$PROGRAM_SPINNER_LAST}"
printf >&2 "${PROGRAM_SPINNER_SPACES:0:$last}"
printf >&2 "${PROGRAM_SPINNER_BACKSPACES:0:$PROGRAM_SPINNER_LAST}"
PROGRAM_SPINNER_RUNNING=0
PROGRAM_SPINNER_LAST=0
}
# -----------------------------------------------------------------------------
# functions to parse the configuration file
declare -A DNSBL=()
declare -A DNSBL_SCORES=()
declare -A DNSBL_PREKEYS=()
declare -a work_dnsbl=()
dnsbl() {
#[ ${RUNNING_ON_TERMINAL} -eq 1 ] && spinner_end
local score="${1}" list= prekey= decode=
shift
if [ "${score}" = "clear" ]
then
DNSBL=()
DNSBL_SCORES=()
work_dnsbl=()
else
work_dnsbl=()
while [ ! -z "${1}" ]
do
case "${1}" in
prekey)
prekey="${2}"
shift 2
continue
;;
decode)
decode="${2}"
shift 2
continue
;;
*) ;;
esac
work_dnsbl=("${work_dnsbl[@]}" "${1}")
shift
done
for list in "${work_dnsbl[@]}"
do
DNSBL_SCORES[${list}]=$[score]
DNSBL[${list}]="${list}"
[ ! -z "${prekey}" ] && DNSBL_PREKEYS[${list}]="${prekey}."
[ ! -z "${decode}" ] && DNSBL_SCORES[custom/${list}]="${decode}"
done
fi
#[ ${RUNNING_ON_TERMINAL} -eq 1 ] && spinner "parsing config..."
return 0
}
score() {
#[ ${RUNNING_ON_TERMINAL} -eq 1 ] && spinner_end
local score="${1}" result= list=
shift
if [ -z "${score}" ]
then
echo >&2 "ERROR: Empty score for dnsbl ${work_dnsbl[@]} result ${*}"
score="0"
fi
for result in "${@}"
do
for list in "${work_dnsbl[@]}"
do
DNSBL_SCORES[${result}/${work_dnsbl}]=$[score]
done
done
#[ ${RUNNING_ON_TERMINAL} -eq 1 ] && spinner "parsing config..."
return 0
}
# -----------------------------------------------------------------------------
# Configuration
# --- BEGIN OF DNSBL-IPSET DEFAULTS ---
# where is the iptables log file to tail?
# leave empty for auto-detection - may not work for you - please set it
IPTABLES_LOG=""
# which string to find in the log?
ULOG_MATCH="AUDIT"
# where to put our logs?
LOG_DIR="/var/log/dnsbl-ipset"
# which IPSETs to examine to exclude IPs from checking?
# space separated list of any number of ipsets.
# add here any ipsets that include IPs that should never be blacklisted.
EXCLUSION_IPSETS="bogons fullbogons whitelist"
# which IPSET will receive the blacklisted IPs?
# this has to exist - on first error to add the program will exit
# this ipset will also be checked for excluding new queries
BLACKLIST_IPSET="dnsbl"
# what additional options to give when adding an IP to the blacklist ipset?
BLACKLIST_IPSET_OPTIONS="timeout $[7 * 24 * 3600]"
# set this to 1 to have comments on the blacklist ipset
# the comments will include the score and DNSBLs matched it
BLACKLIST_IPSET_COMMENTS=1
# which IPSET will cache the checked IPs?
# this ipset will also be checked for excluding new queries
# it will be automatically created if it does not exist
CACHE_IPSET="dnsbl_cache"
# what additional options to give when adding IPs to the CACHE_IPSET?
CACHE_IPSET_OPTIONS="timeout $[24 * 3600]"
# how to create the cache ipset - if it does not exist?
# this ipset will be created only if it does not exist
CACHE_IPSET_CREATE_OPTIONS="timeout $[24 * 3600] maxelem 2000000"
# which is the BLACKLIST score?
# any IP that will get a score above or equal to this, will be
# added to the BLACKLIST_IPSET
BLACKLIST_SCORE="100"
# delay to issue between IP checks
# if you lower this a lot, DNSBLs will refuse to talk to you
DELAY_BETWEEN_CHECKS="0.2"
# when we will have this many IP checks in progress
# we will stop processing until this drops below this point
THROTTLE_THRESHOLD="500"
# where is the throttle lock file?
THROTTLE_LOCK_FILE="/var/run/dnsbl-ipset.lock"
# enable this for more logging
DEBUG=0
# -----------------------------------------------------------------------------
# Custom Check on Blacklisted IPs
# this function is called for every IP the tool is going to blacklist
blacklist_check() {
local counter="${1}" ip="${2}" score="${3}"
shift 3
# counter = the number of DNSBLs matched it
# ip = the IP that is going to be blacklisted
# score = the score this IP got from the all lists
# the rest of the parameters are the lists matched it
# do anything you like here to check the IP
# example:
# whois "${ip}" | grep -iE "(owner|address|organization|country)"
# return 0 = ok, blacklist it
# return 1 = no, don't blacklist it
return 0
}
# -----------------------------------------------------------------------------
# Default Score Configuration
# clear any previous configuration
dnsbl clear
# the default settings have been set to benefit dynamic IP ranges that might be used by users
# TEMPLATE
# >> dnsbl DEFAULT_SCORE DNSBL
#
# The DEFAULT_SCORE will be used if a more specific score is not given
#
# optionally, followed by:
# >> score SCORE IP_RESOLVED
#
# IP_RESOLVED is looked up like this:
# If the DNS resolution on DNSBL dnsb.org gives 127.1.2.3, the program will lookup
# IP_RESOLVED in this order (the first matched will be used):
#
# 127.1.2.3
# 127.1.x.3
# 127.x.x.3
# 127.1.2.3
# 127.1.2
# 127.1.x
# 127.x.2
# 127.1
# 127.x
# 127
# DEFAULT_SCORE
#
# -----------------------------------------------------------------------------
# our score definitions
IGNORE="0" # do not take into account this result
PROXY="1000" # a verified open proxy
EXPLOIT="100" # a verified exploit
SPAM="15" # the host is known to send spam
SPAMPRO="100" # a professional spammer
SPAMWAVE="200" # participated in a recent spam wave
DYNAMICIP="-500" # a dynamic IP
BADKARMA="300" # the host is known to have bad karma
GOODKARMA="-500" # the host is known to have good karma
# -----------------------------------------------------------------------------
# Project Honey Pot
# This is an excellent DNSBL for HTTP traffic.
# To enable it, register at the site http://www.projecthoneypot.org/
# then go to http://www.projecthoneypot.org/httpbl_configure.php
# set your key to httpbl_key
httpbl_key=""
if [ ! -z "${httpbl_key}" ]
then
dnsbl ${IGNORE} dnsbl.httpbl.org prekey "${httpbl_key}"
for threat in {128..255}
do
for days in {0..7}
do
score ${BADKARMA} 127.${days}.${threat}
done
for days in {8..14}
do
score $[BADKARMA/2] 127.${days}.${threat}
done
done
fi
# -----------------------------------------------------------------------------
dnsbl ${IGNORE} zen.spamhaus.org
score ${SPAMPRO} 127.0.0.2 # sbl.spamhaus.org, Spamhaus SBL Data, Static UBE sources, verified spam services (hosting or support) and ROKSO spammers
score ${SPAMPRO} 127.0.0.3 # sbl.spamhaus.org, Spamhaus SBL CSS Data, Static UBE sources, verified spam services (hosting or support) and ROKSO spammers
score ${EXPLOIT} 127.0.0.4 # xbl.spamhaus.org, CBL Data, Illegal 3rd party exploits, including proxies, worms and trojan exploits
score ${EXPLOIT} 127.0.0.5 # xbl.spamhaus.org = Illegal 3rd party exploits, including proxies, worms and trojan exploits
score ${EXPLOIT} 127.0.0.6 # xbl.spamhaus.org = Illegal 3rd party exploits, including proxies, worms and trojan exploits
score ${EXPLOIT} 127.0.0.7 # xbl.spamhaus.org = Illegal 3rd party exploits, including proxies, worms and trojan exploits
score ${DYNAMICIP} 127.0.0.10 # pbl.spamhaus.org = End-user Non-MTA IP addresses set by ISP outbound mail policy
score ${DYNAMICIP} 127.0.0.11 # pbl.spamhaus.org = End-user Non-MTA IP addresses set by ISP outbound mail policy
dnsbl ${IGNORE} swl.spamhaus.org
score ${GOODKARMA} 127.0.2 # Spamhaus Whitelists
dnsbl ${IGNORE} dnsbl.sorbs.net
score ${PROXY} 127.0.0.2 # http.dnsbl.sorbs.net - List of Open HTTP Proxy Servers
score ${PROXY} 127.0.0.3 # socks.dnsbl.sorbs.net - List of Open SOCKS Proxy Server
score ${PROXY} 127.0.0.4 # misc.dnsbl.sorbs.net - List of open Proxy Servers not listed in the SOCKS or HTTP lists
score ${IGNORE} 127.0.0.5 # smtp.dnsbl.sorbs.net - List of Open SMTP relay servers
score ${SPAM} 127.0.0.6 # new.spam.dnsbl.sorbs.net - List of hosts that have been noted as sending spam/UCE/UBE to the admins of SORBS within the last 48 hours.
score ${EXPLOIT} 127.0.0.7 # web.dnsbl.sorbs.net - List of web (WWW) servers which have spammer abusable vulnerabilities (e.g. FormMail scripts) Note: This zone now includes non-webserver IP addresses that have abusable vulnerabilities.
score ${IGNORE} 127.0.0.8 # block.dnsbl.sorbs.net - List of hosts demanding that they never be tested by SORBS.
score ${EXPLOIT} 127.0.0.9 # zombie.dnsbl.sorbs.net - List of networks hijacked from their original owners, some of which have already used for spamming.
score ${DYNAMICIP} 127.0.0.10 # dul.dnsbl.sorbs.net - Dynamic IP Address ranges (NOT a Dial Up list!)
score ${IGNORE} 127.0.0.11 # badconf.rhsbl.sorbs.net - List of domain names where the A or MX records point to bad address space.
score ${DYNAMICIP} 127.0.0.12 # nomail.rhsbl.sorbs.net - List of domain names where the owners have indicated no email should ever originate from these domains.
score ${DYNAMICIP} 127.0.0.14 # noserver.dnsbl.sorbs.net - IP addresses and Netblocks of where system administrators and ISPs owning the network have indicated that servers should not be present.
dnsbl ${IGNORE} all.spamrats.com
score ${DYNAMICIP} 127.0.0.36 # Dyna, IP Addresses that have been found sending an abusive amount of connections, or trying too many invalid users at ISP and Telco's mail servers, and are also known to conform to a naming convention that is indicative of a home connection or dynamic address space.
score ${SPAM} 127.0.0.37 # Noptr, IP Addresses that have been found sending an abusive amount of connections, or trying too many invalid users at ISP and Telco's mail servers, and are also known to have no reverse DNS, a technique often used by bots and spammers
score ${SPAM} 127.0.0.38 # Spam, IP Addresses that do not conform to more commonly known threats, and is usually because of compromised servers, hosts, or open relays. However, since there is little accompanying data this list COULD have false-positives, and we suggest that it only is used if you support a more aggressive stance
dnsbl ${IGNORE} hostkarma.junkemailfilter.com
score ${GOODKARMA} 127.0.0.1 # whitelist
score ${BADKARMA} 127.0.0.2 # blacklist
score $[BADKARMA/3] 127.0.0.3 # yellowlist
score $[BADKARMA/2] 127.0.0.4 # brownlist
score ${IGNORE} 127.0.0.5 # no blacklist
dnsbl ${IGNORE} rep.mailspike.net # IP Reputation
score ${BADKARMA} 127.0.0.10 # Worst possible
score $[BADKARMA * 2 / 3] 127.0.0.11 # Very bad
score $[BADKARMA / 2] 127.0.0.12 # Bad
score $[BADKARMA / 3] 127.0.0.13 # Suspicious
score $[BADKARMA / 4] 127.0.0.14 # Neutral - probably spam
score ${IGNORE} 127.0.0.15 # Neutral
score $[GOODKARMA / 4] 127.0.0.16 # Neutral - probably legit
score $[GOODKARMA / 3] 127.0.0.17 # Possibly legit sender
score $[GOODKARMA / 2] 127.0.0.18 # Good
score $[GOODKARMA * 2 / 3] 127.0.0.19 # Very Good
score ${GOODKARMA} 127.0.0.20 # Excellent
dnsbl 0 list.blogspambl.com
score ${BADKARMA} 127.0.0.2
dnsbl 0 wormrbl.imp.ch
score ${EXPLOIT} 127.0.0.5 # worms, viruses of the last 3 days
dnsbl ${GOODKARMA} list.dnswl.org # all responses include valid mail servers
dnsbl ${SPAMWAVE} z.mailspike.net # participating in a distributed spam wave in the last 48 hours
dnsbl ${GOOFKARMA} wl.mailspike.net # whitelist
dnsbl $[BADKARMA/4] b.barracudacentral.org # Barracuda Reputation Block List, http://barracudacentral.org/rbl/listing-methodology
dnsbl $[BADKARMA/4] korea.services.net # South Korean IP address space - this is not necessarily bad
dnsbl ${SPAM} all.s5h.net
dnsbl ${SPAM} spam.dnsbl.sorbs.net # spam.dnsbl.sorbs.net - List of hosts that have been noted as sending spam/UCE/UBE to the admins of SORBS at any time, and not subsequently resolving the matter and/or requesting a delisting. (Includes both old.spam.dnsbl.sorbs.net and escalations.dnsbl.sorbs.net).
# cbl.abuseat.org may be also included in xbl.spamhaus.org
# in this case, it should not be added again.
#dnsbl 200 cbl.abuseat.org # The CBL only lists IPs exhibiting characteristics which are specific to open proxies of various sorts (HTTP, socks, AnalogX, wingate, Bagle call-back proxies etc) and dedicated Spam BOTs (such as Cutwail, Rustock, Lethic etc) which have been abused to send spam, worms/viruses that do their own direct mail transmission, or some types of trojan-horse or "stealth" spamware, dictionary mail harvesters etc.
dnsbl ${SPAM} dnsbl.justspam.org # If an IP that we never got legit email from is seen spamming and said IP is already listed by at least one of the other well-known and independent blacklists, then it is added to our blacklist dnsbl.justspam.org.
dnsbl ${IGNORE} rbl.megarbl.net
score ${SPAM} 127.0.0.2 # spam source
#dnsbl ${IGNORE} dnsbl.inps.de # is listing IPs if they are listed on other DNSBLs
dnsbl ${IGNORE} bl.spamcop.net
score ${SPAM} 127.0.0.2 # spam source
dnsbl ${IGNORE} db.wpbl.info
score ${SPAM} 127.0.0.2 # spam source
dnsbl ${IGNORE} dnsbl.anticaptcha.net
score ${SPAM} 127.0.0.3 # spam source
score ${SPAM} 127.0.0.10 # spam source
dnsbl ${IGNORE} ubl.unsubscore.com
score ${SPAM} 127.0.0.2 # spam source
dnsbl ${IGNORE} bl.tiopan.com
score ${SPAM} 127.0.0.2 # spam source
dnsbl ${SPAM} ix.dnsbl.manitu.net # spam source?
dnsbl ${SPAM} psbl.surriel.com # spam source
# --- other lists to choose from ---
# access.redhawk.org
# blackholes.five-ten-sg.com
# blackholes.wirehub.net
# blacklist.sci.kun.nl
# blacklist.woody.ch
# bl.emailbasura.org
# blocked.hilli.dk
# bl.spamcannibal.org
# bogons.cymru.com
# cblless.anti-spam.org.cn
# cdl.anti-spam.org.cn
# combined.abuse.ch
# combined.rbl.msrbl.net
# dev.null.dk
# dialup.blacklist.jippg.org
# dialups.mail-abuse.org
# dialups.visi.com
# dnsbl-1.uceprotect.net
# dnsbl-2.uceprotect.net
# dnsbl-3.uceprotect.net
# dnsbl.abuse.ch
# dnsbl.antispam.or.id
# dnsbl.cyberlogic.net
# dnsbl.dronebl.org
# dnsbl.kempt.net
# dnsbl.tornevall.org
# drone.abuse.ch
# dynip.rothen.com
# exitnodes.tor.dnsbl.sectoor.de
# hil.habeas.com
# images.rbl.msrbl.net
# intruders.docs.uu.se
# ips.backscatterer.org
# mail-abuse.blacklist.jippg.org
# msgid.bl.gweep.ca
# no-more-funn.moensted.dk
# opm.tornevall.org
# phishing.rbl.msrbl.net
# proxy.bl.gweep.ca
# pss.spambusters.org.ar
# rbl.interserver.net
# rbl.schulte.org
# rbl.snark.net
# relays.bl.gweep.ca
# relays.bl.kundenserver.de
# relays.nether.net
# short.rbl.jp
# spam.abuse.ch
# spamguard.leadmon.net
# spamlist.or.kr
# spam.olsentech.net
# spamrbl.imp.ch
# spam.rbl.msrbl.net
# spamsources.fabel.dk
# tor.dnsbl.sectoor.de
# virbl.bit.nl
# virus.rbl.jp
# virus.rbl.msrbl.net
# wormrbl.imp.ch
# --- END OF DNSBL-IPSET DEFAULTS ---
# -----------------------------------------------------------------------------
# configuration file management
FIREHOL_CONFIG_DIR="/etc/firehol"
# Generate config file
if [ ! -f "${FIREHOL_CONFIG_DIR}/dnsbl-ipset.conf" ]
then
grep -E "^# --- BEGIN OF DNSBL-IPSET DEFAULTS ---" -A 1000 "${PROGRAM_FILE}" |\
grep -E "^# --- END OF DNSBL-IPSET DEFAULTS ---" -B 1000 >"${FIREHOL_CONFIG_DIR}/dnsbl-ipset.conf" || exit 1
chown root:root "${FIREHOL_CONFIG_DIR}/dnsbl-ipset.conf" || exit 1
chmod 600 "${FIREHOL_CONFIG_DIR}/dnsbl-ipset.conf" || exit 1
echo >&2 "Generated default config file '${FIREHOL_CONFIG_DIR}/dnsbl-ipset.conf'."
echo >&2 "Please run me again to execute..."
exit 1
fi
enable -n trap
enable -n exit
echo "Parsing config..."
source "${FIREHOL_CONFIG_DIR}/dnsbl-ipset.conf" || exit 1
echo "Config parsed."
enable trap
enable exit
# -----------------------------------------------------------------------------
# command line processing
if [ -z "${IPTABLES_LOG}" ]
then
for x in /var/log/ulogd/ulogd_syslogemu.log /var/log/ulogd/syslogemu.log
do
if [ -f "${x}" ]
then
IPTABLES_LOG="${x}"
break
fi
done
fi
usage() {
cat <<EOF
${PROGRAM_FILE} [file IPTABLES_LOG] [grep ULOG_MATCH] [test] [flush]
defaults:
IPTABLES_LOG="${IPTABLES_LOG}"
LOG_MATCH="${ULOG_MATCH}"
test will show the IPs matched from the log file.
flush will empty ${BLACKLIST_IPSET} and ${CACHE_IPSET} ipsets
EOF
exit 0
}
TEST_SRC_DST=0
while [ ! -z "${1}" ]
do
case "${1}" in
match|search|grep|-s)
ULOG_MATCH="${2}"
shift
;;
file|tail|-f)
IPTABLES_LOG="${2}"
shift
;;
test|-t)
TEST_SRC_DST=1
RUNNING_ON_TERMINAL=0
;;
debug|-d)
DEBUG=1
;;
flush|clear|clean|restart)
ipset --flush "${CACHE_IPSET}"
ipset --flush "${BLACKLIST_IPSET}"
echo >&2 "IP sets ${CACHE_IPSET} and ${BLACKLIST_IPSET} have been flushed."
exit 0
;;
help|-h|--help) usage ;;
*) echo >&2 "Cannot understand option '${1}'."; usage ;;
esac
shift
done
# -----------------------------------------------------------------------------
# post-configuration checks
[ ${DEBUG} -eq 1 ] && RUNNING_ON_TERMINAL=0
if [ -z "${IPTABLES_LOG}" -o ! -f "${IPTABLES_LOG}" ]
then
echo >&2 "Cannot find ulogd iptables log ${IPTABLES_LOG}"
exit 1
fi
if [ ! -d "${LOG_DIR}" ]
then
mkdir -p "${LOG_DIR}" || exit 1
fi
cd "${LOG_DIR}" || exit 1
ipset --list "${BLACKLIST_IPSET}" >/dev/null
if [ $? -ne 0 ]
then
echo >&2 "Cannot find BLACKLIST_IPSET '${BLACKLIST_IPSET}'."
echo >&2 "Please add it in firehol.conf like this:"
echo >&2 "ipset4 create ${BLACKLIST_IPSET} hash:ip timeout $[86400 * 7] maxelem 1000000 prevent_reset_on_restart comment"
echo >&2 "And restart firehol to activate it."
exit 1
fi
ipset --list "${CACHE_IPSET}" -t >/dev/null 2>&1
if [ $? -ne 0 ]
then
ipset --create ${CACHE_IPSET} hash:ip ${CACHE_IPSET_CREATE_OPTIONS} || exit 1
fi
for ipset in ${EXCLUSION_IPSETS}
do
ipset --list ${ipset} -t >/dev/null
if [ $? -ne 0 ]
then
echo >&2 "ipset '${ipset}' does not exist. Please set EXCLUSION_IPSETS properly."
exit 1
fi
done
cleanup() {
echo >&2 "Cleaning up..."
echo >&2 "All done, bye..."
trap exit EXIT
trap exit HUP
trap exit INT
exit 0
}
trap cleanup EXIT
trap cleanup HUP
trap cleanup INT
# -----------------------------------------------------------------------------
# program functions
httpbl_parser() {
local ip0="${1}" ip1="${2}" ip2="${3}" ip3="${4}"
local days=$[ip1] threat=$[ip2] type=$[ip3]
local search=0 suspicious=0 harvester=0 cspammer=0
[ ! $[ip0] -eq 127 ] && return 0
[ $[type & 1] -eq 1 ] && suspicious=1
[ $[type & 2] -eq 1 ] && harvester=1
[ $[type & 4] -eq 1 ] && cspammer=1
[ $[type] -eq 0 ] && search=1
# TODO
# THIS FUNCTION IS INCOMPLETE - DO NOT USE YET
}
DNSBL_GET_SCORE="0"
dnsbl_get_score() {
local reply="${1}" dnsbl="${2}" ip=() x=
# parse the reply IP to its parts
IFS="." read -ra ip <<< "${reply}"
# if it has a custom callback, call it
if [ ! -z "${DNSBL_SCORES[custom/${dnsbl}]}" ]
then
# call the custom function
DNSBL_GET_SCORE=
${DNSBL_SCORES[custom/${dnsbl}]} ${ip[0]} ${ip[1]} ${ip[2]} ${ip[3]}
if [ -z "${DNSBL_GET_SCORE}" ]
then
echo >&2 "SCORE: callback ${DNSBL_SCORES[custom/${dnsbl}]} did not set a score"
DNSBL_GET_SCORE="0"
fi
return $?
fi
# check for a score all possible combinations
# from more specific to more generic
for x in \
${ip[0]}.${ip[1]}.${ip[2]}.${ip[3]}/${dnsbl} \
${ip[0]}.${ip[1]}.x.${ip[3]}/${dnsbl} \
${ip[0]}.x.x.${ip[3]}/${dnsbl} \
${ip[0]}.${ip[1]}.${ip[2]}/${dnsbl} \
${ip[0]}.${ip[1]}.x/${dnsbl} \
${ip[0]}.x.${ip[2]}/${dnsbl} \
${ip[0]}.${ip[1]}/${dnsbl} \
${ip[0]}.x/${dnsbl} \
${ip[0]}/${dnsbl} \
${dnsbl}
do
if [ ! -z "${DNSBL_SCORES[${x}]}" ]
then
# found it
[ ${DEBUG} -eq 1 ] && echo >&2 "SCORE: ${x} = ${DNSBL_SCORES[${x}]}"
DNSBL_GET_SCORE="${DNSBL_SCORES[${x}]}"
return 0
fi
done
# not found any
echo >&2 "ERROR: SCORE NOT FOUND: ${reply}/${dnsbl}"
DNSBL_GET_SCORE="0"
return 0
}
match() {
local reply="${1}" ip3="${2}" ip2="${3}" ip1="${4}" ip0="${5}" dnsbl= ip=
shift 5
dnsbl="${*}"
dnsbl="${dnsbl// /.}"
ip="${ip0}.${ip1}.${ip2}.${ip3}"
# find the score
dnsbl_get_score "${reply}" "${dnsbl}"
local score="${DNSBL_GET_SCORE}"
ADNS_COUNT[${ip}]=$[ ADNS_COUNT[${ip}] + 1 ]
ADNS_SCORE[${ip}]=$[ ADNS_SCORE[${ip}] + score ]
ADNS_LISTS[${ip}]="${ADNS_LISTS[${ip}]} ${score}/${reply}/${dnsbl}"
# save the matches log
echo "${score} ${ip} # ${reply} from ${dnsbl}" >>matches.log
# let the user know
[ ${DEBUG} -eq 1 ] && echo >&2 "MATCH (${score}, total ${ADNS_SCORE[${ip}]}): ${ip} on ${reply}/${dnsbl}"
return 0
}
blacklist() {
local counter="${1}" ip="${2}" tscore="${3}" comment=
shift 3
if [ ${counter} -eq 0 ]
then
comment="not matched by any list"
elif [ ${counter} -eq 1 ]
then
comment="score ${tscore} from ${counter} list:${*}"
else
comment="score ${tscore} from ${counter} lists:${*}"
fi
if [ ${tscore} -ge ${BLACKLIST_SCORE} ]
then
blacklist_check ${counter} ${ip} ${tscore} "${@}"
if [ $? -ne 0 ]
then
local d=$[tscore + 1 - BLACKLIST_SCORE]
tscore=$[tscore - d]
comment="score ${tscore} from custom check $[-d] and ${counter} lists:${*}"
fi
fi
if [ ${tscore} -ge ${BLACKLIST_SCORE} ]
then
printf >&2 " + ${COLOR_BGRED}${COLOR_WHITE}${COLOR_BOLD} %-9.9s %-15.15s ${COLOR_RESET}${COLOR_CYAN} # ${comment}${COLOR_RESET}\n" BLACKLIST "${ip}"
echo "${ip} # ${comment}" >>blacklist.log
# if it is already blacklisted, return
ipset --test ${BLACKLIST_IPSET} ${ip} 2>/dev/null && return 1
# blacklist it
if [ ${BLACKLIST_IPSET_COMMENTS} -eq 1 ]
then
ipset --add ${BLACKLIST_IPSET} ${ip} ${BLACKLIST_IPSET_OPTIONS} comment "${comment:0:255}" || exit 1
else
ipset --add ${BLACKLIST_IPSET} ${ip} ${BLACKLIST_IPSET_OPTIONS} || exit 1
fi
return 0
fi
printf >&2 " - ${COLOR_BGGREEN}${COLOR_BLACK} %-9.9s ${COLOR_RESET} %-15.15s ${COLOR_CYAN} # ${comment}${COLOR_RESET}\n" CLEAN "${ip}"
echo "${ip} # ${comment}" >>clean.log
return 1
}
generate_dnsbl_hostnames() {
if [[ "${1}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]
then
# check if it is excluded
local x=
for x in ${EXCLUSION_IPSETS} ${CACHE_IPSET} ${BLACKLIST_IPSET}
do
ipset --test "${x}" "${1}" 2>/dev/null && return 1
done
# split the IP in its parts
local ip=()
IFS="." read -ra ip <<< "${1}"
local i="${ip[0]}.${ip[1]}.${ip[2]}.${ip[3]}"
# let the user know
[ ${DEBUG} -eq 1 ] && echo >&2 "RECEIVED: ${i}"
echo "${i}" >>ips.log
# cache it
ipset --add ${CACHE_IPSET} ${i} ${CACHE_IPSET_OPTIONS} || exit 1
while [ -f "${THROTTLE_LOCK_FILE}" ]
do
[ ${DEBUG} -eq 1 ] && echo >&2 " >>> THROTLING: waiting..."
sleep 1
done
# generate the lookup hostnames for all configured lists
local x= prekey=
for x in ${!DNSBL[@]}
do
echo "${DNSBL_PREKEYS[${x}]}${ip[3]}.${ip[2]}.${ip[1]}.${ip[0]}.${x}"
done
return 0
else
# oops! this does not look like an IP
echo >&2 "ERROR: INVALID SOURCE IP: ${1}"
return 1
fi
}
generate_hostnames_from_src_dst() {
local last= a= b= x=
if [ "${TEST_SRC_DST}" -eq 1 ]
then
cat >&2
exit 0
fi
while read a b
do
[ "${a}" = "${b}" ] && a=
[ "${a}" = "${last}" ] && a=
[ "${b}" = "${last}" ] && b=
for x in ${a} ${b}
do
generate_dnsbl_hostnames "${x}"
[ ! -z "${DELAY_BETWEEN_CHECKS}" ] && sleep ${DELAY_BETWEEN_CHECKS}
last="${x}"
done
done
}
declare -A ADNS_REMAINING=()
declare -A ADNS_COUNT=()
declare -A ADNS_SCORE=()
declare -A ADNS_LISTS=()
ADNS_COMPLETED=0
ADNS_BLACKLISTED=0
parse_adns_asynch() {
local id n status t1 reason host dollar msg1 msg2 msg3 msg4 msg5 ip=() throttle="waiting" i=
# remove the throttle lock file, if present
[ -f "${THROTTLE_LOCK_FILE}" ] && rm "${THROTTLE_LOCK_FILE}"
[ ${RUNNING_ON_TERMINAL} -eq 1 ] && spinner "${#ADNS_REMAINING[@]}, waiting for the first IP..."
while read id n status t1 reason host dollar msg1 msg2 msg3 msg4 msg5
do
[ ${RUNNING_ON_TERMINAL} -eq 1 ] && spinner_end
# split the host
IFS="." read -ra ip <<< "${host}"
if [[ ! "${ip[0]}" =~ ^[0-9]+$ ]]
then
# it has a prekey
unset ip[0]
ip=("${ip[@]}")
fi
i="${ip[3]}.${ip[2]}.${ip[1]}.${ip[0]}"
# if we don't know it, add it
if [ -z "${ADNS_REMAINING[$i]}" ]
then
echo >&2 " > IP ${i}"
ADNS_REMAINING[$i]="${#DNSBL[@]}"
ADNS_COUNT[$i]=0
ADNS_SCORE[$i]=0
ADNS_LISTS[$i]=""
[ ${DEBUG} -eq 1 ] && echo "FIRST ${i} (${ADNS_REMAINING[$i]})"
fi
# decrement it my one
ADNS_REMAINING[$i]=$[ADNS_REMAINING[$i] - 1]
[ ${DEBUG} -eq 1 ] && echo "MINUS ${i} (${ADNS_REMAINING[$i]})"
# handle the response
case "${status}" in
ok) # positive response, parse it
local h= a= inet= reply=
while [ ${n} -gt 0 ]
do
read h a inet reply
[ ! "${h}" = "${host}" ] && echo "ERROR: ${h} <> ${host}"
match "${reply}" "${ip[@]}"
n=$[n - 1]
done
;;
permfail)
;;
default)
echo >&2 "ERROR: Unknown response ${status}"
;;
esac
# if it is done, remove it
if [ ${ADNS_REMAINING[$i]} -eq 0 ]
then
blacklist "${ADNS_COUNT[$i]}" "${i}" "${ADNS_SCORE[$i]}" "${ADNS_LISTS[$i]}" && ADNS_BLACKLISTED=$[ADNS_BLACKLISTED + 1]
ADNS_COMPLETED=$[ADNS_COMPLETED + 1]
unset ADNS_REMAINING[$i]
unset ADNS_COUNT[$i]
unset ADNS_SCORE[$i]
unset ADNS_LISTS[$i]
[ ${DEBUG} -eq 1 ] && echo "DONE ${i}"
#[ ${DEBUG} -eq 1 ] && declare -p ADNS_REMAINING ADNS_SCORE ADNS_COUNT ADNS_LISTS
fi
if [ "${#ADNS_REMAINING[@]}" -ge ${THROTTLE_THRESHOLD} ]
then
if [ ! -f "${THROTTLE_LOCK_FILE}" ]
then
touch "${THROTTLE_LOCK_FILE}"
[ ${DEBUG} -eq 1 ] && echo >&2 " >>> THROTHLING: there are ${#ADNS_REMAINING[@]} IPs in queue..."
fi
throttle="THROTTLING"
elif [ -f "${THROTTLE_LOCK_FILE}" ]
then
[ ${DEBUG} -eq 1 ] && echo >&2 " >>> THROTLING: resuming operations...."
rm "${THROTTLE_LOCK_FILE}"
throttle="waiting"
else
throttle="waiting"
fi
if [ ${RUNNING_ON_TERMINAL} -eq 1 ]
then
if [ ${ADNS_COMPLETED} -gt 0 ]
then
spinner "${#ADNS_REMAINING[@]}, checked ${ADNS_COMPLETED}, blacklisted ${ADNS_BLACKLISTED} ($[ADNS_BLACKLISTED * 100 / ADNS_COMPLETED] pcent) ${throttle}..."
else
spinner "${#ADNS_REMAINING[@]} waiting..."
fi
fi
done
}
# -----------------------------------------------------------------------------
# the main loop
# A pipeline:
# 1. tail the log
# 2. grep the lines we are interested
# 3. replace the lines with: SRC_IP DST_IP
# 4. generate hostnames for each IP
# 5. let adnshost do the lookups
# 6. parse the adnshost responses and take action
echo >&2 "Using ulogd iptables log: ${IPTABLES_LOG}"
echo >&2 "Searching for: ${ULOG_MATCH}"
echo >&2 "DNSBLs configured: ${!DNSBL[@]}"
echo >&2
echo >&2 "Please wait some time... pipes are filling up... (this is not a joke)"
[ ${DEBUG} -eq 1 ] && declare -p DNSBL DNSBL_SCORES DNSBL_PREKEYS
tail -s 0.2 -F "${IPTABLES_LOG}" |\
grep --line-buffered -E "^.*${ULOG_MATCH}.* SRC=[0-9.]+ DST=[0-9.]+ .*$" |\
sed --unbuffered "s/^.* SRC=\([0-9\.]\+\) DST=\([0-9\.]\+\) .*$/\1 \2/g" |\
generate_hostnames_from_src_dst |\
adnshost --asynch --fmt-asynch --no-env --pipe |\
parse_adns_asynch
exit 0
Reference in New Issue View Git Blame Copy Permalink