Responsible for downloading blacklists and creating/modifying IPSets Does all work involving creating, deleting and chaging IPTables and IPSets.
Signed-off-by: Tim FitzGeorge ipfr@tfitzgeorge.me.uk --- src/scripts/ipblacklist | 1558 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1558 insertions(+) create mode 100755 src/scripts/ipblacklist
diff --git a/src/scripts/ipblacklist b/src/scripts/ipblacklist new file mode 100755 index 000000000..b3f8048d9 --- /dev/null +++ b/src/scripts/ipblacklist @@ -0,0 +1,1558 @@ +#! /usr/bin/perl + +############################################################################ +# # +# IP Address blocklists for IPFire # +# # +# This is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 3 of the License, or # +# (at your option) any later version. # +# # +# This is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with IPFire; if not, write to the Free Software # +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +# # +# Copyright (C) 2018 - 2019 The IPFire team # +# # +############################################################################ +# # +# This script use a file containing blacklist details in # +# /var/ipfire/ipblacklist/sources as well as # +# /var/ipfire/ipblacklistsettings containing an enable/disable flag for # +# each source. # +# # +# Two IPTables chains are used, IPBLACKLISTREDIN and IPBLACKLISTREDOUT, # +# which are inserted into the main INPUT, OUTPUT and FORWARD chains. # +# # +# For each blacklist that is loaded, a chain is created to optionally log # +# and then to drop matching packets. An IPSet is created containing the # +# addresses or networks blocked by the blacklist, and then rules are added # +# to the IPBLACKLISTREDIN and IPBLACKLISTREDOUT chains to jump to this # +# chain if appropriate packet list matches in the set. # +# # +# When checking for updates, the modification time is read for each source # +# and if necessary the list is downloaded. The downloaded list is # +# compared to the existing IPSet contents and entries created or deleted # +# as necessary. # +# # +############################################################################ + +use strict; +use warnings; + +use Carp; +use Sys::Syslog qw(:standard :macros); +use HTTP::Request; +use LWP::UserAgent; + +require "/var/ipfire/general-functions.pl"; + +############################################################################ +# Configuration variables +# +# These variables give the locations of various files used by this script +############################################################################ + +my $settingsdir = "/var/ipfire/ipblacklist"; +my $savedir = "/var/lib/ipblacklist"; +my $tmpdir = "/var/tmp"; + +my $settings = "$settingsdir/settings"; +my $sources = "$settingsdir/sources"; +my $checked = "$settingsdir/checked"; +my $modified = "$settingsdir/modified"; +my $iptables_list = "/var/tmp/iptables.txt"; +my $getipstat = "/usr/local/bin/getipstat"; +my $iptables = "/sbin/iptables"; +my $ipset = "/usr/sbin/ipset"; +my $fcrontab = "/usr/bin/fcrontab"; +my $lockfile = "/var/run/ipblacklist.pid"; +my $proxy_settings = "${General::swroot}/proxy/settings"; +my $red_setting = "/var/ipfire/red/iface"; +my $detailed_log = "$tmpdir/ipblacklist_log.txt"; +my $autoblacklist = 'AUTOBLACKLIST'; + +my %parsers = ( 'text-with-hash-comments' => &parse_text_with_hash_comments, + 'text-with-semicolon-comments' => &parse_text_with_semicolon_comments, + 'dshield' => &parse_dshield ); + +############################################################################ +# Default settings +# Should be overwritten by reading settings files +############################################################################ + +my %sources = ( ); + +my %settings = ( 'DEBUG' => 0, + 'LOGGING' => 'on', + 'RATE' => 24, + 'ENABLE' => 'off' ); + +my %proxy_settings = ( 'UPSTREAM_PROXY' => '' ); # No Proxy in use + +############################################################################ +# Function prototypes +############################################################################ + +sub abort( $ ); +sub autoblacklist_update(); +sub autoblacklist_clear(); +sub create_autoblacklist(); +sub create_list( $ ); +sub create_ipset( $$$ ); +sub debug( $$ ); +sub delete_autoblacklist(); +sub delete_list( $ ); +sub disable_logging(); +sub disable_updates(); +sub do_delete(); +sub do_start(); +sub do_stop(); +sub do_update(); +sub download_list( $$$ ); +sub download_check_header_time( $$$ ); +sub download_wget( $$$ ); +sub enable_logging(); +sub enable_updates(); +sub get_ipsets(); +sub iptables( $ ); +sub ipset( $ ); +sub stop_ipset(); +sub is_connected(); +sub log_message( $$ ); +sub parse_dshield( $ ); +sub parse_text_with_hash_comments( $ ); +sub parse_text_with_semicolon_comments( $ ); +sub read_ipset( $$$ ); +sub update_list( $$$ ); + +############################################################################ +# Variables +############################################################################ + +my %chains; # The Blacklist IPSets already loaded +my %old_blacklist; # Already blocked IP Addresses and/or networks + # downloaded for current blacklist +my $update_status = 0; # Set to 1 to update status file +my $ipset_running = 0; # Set to 1 if IPSet process is running +my %status; # Status information +my %checked; # Time blacklists last changed +my %modified; # Time blacklists last modified +my $red_iface; # Name of red interface +my $hours = 3600; # One hour in seconds +my $margin = 600; # Allowance for run time etc +my $count = 30; # Maximum time to wait for another instance (300s) +my @wget_status = ( 'Success', 'Error', 'Parse Error', 'File I/O Error', + 'Network Error', 'SSL Verification Error', + 'Authentication Error', 'Protocol Error', 'Server Error' ); + + +############################################################################ +# Synchronise runs +############################################################################ + +# This script can be triggered either by cron or the WUI. If another +# instance is running, wait for it to finish. + +while (-r $lockfile and $count > 0) +{ + open LOCKFILE, '<', $lockfile or die "Can't open lockfile"; + my $pid = <LOCKFILE>; + close LOCKFILE; + + chomp $pid; + + last unless (-e "/proc/$pid"); + + sleep 10; + $count--; +} + +# Create pid file before starting main processing + +open LOCKFILE, '>', '/var/run/ipblacklist.pid' or die "Can't open PID file: $!"; +print LOCKFILE "$$\n"; +close LOCKFILE; + +############################################################################ +# Set up for update +############################################################################ + +mkdir $settingsdir unless (-d $settingsdir); + +# Connect to the system log + +openlog( "ipblacklist", "nofatal", LOG_USER); +log_message LOG_INFO, "Starting IP Blacklist processing"; + +# Read settings + +General::readhash( $settings, %settings ) if (-e $settings); +General::readhash( $checked, %checked ) if (-e $checked); +General::readhash( $modified, %modified ) if (-e $modified); +General::readhash( $proxy_settings, %proxy_settings ) if (-e $proxy_settings); + +if (-r $sources) +{ + debug 1, "Reading sources file"; + + eval qx|/bin/cat $sources|; +} + +# Find out the red interface name + +if (-r $red_setting) +{ + open IN, '<', $red_setting or die "Can't open red interface name file: $!"; + + $red_iface = <IN>; + chomp $red_iface; + + close IN; +} + +if (@ARGV) +{ + foreach my $cmd (@ARGV) + { + if ('update' =~ m/^$cmd/i) + { + # Called hourly when enabled and on setting changes. + # Update the blacklists. + + if ($settings{'ENABLE'} eq 'on') + { + do_update(); + } + } + elsif ('start' =~ m/^$cmd/i) + { + # Called during system startup. + # Restore saved blacklists. + # Don't do an update since that takes too long. + + do_start() if ($settings{'ENABLE'} eq 'on'); + } + elsif ('stop' =~ m/^$cmd/i) + { + # Called when shutting down. + # Delete IPSets and IPTables chains + + do_stop(); + } + elsif ('restore' =~ m/^$cmd/i) + { + # Called after restoring backup. + # Delete IPSets and IPTables chains, then re-create them with the new + # (restored) settings and make sure updates are enabled. + + do_stop(); + + if ($settings{'ENABLE'} eq 'on') + { + do_start(); + enable_updates(); + } + else + { + disable_updates(); + } + } + elsif ('log-on' =~ m/^$cmd/i) + { + # Called from WUI. + # Create entries in IPTables chains to log dropped packets. + + if ($settings{'ENABLE'} eq 'on') + { + enable_logging(); + } + } + elsif ('log-off' =~ m/^$cmd/i) + { + # Called from WUI + # Delete entries in IPTables chains to log dropped packets. + + disable_logging(); + } + elsif ('enable' =~ m/^$cmd/i) + { + # Called from WUI to enable blacklists + # Do an update and then enable automatic updates + + if ($settings{'ENABLE'} eq 'on') + { + do_update(); + enable_updates(); + } + } + elsif ('disable' =~ m/^$cmd/i) + { + # Called from WUI to disable blacklists. + # Disable updates, delete IPSets, IPTables chains and save files. + + disable_updates(); + do_delete(); + } + elsif ('autoblacklist-update' =~ m/^$cmd/i) + { + # Updates AUTOBLACKLIST options + + autoblacklist_update(); + } + elsif ('autoblacklist-clear' =~ m/^$cmd/i) + { + # Clears AUTOBLACKLIST contents + + autoblacklist_clear(); + } + else + { + print "Usage: $0 [update|start|stop|restart|log-on|log-off|enable|disable|autoblacklist-update|autoblacklist-clear]\n"; + } + } +} +elsif ($settings{'ENABLE'} eq 'on') +{ + do_update(); +} + +stop_ipset(); + +if ($update_status) +{ + debug 1, "Writing updated status file"; + + General::writehash( $checked, %checked ); + General::writehash( $modified, %modified ); +} + +# Remove the pid file + +unlink $lockfile; + +log_message LOG_INFO, "Finished IP Blacklist processing"; +closelog(); + + +#------------------------------------------------------------------------------ +# sub do_stop +# +# Deletes all the IPTables chains and the IPSets +#------------------------------------------------------------------------------ + +sub do_stop() +{ + get_ipsets(); + + log_message LOG_NOTICE, "Stopping IP Blacklists"; + + foreach my $list ( $autoblacklist, sort keys %sources ) + { + if (exists $chains{$list}) + { + delete_list( $list ); + } + } +} + + +#------------------------------------------------------------------------------ +# sub do_start +# +# Recreates the IPTables chains and the IPSets from the saved values +#------------------------------------------------------------------------------ + +sub do_start() +{ + log_message LOG_NOTICE, "Starting IP Blacklists"; + + foreach my $list ( sort keys %sources ) + { + if (-e "$savedir/$list.conf") + { + log_message LOG_INFO, "Restoring blacklist $list"; + system( "$ipset restore -f $savedir/$list.conf" ); + + create_list( $list ); + } + } + + if ($settings{$autoblacklist} eq 'on') + { + create_autoblacklist(); + } +} + + +#------------------------------------------------------------------------------ +# sub do_delete +# +# Deletes the IPTables chains, the IPSets and the saved values. +#------------------------------------------------------------------------------ + +sub do_delete() +{ + # Get the list of current ipsets + + get_ipsets(); + + log_message LOG_NOTICE, "Deleting IP Blacklists"; + + foreach my $source ( sort keys %sources ) + { + if (exists $chains{$source}) + { + delete_list( $source ); + } + + if (-e "$savedir/$source.conf") + { + unlink "$savedir/$source.conf"; + } + } + + if ($settings{$autoblacklist} eq 'on') + { + delete_autoblacklist(); + } +} + + +#------------------------------------------------------------------------------ +# sub do_update +# +# Updates all the blacklists. +# Creates or deletes the blacklist firewall rules as necessary and checks for +# updates to the blacklists. +#------------------------------------------------------------------------------ + +sub do_update() +{ + return unless (is_connected()); + + my $type = 'hash:ip'; + + # Get the list of current ipsets + + get_ipsets(); + + # Check sources + + debug 1, "Checking blacklist sources"; + + foreach my $list ( sort keys %sources ) + { + my @new_blacklist = (); + my $name = $sources{$list}{'name'}; + my $rate = $sources{$list}{'rate'}; + my $last_checked = $checked{$list} || 0; + my $enabled = 0; + + if (exists $modified{$list}) + { + # Limit the check rate to the minimum defined in the WUI, unless we're + # creating the list + + $rate = $settings{'RATE'} if ($settings{'RATE'} > $rate); + } + + if (exists $settings{$list}) + { + $enabled = $settings{$list} eq 'on'; + } + + debug 1, "Checking blacklist source: $name"; + + if ($enabled) + { + # Has enough time passed since the last time we checked the list? + + if (($last_checked + $rate * $hours) < (time() + $margin)) + { + download_list( $list, @new_blacklist, $type ); + + next unless (@new_blacklist); + + if (not exists $chains{$list}) + { + # Doesn't currently exist: Create it. + + create_ipset( $list, $type, scalar @new_blacklist ); + create_list( $list ); + } + + update_list( $list, @new_blacklist, $type ); + } + } + elsif (exists $chains{$list}) + { + # Exists, but not enabled: Delete it. + + delete_list( $list ); + + # Delete the save file + # Don't delete the checked time from the status, in case the list is + # re-enabled quickly - don't want to exceed maximum allowed download + # rate. + + unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf"); + + delete $modified{$list} if (exists $modified{$list}); + $update_status = 1; + } + } + + # Check for any deleted lists + + foreach my $list (keys %sources) + { + if (not exists $sources{$list}) + { + delete_list( $list ); + + # Delete the save file + + unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf"); + + # Delete from the status + + delete $modified{$list} if (exists $modified{$list}); + delete $checked{$list} if (exists $checked{$list}); + $update_status = 1; + } + } + + if ($settings{$autoblacklist} eq 'on') + { + create_autoblacklist() if (not exists $chains{$autoblacklist}); + } + else + { + delete_autoblacklist() if (exists $chains{$autoblacklist}); + } + + log_message LOG_INFO, "Completed IP Blacklist update"; +} + + +#------------------------------------------------------------------------------ +# sub autoblacklist_update() +# +# Updates the settings for the AUTOBLACKLIST +#------------------------------------------------------------------------------ + +sub autoblacklist_update() +{ + # Get the list of current ipsets + + get_ipsets(); + + # Delete the existing AUTOBLACKLIST, if it currently exists. + + delete_autoblacklist() if (exists $chains{$autoblacklist}); + + # Re-create the AUTOBLACKLIST with the correct parameters. + + create_autoblacklist() if ($settings{$autoblacklist} eq 'on'); +} + + +#------------------------------------------------------------------------------ +# sub autoblacklist_clear() +# +# Clears the contents of the AUTOBLACKLIST +#------------------------------------------------------------------------------ + +sub autoblacklist_clear() +{ + log_message LOG_INFO, "Flush Automatic blacklist"; + ipset( "flush $autoblacklist" ); +} + + +#------------------------------------------------------------------------------ +# sub is_connected() +# +# Checks that the system is connected to the internet. +# +# This looks for a file created by IPFire when connected to the internet +#------------------------------------------------------------------------------ + +sub is_connected() +{ + return (-e "${General::swroot}/red/active"); +} + + +#------------------------------------------------------------------------------ +# sub create_list( list ) +# +# Creates a new IPTables chain for a blacklist source. +# The set must be created before calling this function. +# +# Parameters: +# list The name of the blacklist +#------------------------------------------------------------------------------ + +sub create_list( $ ) +{ + my ($list) = @_; + + log_message LOG_INFO, "Create IPTables chains for blacklist $list"; + + # Create new chain in filter table + + iptables( " -N ${list}_BLOCK" ) == 0 or + ( abort "Could not create IPTables chain ${list}_BLOCK", return ); + + # Add the logging and drop rules + + if ($settings{'LOGGING'} eq 'on') + { + iptables( "-A ${list}_BLOCK -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" ) == 0 or + ( abort "Could not create IPTables chain $list LOG rule", return ); + } + + iptables( "-A ${list}_BLOCK -j DROP" ) == 0 or + ( abort "Could not create IPTables chain $list drop rule", return ); + + # Add the rules to check against the set + + iptables( "-A IPBLACKLISTREDIN -p ALL -m set --match-set $list src -j ${list}_BLOCK" ); + iptables( "-A IPBLACKLISTREDOUT -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ); +} + + +#------------------------------------------------------------------------------ +# sub create_autoblacklist() +# +# Creates a new IPTables chain for the AUTOBLACKLIST. This also creates the +# IPSet with the correct timeout. +#------------------------------------------------------------------------------ + +sub create_autoblacklist() +{ + return unless ($red_iface); # Can't add rule to policy unless this is set + + # Create the set for the AUTOBLACKLIST + + ipset( "create $autoblacklist hash:ip timeout $settings{BLOCK_PERIOD}" ); + + # Create new chain in filter table + + create_list( $autoblacklist ); + + # For the AUTOBLACKLIST there are extra rules to reset the timeout on the + # blockled addresses + + iptables( "-I ${autoblacklist}_BLOCK -m set --match-set $autoblacklist src -j SET --add-set $autoblacklist src --exist" ); + iptables( "-I ${autoblacklist}_BLOCK -m set --match-set $autoblacklist dst -j SET --add-set $autoblacklist dst --exist" ); + + # For the AUTOBLACKLIST there is an extra rule to add an entry to the list + # of blocked addresses. This is added to the input policy chain. + + iptables( "-I POLICYIN 1 -i $red_iface -m hashlimit --hashlimit-mode srcip --hashlimit-above $settings{BLOCK_THRESHOLD}/hour --hashlimit-name $autoblacklist -j SET --add-set $autoblacklist src" ); +} + + +#------------------------------------------------------------------------------ +# sub delete_list( $list ) +# +# Deletes an IPTables chain when a blacklist source is disabled. Also flushes +# and destroys the IPSet. +# +# Parameters: +# list The name of the blacklist +#------------------------------------------------------------------------------ + +sub delete_list( $ ) +{ + my ($list) = @_; + + log_message LOG_INFO, "Delete IPTables chains for blacklist $list"; + + # Remove the blacklist chains from the main INPUT and OUTPUT chains + + iptables( "-D IPBLACKLISTREDIN -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not remove IPSet $list from IPBLACKLISTREDIN chain"; + + iptables( "-D IPBLACKLISTREDOUT -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not remove IPSet $list from IPBLACKLISTREDOUT chain"; + + # Flush and delete the chain + + iptables( "-F ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not flush IPTables chain ${list}_BLOCK"; + + iptables( "-X ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not delete IPTables chain ${list}_BLOCK"; + + # Flush and delete the set + + ipset( "flush $list" ); + + ipset( "destroy $list" ); +} + + +#------------------------------------------------------------------------------ +# sub delete_autoblacklist() +# +# Deletes the autoblacklist IPTables chain when it is disabled. Also flushes +# and destroys the IPSet. +#------------------------------------------------------------------------------ + +sub delete_autoblacklist() +{ + # For the AUTOBLACKLIST there is an extra rule to remove + + unless ($red_iface) + { + iptables( "-D POLICYIN -i $red_iface -m hashlimit --hashlimit-mode srcip --hashlimit-above $settings{BLOCK_THRESHOLD}/hour --hashlimit-name $autoblacklist -j SET --add-set $autoblacklist src" ); + } + + # Now do a normal delete + + delete_list( $autoblacklist ); +} + + +#------------------------------------------------------------------------------ +# sub download_list( chain, ref_list, ref_type ) +# +# Updates the IP Addresses for a blacklist. Depending on the blacklist one of +# two methods are used: +# +# - For some lists the header is downloaded and the modification date checked. +# If newer than the existing list, the update is downloaded. +# - For other lists this is not supported,so the whole file has to be +# downloaded regardless. +# +# Once downloaded the list is parsed to get the IP addresses and/or networks. +# +# Parameters: +# list The name of the blacklist +# ref_list A reference to an array to store the downloaded blacklist +# ref_type A reference to store the type of the blacklist +#------------------------------------------------------------------------------ + +sub download_list( $$$ ) +{ + my ($list, $new_blacklist, $type) = @_; + + $checked{$list} = time(); + $update_status = 1; + + # Check the parser for the blacklist + + if (not exists $parsers{ $sources{$list}{'parser'} }) + { + log_message LOG_ERR, "Can't find parser $sources{$list}{'parser'} for $list blacklist"; + return; + } + + if ($sources{$list}{'method'} eq 'check-header-time') + { + download_check_header_time( $list, $new_blacklist, $type ); + } + else + { + download_wget( $list, $new_blacklist, $type ); + } +} + + +#------------------------------------------------------------------------------ +# sub download_check_header_time( chain, ref_list, ref_type ) +# +# Updates the IP Addresses for a blacklist. The header is downloaded and the +# modification date checked. If newer than the existing list, the update is +# downloaded. +# +# Once downloaded the list is parsed to get the IP addresses and/or networks. +# +# Parameters: +# list The name of the blacklist +# ref_list A reference to an array to store the downloaded blacklist +# ref_type A reference to store the type of the blacklist +#------------------------------------------------------------------------------ + +sub download_check_header_time( $$$ ) +{ + my ($list, $new_blacklist, $type) = @_; + my $found_ip = 0; + my $found_net = 0; + + # Get the parser for the blacklist + + my $parser = $parsers{ $sources{$list}{'parser'} }; + + log_message LOG_INFO, "Checking modification time for blacklist $list update with LWP"; + + # Create a user agent for downloading the blacklist + # Limit the download size for safety (10 MiB) + + my $ua = LWP::UserAgent->new( max_size => 10485760 ); + + # Get the Proxy settings + + if ($proxy_settings{'UPSTREAM_PROXY'}) + { + if ($proxy_settings{'UPSTREAM_USER'}) + { + $ua->proxy("http" => "http://$proxy_settings%7B%27UPSTREAM_USER%27%7D:$proxy_settings%7B%27UPSTREA..."); + $ua->proxy("https" => "http://$proxy_settings%7B%27UPSTREAM_USER%27%7D:$proxy_settings%7B%27UPSTREA..."); + } + else + { + $ua->proxy("http" => "http://$proxy_settings%7B%27UPSTREAM_PROXY%27%7D/"); + $ua->proxy("https" => "http://$proxy_settings%7B%27UPSTREAM_PROXY%27%7D/"); + } + } + + # Get the blacklist modification time from the internet + + my $request = HTTP::Request->new( HEAD => $sources{$list}{'url'} ); + + my $response = $ua->request( $request ); + + if (not $response->is_success) + { + log_message LOG_WARNING, "Failed to download $list header $sources{$list}{'url'}: ". $response->status_line; + + return; + } + + # Has the blacklist been modified since we last read it? + + if (exists $modified{$list} and $modified{$list} >= $response->last_modified) + { + # We've already got this version of the blacklist + + debug 1, "Blacklist $list not modified"; + return; + } + + debug 1, "Blacklist $list Modification times: old " . $modified{$list} . ", new " . $response->last_modified if (exists $modified{$list}); + log_message LOG_INFO, "Downloading blacklist $list with LWP"; + + # Download the blacklist + + $request = HTTP::Request->new( GET => $sources{$list}{'url'} ); + $response = $ua->request($request); + + if (not $response->is_success) + { + log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: ". $response->status_line; + + return; + } + + $modified{$list} = $response->last_modified; + + foreach my $line (split /[\r\n]+/, $response->content) + { + chomp $line; + + my $address = &$parser( $line ); + + next unless ($address and $address =~ m/\d+.\d+.\d+.\d+/); + + if ($address =~ m|/\d+|) + { + $found_net = 1; + } + else + { + $found_ip = 1; + } + + push @{ $new_blacklist }, $address; + } + + if ($found_net and $found_ip) + { + # Convert mixed address and network set to all network + + foreach my $address (@{ $new_blacklist }) + { + $address .= '/32' unless ($address =~ m|/\d+|); + } + + $found_ip = 0; + } + + $$type = $found_net ? 'hash:net' : 'hash:ip'; +} + + +#------------------------------------------------------------------------------ +# sub download_wget( chain, ref_list, ref_type ) +# +# Updates the IP Addresses for a blacklist. The whole file is download with +# wget and then the modification time compared with the stored modification +# time. If the update is newer then the downloaded list is parsed. +# +# Once downloaded the list is parsed to get the IP addresses and/or networks. +# +# Parameters: +# list The name of the blacklist +# ref_list A reference to an array to store the downloaded blacklist +# ref_type A reference to store the type of the blacklist +#------------------------------------------------------------------------------ + +sub download_wget( $$$ ) +{ + my ($list, $new_blacklist, $type) = @_; + my $wget_proxy = ''; + my $found_ip = 0; + my $found_net = 0; + + my $parser = $parsers{ $sources{$list}{'parser'} }; + + log_message LOG_INFO, "Downloading blacklist $list update with wget"; + + # Get the Proxy settings + + if ($proxy_settings{'UPSTREAM_PROXY'}) + { + if ($proxy_settings{'UPSTREAM_USER'}) + { + $wget_proxy = "--proxy=on --proxy-user=$proxy_settings{'UPSTREAM_USER'} --proxy-passwd=$proxy_settings{'UPSTREAM_PASSWORD'} -e http_proxy=http://$proxy_settings%7B%27UPSTREAM_PROXY%27%7D/"; + } + else + { + $wget_proxy = "--proxy=on -e http_proxy=http://$proxy_settings%7B%27UPSTREAM_PROXY%27%7D/"; + } + } + + my $retv = system( "wget $wget_proxy --no-show-progress -o $detailed_log -O $tmpdir/ipblacklist_$list $sources{$list}{'url'}" ); + + if ($retv != 0) + { + my $error = $wget_status[ $retv/256 ]; + log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: $error"; + return; + } + + my @file_info = stat( "$tmpdir/ipblacklist_$list" ); + + if (exists $modified{$list} and $modified{$list} >= $file_info[9]) + { + # We've already got this version of the blocklist + + debug 1, "Blacklist $list not modified"; + unlink "$tmpdir/ipblacklist_$list"; + return; + } + + open LIST, '<', "$tmpdir/ipblacklist_$list" or (abort "Can't open downloaded blacklist for $list: $!", return); + + $modified{$list} = $file_info[9]; + + foreach my $line (<LIST>) + { + chomp $line; + + my $address = &$parser( $line ); + + next unless ($address); + next unless ($address =~ m|\d+.\d+.\d+.\d+|); + + if ($address =~ m|/\d+|) + { + $found_net = 1; + } + else + { + $found_ip = 1; + } + + push @{ $new_blacklist }, $address; + } + + close LIST; + + unlink "$tmpdir/ipblacklist_$list"; + + if ($found_net and $found_ip) + { + # Convert mixed address and network set to all network + + foreach my $address (@{ $new_blacklist }) + { + $address .= '/32' unless ($address =~ m|/\d+|); + } + + $found_ip = 0; + } + + $$type = $found_net ? 'hash:net' : 'hash:ip'; +} + + +#------------------------------------------------------------------------------ +# sub read_ipset( list, old, type ) +# +# Reads the existing contents of the set +# +# Parameters: +# chain The name of the blacklist +# old Reference to array to contain blacklist +# type Reference to type +#------------------------------------------------------------------------------ + +sub read_ipset( $$$ ) +{ + my ($list, $old, $type) = @_; + my $found_net = 0; + my $found_ip = 0; + + debug 2, "Reading existing ipset for blacklist $list"; + + foreach my $line (qx/$ipset list $list/) + { + next unless ($line =~ m|(\d+.\d+.\d+.\d+(?:/\d+)?)|); + + my $address = $1; + + if (($address =~ m|/\d+$|) and ($address !~ m|/32$|)) + { + $found_net = 1; + } + else + { + $found_ip = 1; + $address =~ s|/32$||; + } + + $$old{$address} = 1; + } + + if ($found_ip and $found_net) + { + # Convert mixed address and network set to all network + + my @ads_list = keys %{ $old }; + + foreach my $address (@ads_list) + { + unless ($address =~ m|/\d+|) + { + delete $$old{$address}; + $$old{"$address/32"} = 1; + } + } + + $found_ip = 0; + } + + $$type = $found_net ? 'hash:net' : 'hash:ip'; +} + + +#------------------------------------------------------------------------------ +# sub update_list( chain, new, new_type ) +# +# Updates the IP Addresses for a blacklist +# +# The new list is compared to the existing list and new entries added or old +# entries deleted as necessary. +# +# Parameters: +# list The name of the blacklist +# new Reference to array of new blacklist entries +# new_type The type of the updated list (hash:ip or hash:net) +#------------------------------------------------------------------------------ + +sub update_list( $$$ ) +{ + my ($list, $new, $new_type) = @_; + my %old; + my $old_type; + my $changes = 0; + + debug 2, "Checking for $list blacklist update from $sources{$list}{'url'}"; + + log_message LOG_INFO, "Updating $list blacklist"; + + read_ipset( $list, %old, $old_type ); + + # Check the IPSet type hasn't changed + + if ($new_type ne $old_type) + { + # Change the IPSet type. This requires removing references to it first. + # We could delete and then create the chain, but doing it like this keeps + # the statistics. + + log_message LOG_NOTICE, "Blacklist $list changed type from $old_type to $new_type"; + + # Remove the IPSet from the IPTables chains + + iptables( "-D 'IPBLACKLISTREDIN' -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not remove ${list} from IPBLACKLISTREDIN chain"; + + iptables( "-D 'IPBLACKLISTREDOUT' -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not remove ${list} from IPBLACKLISTREDOUT chain"; + + # Flush and delete the old set + + ipset( "flush $list" ); + ipset( "destroy $list" ); + + %old = (); + + # Create the new ipset + + create_ipset( $list, $new_type, scalar @{ $new } ); + + # Add the rules to check against the set + + iptables( "-A 'IPBLACKLISTREDIN' -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not add IPSet $list to IPBLACKLISTREDIN chain"; + + iptables( "-A 'IPBLACKLISTREDOUT' -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or + log_message LOG_ERR, "Could not add IPSet $list to IPBLACKLISTREDOUT chain"; + } + + # Process the blacklist + + foreach my $address ( @{ $new } ) + { + # We've got an address. Add to IPSet if it's new + + if (exists $old{$address}) + { + delete $old{$address}; # Not new + } + else + { + ipset( "add $list $address -exist" ); + + $changes++; + } + + debug 3, "Add net $address to blacklist $list"; + } + + # Delete old entries that aren't needed any more + + debug 2, "Removing deleted rules from IPTables chain for blacklist $list"; + + foreach my $address ( keys %old ) + { + ipset( "del $list $address" ); + + $changes++; + + debug 3, "Delete old net $address from blacklist $list"; + } + + + log_message LOG_INFO, "Finished updating $list blacklist with $changes changes"; + + # Save the blacklist for the next reboot + + mkdir "$savedir" unless (-d "$savedir" ); + + ipset( "save $list -file $savedir/$list.conf" ); + + stop_ipset(); +} + + +#------------------------------------------------------------------------------ +# sub get_ipsets( ) +# +# Gets a list of the current IPSets +#------------------------------------------------------------------------------ + +sub get_ipsets( ) +{ + debug 1, "Reading list of existing ipsets"; + + my @sets = qx($ipset -n list); + + # Parse the tables + + foreach my $line (@sets) + { + chomp $line; + + next unless ($line); + + $chains{$line} = 1; + } +} + + +#------------------------------------------------------------------------------ +# sub create_ipset( name, type, size ) +# +# Creates a new IPSet. The current and maximum size of the set are determined +# by taking the next power of two greater than the numer of entries, subject to +# a minimum size. This allows for future expansion. +# +# Parameters: +# name The name of the blacklist +# type The type of the blacklist (hash:ip or hash:net) +# size The number of entries in the lsit +#------------------------------------------------------------------------------ + +sub create_ipset( $$$ ) +{ + my ($name, $type, $size) = @_; + + my $hashsize = 1; + $hashsize <<= 1 while ($hashsize < $size); + my $maxsize = ($hashsize < 16384) ? 32768 : $hashsize * 2; + + # Create the new ipset + ipset( "create $name $type hashsize $hashsize maxelem $maxsize" ); + stop_ipset(); # Need to do this to action the IPSet commands +} + + +#------------------------------------------------------------------------------ +# sub enable_logging() +# +# Enable logging of packets dropped by IP Blacklist rules. +#------------------------------------------------------------------------------ + +sub enable_logging() +{ + get_ipsets(); + + log_message LOG_NOTICE, "Enabling IP Blacklist logging"; + + foreach my $list ( sort keys %sources ) + { + if (exists $chains{$list}) + { + iptables( "-I ${list}_BLOCK 1 -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" ); + } + } +} + + +#------------------------------------------------------------------------------ +# sub disable_logging() +# +# Disable logging of packets dropped by IP Blacklist rules. +#------------------------------------------------------------------------------ + +sub disable_logging() +{ + get_ipsets(); + + log_message LOG_NOTICE, "Disabling IP Blacklist logging"; + + foreach my $list ( sort keys %sources ) + { + if (exists $chains{$list}) + { + iptables( "-D ${list}_BLOCK -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" ); + } + } +} + + +#------------------------------------------------------------------------------ +# sub enable_updates() +# +# Adds a command to the fcrontab to run the update hourly. +# The update is executed at an offset from the hour so that all the users don't +# try to download the updates at exactly the same time - the blacklists are +# provided free, so it's good manners to spread the load on the servers. +#------------------------------------------------------------------------------ + +sub enable_updates() +{ + my @lines = qx/$fcrontab -l/; + my $found = 0; + + # Check for an existing fcrontab entry. + + foreach my $line (@lines) + { + if ($line =~ m|/usr/local/bin/ipblacklist|) + { + return if ($line !~ m/^#/); # Already enabled + + # Found - uncomment the line + + $line =~ s/^#+//; + $found = 1; + log_message LOG_INFO, "Enable IP Address Blacklist update in crontab"; + last; + } + } + + if (not $found) + { + # Add a new entry + + my $start = int( rand(50) ) + 5; + + push @lines, "\n"; + push @lines, "# IP Blacklist update\n"; + push @lines, "%hourly,nice(1),random,serial $start /usr/local/bin/ipblacklist\n"; + log_message LOG_INFO, "Add IP Address Blacklist update to crontab"; + } + + open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return); + print FCRONTAB @lines; + close FCRONTAB; +} + + +#------------------------------------------------------------------------------ +# sub disable_updates() +# +# Comments out the entry in the fcrontab that runs the updates. +#------------------------------------------------------------------------------ + +sub disable_updates() +{ + my @lines = qx/$fcrontab -l/; + my $found = 0; + + foreach my $line (@lines) + { + if ($line =~ m|/usr/local/bin/ipblacklist|) + { + return if ($line =~ m/^#/); # Already disabled + + # Found - comment the line + + $line =~ s/^#*/#/; + $found = 1; + log_message LOG_INFO, "Disable IP Address Blacklist updates"; + last; + } + } + + return unless ($found); # Don't update crontab unnecessarily + + open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return); + print FCRONTAB @lines; + close FCRONTAB; +} + + +#------------------------------------------------------------------------------ +# sub parse_text_with_hash_comments( line ) +# +# Parses an input line removing comments. +# +# Parameters: +# line The line to parse +# +# Returns: +# Either an IP Address or a null string +#------------------------------------------------------------------------------ + +sub parse_text_with_hash_comments( $ ) +{ + my ($line) = @_; + + return "" if ($line =~ m/^\s*#/); + + $line =~ s/#.*$//; + + $line =~ m|(\d+.\d+.\d+.\d+(?:/\d+)?)|; + + return $1; +} + + +#------------------------------------------------------------------------------ +# sub parse_text_with_semicolon_comments( line ) +# +# Parses an input line removing comments. +# +# Parameters: +# line The line to parse +# +# Returns: +# Either and IP Address or a null string +#------------------------------------------------------------------------------ + +sub parse_text_with_semicolon_comments( $ ) +{ + my ($line) = @_; + + return "" if ($line =~ m/^\s*;/); + + $line =~ s/;.*$//; + + $line =~ m|(\d+.\d+.\d+.\d+(?:/\d+)?)|; + + return $1; +} + + +#------------------------------------------------------------------------------ +# sub parse_dshield( line ) +# +# Parses an input line removing comments. +# +# The format is: +# Start Addrs End Addrs Netmask Nb Attacks Network Name Country email +# We're only interested in the start address and netmask. +# +# Parameters: +# line The line to parse +# +# Returns: +# Either and IP Address or a null string +#------------------------------------------------------------------------------ + +sub parse_dshield( $ ) +{ + my ($line) = @_; + + return "" if ($line =~ m/^\s*#/); + + $line =~ s/#.*$//; + + $line =~ m|(\d+.\d+.\d+.\d+(?:/\d+)?)\s+\d+.\d+.\d+.\d+(?:/\d+)?\s+(\d+)|; + + return unless ($1); + return "$1/32" unless ($2); + + return "$1/$2"; +} + + +#------------------------------------------------------------------------------ +# sub iptables( cmd ) +# +# Executes an IPTables command, waiting for the lock to ensure only one change +# is made at a time. +# +# Parameters: +# cmd The command to execute +# +# Returns: +# Status of command +#------------------------------------------------------------------------------ + +sub iptables( $ ) +{ + my ($cmd) = @_; + + return system( "$iptables $cmd" ); +} + + +#------------------------------------------------------------------------------ +# sub ipset( cmd ) +# +# Executes an IPSet command. The command is piped to a sub-process running +# ipset, rather than exected separately. This saves the overhead of starting a +# new process for each command. The sub-process is started if it's not already +# running. +# +# Parameters: +# cmd The command to execute +#------------------------------------------------------------------------------ + +sub ipset( $ ) +{ + my ($cmd) = @_; + + if (not $ipset_running) + { + local $SIG{PIPE} = 'IGNORE'; + open IPSET, "|-", $ipset, "restore" or die "Can't start ipset: $!"; + $ipset_running = 1; + } + + print IPSET "$cmd\n"; +} + + +#------------------------------------------------------------------------------ +# sub stop_ipset( ) +# +# Stops the ipset sub-process. +#------------------------------------------------------------------------------ + +sub stop_ipset( ) +{ + if ($ipset_running) + { + close IPSET or abort "ipset process died: $! $?"; + $ipset_running = 0; + } +} + + +#------------------------------------------------------------------------------ +# sub abort( message, parameters... ) +# +# Aborts the current activity, printing out an error message. +# +# Parameters: +# message Message to be printed +#------------------------------------------------------------------------------ + +sub abort( $ ) +{ + my ($message) = @_; + + log_message( LOG_ERR, $message ); + carp $message; +} + + +#------------------------------------------------------------------------------ +# sub log_message( level, message ) +# +# Logs a message. If the script is run from a terminal messages are also +# output on STDOUT. +# +# Parameters: +# level Severity of message +# message Message to be logged +#------------------------------------------------------------------------------ + +sub log_message( $$ ) +{ + my ($level, $message) = @_; + + print "($level) $message\n" if (-t STDIN); + syslog( $level, $message ); +} + + +#------------------------------------------------------------------------------ +# sub debug( level, message ) +# +# Optionally logs a debug message. If the script is run from a terminal, level +# 1 debug messages are output regardless of the debug setting. +# +# Parameters: +# level Debug level +# message Message to be logged +#------------------------------------------------------------------------------ + +sub debug( $$ ) +{ + my ($level, $message) = @_; + + if (($level <= $settings{'DEBUG'}) or + ($level == 1 and -t STDIN)) + { + log_message LOG_DEBUG, $message; + } +}