Responsible for downloading blacklists and creating/modifying IPSets Does all work involving creating, deleting and changing IPTables and IPSets.
Signed-off-by: Tim FitzGeorge ipfr@tfitzgeorge.me.uk --- src/scripts/ipblacklist | 1382 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1382 insertions(+) create mode 100755 src/scripts/ipblacklist
diff --git a/src/scripts/ipblacklist b/src/scripts/ipblacklist new file mode 100755 index 000000000..6f950214c --- /dev/null +++ b/src/scripts/ipblacklist @@ -0,0 +1,1382 @@ +#! /usr/bin/perl + +############################################################################ +# # +# IP Address blocklists for IPFire # +# # +# This is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 3 of the License, or # +# (at your option) any later version. # +# # +# This is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with IPFire; if not, write to the Free Software # +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +# # +# Copyright (C) 2018 - 2020 The IPFire team # +# # +############################################################################ +# # +# This script uses a file containing blacklist details in # +# /var/ipfire/ipblacklist/sources as well as # +# /var/ipfire/ipblacklistsettings containing an enable/disable flag for # +# each source. # +# # +# Two IPTables chains are used: BLACKLISTIN and BLACKLISTOUT are inserted # +# inserted into the main INPUT, OUTPUT and FORWARD chains; they capture # +# packets other than for the ICMP protocol. # +# # +# For each blacklist that is loaded, a chain is created to optionally log # +# and then to drop matching packets. An IPSet is created containing the # +# addresses or networks blocked by the blacklist, and then rules are added # +# to the BLACKLISTIN and BLACKLISTOUT chains to jump to this chain if a # +# packet list matches the set. # +# # +# When checking for updates, the modification time is used for each source # +# and if necessary the list is downloaded. The downloaded list is # +# compared to the existing IPSet contents and entries created or deleted # +# as necessary. # +# # +############################################################################ + +use strict; +#use warnings; + +use Carp; +use Sys::Syslog qw(:standard :macros); +use HTTP::Request; +use LWP::UserAgent; + +require "/var/ipfire/general-functions.pl"; + +############################################################################ +# Configuration variables +# +# These variables give the locations of various files used by this script +############################################################################ + +my $settingsdir = "/var/ipfire/ipblacklist"; +my $savedir = "/var/lib/ipblacklist"; +my $tmpdir = "/var/tmp"; + +my $settings = "$settingsdir/settings"; +my $sources = "$settingsdir/sources"; +my $checked = "$settingsdir/checked"; +my $modified = "$settingsdir/modified"; +my $iptables_list = "/var/tmp/iptables.txt"; +my $getipstat = "/usr/local/bin/getipstat"; +my $iptables = "/sbin/iptables"; +my $ipset = "/usr/sbin/ipset"; +my $fcrontab = "/usr/bin/fcrontab"; +my $lockfile = "/var/run/ipblacklist.pid"; +my $proxy_settings = "${General::swroot}/proxy/settings"; +my $red_setting = "/var/ipfire/red/iface"; +my $detailed_log = "$tmpdir/ipblacklist_log.txt"; +my $active = "/var/ipfire/red/active"; + +# Other configuration items + +my $margin = 30; # Scheduling allowance for run time etc in seconds +my $count = 30; # Maximum time to wait for another instance (300s) +my $max_dl_fails = 3; # Ignore check rate limit for this number of failures +my $max_size_fraction = 0.7; # Maximum fill fraction of IPSet before enlarging. +my $min_ipset_entries = 1024; # The minimum size of an IPSet. +my $max_dl_bytes = 10_485_760; # Maximum number of bytes to download. +my %parsers = ( 'ip-or-net-list' => &parse_ip_or_net_list, + 'dshield' => &parse_dshield ); + +############################################################################ +# Default settings +# Should be overwritten by reading settings files +############################################################################ + +my %sources = ( ); + +my %settings = ( 'DEBUG' => 0, + 'LOGGING' => 'on', + 'ENABLE' => 'off' ); + +my %proxy_settings = ( 'UPSTREAM_PROXY' => '' ); # No Proxy in use + +############################################################################ +# Function prototypes +############################################################################ + +sub abort( $ ); +sub create_list( $ ); +sub create_ipset( $$$ ); +sub debug( $$ ); +sub delete_list( $ ); +sub disable_logging(); +sub disable_updates(); +sub do_delete(); +sub do_start(); +sub do_stop(); +sub do_update(); +sub download_list( $$$ ); +sub download_check_header_time( $$$ ); +sub download_wget( $$$ ); +sub enable_logging(); +sub enable_updates(); +sub get_ipsets(); +sub get_rate_seconds( $ ); +sub iptables( $ ); +sub ipset( $ ); +sub stop_ipset(); +sub is_connected(); +sub log_message( $$ ); +sub parse_dshield( $ ); +sub parse_ip_or_net_list( $ ); +sub read_ipset( $$$$ ); +sub update_list( $$$ ); + +############################################################################ +# Variables +############################################################################ + +my %chains; # The Blacklist IPSets already loaded +my %old_blacklist; # Already blocked IP Addresses and/or networks + # downloaded for current blacklist +my $update_status = 0; # Set to 1 to update status file +my $ipset_running = 0; # Set to 1 if IPSet process is running +my %status; # Status information +my %checked; # Time blacklists last changed +my %modified; # Time blacklists last modified +my $red_iface; # The name of the red interface + +############################################################################ +# Synchronise runs +############################################################################ + +# This script can be triggered either by cron or the WUI. If another +# instance is running, wait for it to finish or timeout. + +while (-r $lockfile and $count > 0) +{ + open LOCKFILE, '<', $lockfile or (abort "Can't open lockfile", last); + my $pid = <LOCKFILE>; + close LOCKFILE; + + chomp $pid; + + last unless (-e "/proc/$pid"); + + sleep 10; + $count--; +} + +# Create pid file before starting main processing + +open LOCKFILE, '>', '/var/run/ipblacklist.pid' or abort "Can't open PID file: $!"; +print LOCKFILE "$$\n"; +close LOCKFILE; + +############################################################################ +# Set up for update +############################################################################ + +mkdir $settingsdir unless (-d $settingsdir); + +# Connect to the system log + +openlog( "ipblacklist", "nofatal", LOG_USER); +log_message LOG_INFO, "Starting IP Blacklist processing"; + +# Read settings + +General::readhash( $settings, %settings ) if (-e $settings); +General::readhash( $checked, %checked ) if (-e $checked); +General::readhash( $modified, %modified ) if (-e $modified); +General::readhash( $proxy_settings, %proxy_settings ) if (-e $proxy_settings); + +if (-r $sources) +{ + debug 1, "Reading sources file"; + + eval qx|/bin/cat $sources|; +} + +if (-r $red_setting) +{ + open REDIF, '<', $red_setting or (abort "Can't open red interface file", exit); + $red_iface = <REDIF>; + chomp $red_iface; + close REDIF; +} + +if (@ARGV) +{ + foreach my $cmd (@ARGV) + { + if ('update' =~ m/^$cmd/i) + { + # Called hourly when enabled and on setting changes. + # Update the blacklists. + + if ($settings{'ENABLE'} eq 'on') + { + do_update(); + } + } + elsif ('start' =~ m/^$cmd/i) + { + # Called during system startup. + # Restore saved blacklists. + # Don't do an update since can take too long. + + do_start() if ($settings{'ENABLE'} eq 'on'); + } + elsif ('stop' =~ m/^$cmd/i) + { + # Called when shutting down. + # Delete IPSets and IPTables chains + + do_stop(); + } + elsif ('restore' =~ m/^$cmd/i) + { + # Called after restoring backup. + # Delete IPSets and IPTables chains, then re-create them with the new + # (restored) settings and make sure updates are enabled. + + do_stop(); + + if ($settings{'ENABLE'} eq 'on') + { + do_start(); + enable_updates(); + } + else + { + disable_updates(); + } + } + elsif ('log-on' =~ m/^$cmd/i) + { + # Called from WUI. + # Create entries in IPTables chains to log dropped packets. + + if ($settings{'ENABLE'} eq 'on') + { + enable_logging(); + } + } + elsif ('log-off' =~ m/^$cmd/i) + { + # Called from WUI + # Delete entries in IPTables chains to log dropped packets. + + disable_logging(); + } + elsif ('enable' =~ m/^$cmd/i) + { + # Called from WUI to enable blacklists + # Do an update and then enable automatic updates + + if ($settings{'ENABLE'} eq 'on') + { + do_update(); + enable_updates(); + } + } + elsif ('disable' =~ m/^$cmd/i) + { + # Called from WUI to disable blacklists. + # Disable updates, delete IPSets, IPTables chains and save files. + + disable_updates(); + do_delete(); + } + else + { + print "Usage: $0 [update|start|stop|restart|log-on|log-off|enable|disable]\n"; + } + } +} +elsif ($settings{'ENABLE'} eq 'on') # Default action if none specified +{ + do_update(); +} + +stop_ipset(); + +if ($update_status) +{ + debug 1, "Writing updated status file"; + + General::writehash( $checked, %checked ); + General::writehash( $modified, %modified ); +} + +# Remove the pid file + +unlink $lockfile; + +log_message LOG_INFO, "Finished IP Blacklist processing"; +closelog(); + + +#------------------------------------------------------------------------------ +# sub do_stop +# +# Deletes all the IPTables chains and the IPSets +#------------------------------------------------------------------------------ + +sub do_stop() +{ + get_ipsets(); + + log_message LOG_NOTICE, "Stopping IP Blacklists"; + + foreach my $list ( sort keys %sources ) + { + delete_list( $list ) if (exists $chains{$list}); + } +} + + +#------------------------------------------------------------------------------ +# sub do_start +# +# Recreates the IPTables chains and the IPSets from the saved values +#------------------------------------------------------------------------------ + +sub do_start() +{ + log_message LOG_NOTICE, "Starting IP Blacklists"; + + foreach my $list ( sort keys %sources ) + { + delete_list( $list ) if (exists $chains{$list}); # Make sure OK to start + + if ((-e "$savedir/$list.conf") and ($red_iface)) + { + log_message LOG_INFO, "Restoring blacklist $list"; + system( "$ipset restore -f $savedir/$list.conf" ); # Can't use the ipset + # function to do this + + create_list( $list ); + } + } +} + + +#------------------------------------------------------------------------------ +# sub do_delete +# +# Deletes the IPTables chains, the IPSets and the saved values. +#------------------------------------------------------------------------------ + +sub do_delete() +{ + # Get the list of current ipsets + + get_ipsets(); + + log_message LOG_NOTICE, "Deleting IP Blacklists"; + + foreach my $source ( sort keys %sources ) + { + if (exists $chains{$source}) + { + delete_list( $source ); + } + + if (-e "$savedir/$source.conf") + { + unlink "$savedir/$source.conf"; + } + } + + %modified = (); + $update_status = 1; +} + + +#------------------------------------------------------------------------------ +# sub do_update +# +# Updates all the blacklists. +# Creates or deletes the blacklist firewall rules as necessary and checks for +# updates to the blacklists. Each blacklist has its own minimum elapsed time +# between updates, which is specified in the sources file, so the time of each +# check is stored. +#------------------------------------------------------------------------------ + +sub do_update() +{ + return unless ($red_iface); + + # Get the list of current ipsets + + get_ipsets(); + + # Check sources + + debug 1, "Checking blacklist sources"; + + LIST: + foreach my $list ( sort keys %sources ) + { + my @new_blacklist = (); + my $name = $sources{$list}{'name'}; + my $last_checked = $checked{$list} || 0; + my $failures = $checked{"${list}_failures"} || 0; + my $enabled = 0; + + if (exists $settings{$list}) + { + $enabled = $settings{$list} eq 'on'; + } + + if ($enabled and is_connected()) + { + debug 1, "Checking blacklist source: $name"; + + # Calculate the per list rate + + my $rate = get_rate_seconds( $sources{$list}{'rate'} ); + + # Has enough time passed since the last time we checked the list? + # Ignore the limit if the last download failed + + if (($last_checked + $rate) < (time() + $margin) or + ($failures > 0 and $failures < $max_dl_fails)) + { + my $type = 'hash:ip'; + + download_list( $list, @new_blacklist, $type ); + + next LIST unless (@new_blacklist); + + if (not exists $chains{$list}) + { + # Doesn't currently exist: Create it. + + create_ipset( $list, $type, scalar @new_blacklist ); + create_list( $list ); + } + + update_list( $list, @new_blacklist, $type ); + } + } + elsif (exists $chains{$list}) + { + # Exists, but not enabled: Delete it. + + delete_list( $list ); + + # Delete the save file + # Don't delete the checked time from the status, in case the list is + # re-enabled quickly - don't want to exceed maximum allowed download + # rate. + + unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf"); + + delete $modified{$list} if (exists $modified{$list}); + delete $checked{"${list}_failures"} if (exists $checked{"${list}_failures"}); + $update_status = 1; + } + } + + # Check for any lists that don't exist any more + + foreach my $list (keys %modified) + { + next if (exists $sources{$list}); + + delete_list( $list ); + + # Delete the save file + + unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf"); + + # Delete from the status + + delete $modified{$list} if (exists $modified{$list}); + $update_status = 1; + } + + foreach my $list (keys %checked) + { + next if ($list =~ m/_failures/); + next if (exists $sources{$list}); + + delete $checked{$list}; + delete $checked{"${list}_failures"}; + delete $settings{$list} if (exists $settings{$list}); + $update_status = 1; + } +} + + +#------------------------------------------------------------------------------ +# sub get_rate_seconds( text ) +# +# Converts a check rate into seconds. A sanity check is made on the coverted +# value. +# +# Parameters: +# text The value to convert in the form nnnu, where nnn is a number and u +# is either m (minutes), h (hours) or d (days). Hours is assumed if +# not specified and everything after the first letter is ignored. +#------------------------------------------------------------------------------ + +sub get_rate_seconds( $ ) +{ + my ($text) = @_; + + my ($value, $unit) = (uc $text) =~ m/(\d+)([DHM]?)/; + + if ($unit eq 'D') # Days + { + $value *= 60 * 60 * 24; + } + elsif ($unit eq 'M') # Minutes + { + $value *= 60; + } + else # Everything else - assume hours + { + $value *= 60 * 60; + } + + # Sanity check - limit to range 5 min .. 1 week + + # d h m s + $value = 5 * 60 if ($value < 5 * 60); + $value = 7 * 24 * 60 * 60 if ($value > 7 * 24 * 60 * 60); + + return $value; +} + + +#------------------------------------------------------------------------------ +# sub is_connected() +# +# Checks that the system is connected to the internet. +# +# This looks for a file created by IPFire when connected to the internet +#------------------------------------------------------------------------------ + +sub is_connected() +{ + return (-e $active); +} + + +#------------------------------------------------------------------------------ +# sub create_list( list ) +# +# Creates a new IPTables chain for a blacklist source. +# The set must be created before calling this function. Two rules are added to +# the chain: +# (optional) 1 Log the packet +# 2 Drop the packet +# +# The log rule is only added when logging is enabled by the WUI. +# +# Rules are then added to the BLACKLISTIN and BLACKLISTOUT chains that check +# the packet's IP address against the IPSet and then jump to the newly created +# chain. +# +# Parameters: +# list The name of the blacklist +#------------------------------------------------------------------------------ + +sub create_list( $ ) +{ + my ($list) = @_; + + log_message LOG_INFO, "Create IPTables chains for blacklist $list"; + + # Create new chain in filter table + + iptables( "-N ${list}_DROP" ) == 0 or + ( abort "Could not create IPTables chain ${list}_DROP", return ); + + # Add the logging and drop rules + + if ($settings{'LOGGING'} eq 'on') + { + iptables( "-A ${list}_DROP -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" ) == 0 or + ( abort "Could not create IPTables chain $list LOG rule", return ); + } + + iptables( "-A ${list}_DROP -j DROP" ) == 0 or + ( abort "Could not create IPTables chain $list drop rule", return ); + + # Add the rules to check against the set + + iptables( "-A BLACKLISTIN -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ); + iptables( "-A BLACKLISTOUT -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ); +} + + +#------------------------------------------------------------------------------ +# sub delete_list( $list ) +# +# Deletes an IPTables chain when a blacklist source is disabled. Also flushes +# and destroys the IPSet. +# +# Parameters: +# list The name of the blacklist +#------------------------------------------------------------------------------ + +sub delete_list( $ ) +{ + my ($list) = @_; + + log_message LOG_INFO, "Delete IPTables chains for blacklist $list"; + + # Remove the blacklist chains from the main INPUT and OUTPUT chains + + iptables( "-D BLACKLISTIN -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ); + iptables( "-D BLACKLISTOUT -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ); + + # Flush and delete the chain + + iptables( "-F ${list}_DROP" ); + iptables( "-X ${list}_DROP" ); + + # Flush and delete the set + + ipset( "flush $list" ); + ipset( "destroy $list" ); +} + + +#------------------------------------------------------------------------------ +# sub download_list( list, ref_list, ref_type ) +# +# Downloads the IP Addresses for a blacklist. +# +# Once downloaded the list is parsed to get the IP addresses and/or networks. +# +# Parameters: +# list The name of the blacklist +# ref_list A reference to an array to store the downloaded blacklist +# ref_type A reference to store the type of the blacklist +#------------------------------------------------------------------------------ + +sub download_list( $$$ ) +{ + my ($list, $new_blacklist, $type) = @_; + + $checked{$list} = time(); # Record that the list has been checked + $update_status = 1; + + # Check the parser for the blacklist + + if (not exists $parsers{ $sources{$list}{'parser'} }) + { + log_message LOG_ERR, "Can't find parser $sources{$list}{'parser'} for $list blacklist"; + return; + } + + # Add alternative download mechanisms here + + download_check_header_time( $list, $new_blacklist, $type ); +} + + +#------------------------------------------------------------------------------ +# sub download_check_header_time( list, ref_list, ref_type ) +# +# Updates the IP Addresses for a blacklist. The If-Modified-Since header is +# specified in the request so that only updated lists are downloaded (providing +# that the server supports this functionality). +# +# Once downloaded the list is parsed to get the IP addresses and/or networks. +# +# Parameters: +# list The name of the blacklist +# ref_list A reference to an array to store the downloaded blacklist +# ref_type A reference to store the type of the blacklist +# +# Returns: +# The list type: 'hash:ip' or 'hash:net' +#------------------------------------------------------------------------------ + +sub download_check_header_time( $$$ ) +{ + my ($list, $new_blacklist, $type) = @_; + my $found_ip = 0; + my $found_net = 0; + + # Get the parser for the blacklist + + my $parser = $parsers{ $sources{$list}{'parser'} }; + + debug 1, "Checking for blacklist $list updates with LWP"; + + # Create a user agent for downloading the blacklist + # Limit the download size for safety + + my $ua = LWP::UserAgent->new( max_size => $max_dl_bytes ); + + # Get the Proxy settings + + if ($proxy_settings{'UPSTREAM_PROXY'}) + { + if ($proxy_settings{'UPSTREAM_USER'}) + { + $ua->proxy( [["http", "https"] => "http://$proxy_settings%7B%27UPSTREAM_USER%27%7D:$proxy_settings%7B%27UPSTREA..."] ); + } + else + { + $ua->proxy( [["http", "https"] => "http://$proxy_settings%7B%27UPSTREAM_PROXY%27%7D/"] ); + } + } + + # Get the last modified time + + my $modified = gmtime( $modified{$list} || 0 ); + + # Download the blacklist + + my $response = $ua->get( $sources{$list}{'url'}, 'If-Modified-Since' => $modified ); + + if (not $response->is_success) + { + if ($response->code == 304) + { + # Not an error - list has not been modified + debug 1, "Blacklist $list not modified"; + + return; + } + + log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: ". $response->status_line; + $checked{"${list}_failures"}++; + + return; + } + + $modified{$list} = $response->last_modified; + $checked{"${list}_failures"} = 0; + + # Parse the downloaded list, checking if it's a list of addresses or nets + + foreach my $line (split /[\r\n]+/, $response->content) + { + chomp $line; + + my $address = &$parser( $line ); + + next unless ($address and $address =~ m/\d+.\d+.\d+.\d+/); + + if ($address =~ m|/32|) + { + $address =~ s|/32||; + $found_ip = 1; + } + elsif ($address =~ m|/\d+|) + { + $found_net = 1; + } + else + { + $found_ip = 1; + } + + push @{ $new_blacklist }, $address; + } + + if ($found_net and $found_ip) + { + # Convert mixed addresses and networks to all networks + + foreach my $address (@{ $new_blacklist }) + { + $address .= '/32' unless ($address =~ m|/\d+|); + } + + $found_ip = 0; + } + + $$type = $found_net ? 'hash:net' : 'hash:ip'; +} + + +#------------------------------------------------------------------------------ +# sub read_ipset( list, old, type, maxelem ) +# +# Reads the existing contents and type of the set. +# +# Parameters: +# list The name of the blacklist +# old Reference to array to contain blacklist +# type Reference to type +# maxelem Reference to maximum number of elements +#------------------------------------------------------------------------------ + +sub read_ipset( $$$$ ) +{ + my ($list, $old, $type, $maxelem) = @_; + my $found_net = 0; + my $found_ip = 0; + + debug 2, "Reading existing ipset for blacklist $list"; + + foreach my $line (qx/$ipset list $list/) + { + if ($line =~ m|Header:.*maxelem (\d+)|) + { + $$maxelem = $1; + next; + } + + next unless ($line =~ m|(\d+.\d+.\d+.\d+(?:/\d+)?)|); + + my $address = $1; + + if ($address =~ m|/32|) + { + $found_ip = 1; + $address =~ s|/32$||; + } + elsif ($address =~ m|/\d+$|) + { + $found_net = 1; + } + else + { + $found_ip = 1; + } + + $$old{$address} = 1; + } + + if ($found_ip and $found_net) + { + # Convert mixed addresses and networks to all networks + + my @ads_list = keys %{ $old }; + + foreach my $address (@ads_list) + { + unless ($address =~ m|/\d+|) + { + delete $$old{$address}; + $$old{"$address/32"} = 1; + } + } + + $found_ip = 0; + } + + $$type = $found_net ? 'hash:net' : 'hash:ip'; +} + + +#------------------------------------------------------------------------------ +# sub update_list( list, new, new_type ) +# +# Updates the IP Addresses for a blacklist +# +# The new list is compared to the existing list and new entries added or old +# entries deleted as necessary. If the list type ('hash:ip' or 'hash:net') has +# changed then the IPSet is deleted and re-created with the new type. +# +# Parameters: +# list The name of the blacklist +# new Reference to array of new blacklist entries +# new_type The type of the updated list (hash:ip or hash:net) +#------------------------------------------------------------------------------ + +sub update_list( $$$ ) +{ + my ($list, $new, $new_type) = @_; + my %old; + my $old_type; + my $changes = 0; + my $maxelem = 0; + + debug 1, "Checking for $list blacklist update from $sources{$list}{'url'}"; + + if (exists $chains{$list} ) + { + my $recreate_ipset = 0; + + read_ipset( $list, %old, $old_type, $maxelem ); + + # Check the IPSet type hasn't changed + + if ($new_type ne $old_type) + { + log_message LOG_NOTICE, "Blacklist $list changed type from $old_type to $new_type"; + $recreate_ipset = 1; + } + + if ($max_size_fraction * $maxelem < scalar @{ $new } ) + { + log_message LOG_NOTICE, "Blacklist $list changed size from $maxelem"; + $recreate_ipset = 1; + } + + if ($recreate_ipset) + { + # Change the IPSet type and/or size. This requires removing references + # to it first. We could delete and then create the chain, but doing it + # like this keeps the statistics. + + # Remove the IPSet from the IPTables chains + + iptables( "-D 'BLACKLISTIN' -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ) == 0 or + log_message LOG_ERR, "Could not remove ${list} from BLACKLISTIN chain"; + + iptables( "-D 'BLACKLISTOUT' -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ) == 0 or + log_message LOG_ERR, "Could not remove ${list} from BLACKLISTOUT chain"; + + # Flush and delete the old set + + ipset( "flush $list" ); + ipset( "destroy $list" ); + + %old = (); # Since we've deleted the old set it can't have any entries. + + # Create the new ipset + + create_ipset( $list, $new_type, scalar @{ $new } ); + + # Add the rules to check against the set + + iptables( "-A 'BLACKLISTIN' -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ) == 0 or + log_message LOG_ERR, "Could not add IPSet $list to BLACKLISTIN chain"; + + iptables( "-A 'BLACKLISTOUT' -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ) == 0 or + log_message LOG_ERR, "Could not add IPSet $list to BLACKLISTOUT chain"; + } + } + + # Process the blacklist + + foreach my $address ( @{ $new } ) + { + # We've got an address. Add to IPSet if it's new + + if (exists $old{$address}) + { + delete $old{$address}; # Not new - don't delete from chain later + } + else + { + ipset( "add $list $address -exist" ); # New - add it + + $changes++; + } + + debug 3, "Add net $address to blacklist $list"; + } + + # Delete old entries that aren't needed any more + + debug 2, "Removing deleted rules from IPTables chain for blacklist $list"; + + foreach my $address ( keys %old ) + { + ipset( "del $list $address" ); + + $changes++; + + debug 3, "Delete old net $address from blacklist $list"; + } + + log_message LOG_INFO, "Updated $list blacklist with $changes changes"; + + # Save the blacklist for the next reboot + + mkdir "$savedir" unless (-d "$savedir" ); + + ipset( "save $list -file $savedir/$list.conf" ) if ($changes > 0); + + stop_ipset(); +} + + +#------------------------------------------------------------------------------ +# sub get_ipsets( ) +# +# Gets a list of the current IPSets +#------------------------------------------------------------------------------ + +sub get_ipsets( ) +{ + debug 1, "Reading list of existing ipsets"; + + my @sets = qx($ipset -n list); + + # Parse the tables + + foreach my $line (@sets) + { + chomp $line; + + next unless ($line); + + $chains{$line} = 1; + } +} + + +#------------------------------------------------------------------------------ +# sub create_ipset( list, type, size ) +# +# Creates a new IPSet. The current size of the set is determined by taking the +# next power of two greater than the number of entries; the maximum size is set +# to double this, subject to a minimum size. This allows for future expansion. +# +# Parameters: +# list The name of the blacklist +# type The type of the blacklist (hash:ip or hash:net) +# size The number of entries in the lsit +#------------------------------------------------------------------------------ + +sub create_ipset( $$$ ) +{ + my ($list, $type, $size) = @_; + + my $hashsize = 1; + $hashsize <<= 1 while ($hashsize < $size); + my $maxsize = $hashsize * 2; + $maxsize = $min_ipset_entries if ($maxsize < $min_ipset_entries); + + # Create the new ipset + ipset( "create $list $type hashsize $hashsize maxelem $maxsize" ); + stop_ipset(); # Need to do this to action the IPSet commands +} + + +#------------------------------------------------------------------------------ +# sub enable_logging() +# +# Enable logging of packets dropped by IP Blacklist rules. +# This adds a rule to log the packet to each lists' IPTables chain. +#------------------------------------------------------------------------------ + +sub enable_logging() +{ + get_ipsets(); + + log_message LOG_NOTICE, "Enabling IP Blacklist logging"; + + foreach my $list ( sort keys %sources ) + { + if (exists $chains{$list}) + { + iptables( "-I ${list}_DROP 1 -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" ); + } + } +} + + +#------------------------------------------------------------------------------ +# sub disable_logging() +# +# Disable logging of packets dropped by IP Blacklist rules. +# This deletes a rule to log the packet from each lists' IPTables chain. +#------------------------------------------------------------------------------ + +sub disable_logging() +{ + get_ipsets(); + + log_message LOG_NOTICE, "Disabling IP Blacklist logging"; + + foreach my $list ( sort keys %sources ) + { + if (exists $chains{$list}) + { + iptables( "-D ${list}_DROP -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" ); + } + } +} + + +#------------------------------------------------------------------------------ +# sub enable_updates() +# +# Adds a command to the fcrontab to run the update hourly. +# If there is a command already in the fcrontab to do this it will be +# uncommented, otherwise a new line is added. +# +# The update is executed at an offset from the hour so that all the users don't +# try to download the updates at exactly the same time - the blacklists are +# provided free, so it's good manners to spread the load on the servers. The +# offset is initialised to a random number that avoids running on the hour +# (when a lot of other things happen), and every fifteen minutes thereafter. +#------------------------------------------------------------------------------ + +sub enable_updates() +{ + my @lines = qx/$fcrontab -l/; + my $found = 0; + + # Check for an existing fcrontab entry. + + foreach my $line (@lines) + { + if ($line =~ m|/usr/local/bin/ipblacklist|) + { + return if ($line !~ m/^#/); # Already enabled - do nothing + + # Already in fcrontab - uncomment the line + + $line =~ s/^#+//; + $found = 1; + log_message LOG_INFO, "Enable IP Address Blacklist update in crontab"; + last; + } + } + + if (not $found) + { + # Add a new entry to fcrontab + + my $start = int( rand(13) ) + 1; + + my $times = $start; + + for (my $offset = $times+15 ; $offset < 60 ; $offset += 15) + { + $times .= ",$offset"; + } + + push @lines, "\n"; + push @lines, "# IP Blacklist update\n"; + push @lines, "%nice(1) $times * * * * /usr/local/bin/ipblacklist\n"; + log_message LOG_INFO, "Add IP Address Blacklist update to crontab"; + } + + open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return); + print FCRONTAB @lines; + close FCRONTAB; +} + + +#------------------------------------------------------------------------------ +# sub disable_updates() +# +# Comments out the entry in the fcrontab that runs the updates. +#------------------------------------------------------------------------------ + +sub disable_updates() +{ + my @lines = qx/$fcrontab -l/; + my $found = 0; + + foreach my $line (@lines) + { + if ($line =~ m|/usr/local/bin/ipblacklist|) + { + return if ($line =~ m/^#/); # Already disabled - do nothing + + # In fcrontab - comment the line + + $line =~ s/^#*/#/; + $found = 1; + log_message LOG_INFO, "Disable IP Address Blacklist updates"; + last; + } + } + + return unless ($found); # Don't update crontab unnecessarily + + open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return); + print FCRONTAB @lines; + close FCRONTAB; +} + + +#------------------------------------------------------------------------------ +# sub parse_ip_or_net_list( line ) +# +# Parses an input line, looking for lines starting with an IP Address or +# Network specification. +# +# Parameters: +# line The line to parse +# +# Returns: +# Either an IP Address or a null string +#------------------------------------------------------------------------------ + +sub parse_ip_or_net_list( $ ) +{ + my ($line) = @_; + + $line =~ m|^(\d+.\d+.\d+.\d+(?:/\d+)?)|; + + return $1; +} + + +#------------------------------------------------------------------------------ +# sub parse_dshield( line ) +# +# Parses an input line removing comments. +# +# The format is: +# Start Addrs End Addrs Netmask Nb Attacks Network Name Country email +# We're only interested in the start address and netmask. +# +# Parameters: +# line The line to parse +# +# Returns: +# Either and IP Address or a null string +#------------------------------------------------------------------------------ + +sub parse_dshield( $ ) +{ + my ($line) = @_; + + return "" if ($line =~ m/^\s*#/); + + $line =~ s/#.*$//; + + # |Start addrs | |End Addrs | |Mask + $line =~ m|(\d+.\d+.\d+.\d+(?:/\d+)?)\s+\d+.\d+.\d+.\d+(?:/\d+)?\s+(\d+)|; + + return unless ($1); + return "$1/32" unless ($2); + + return "$1/$2"; +} + + +#------------------------------------------------------------------------------ +# sub iptables( cmd ) +# +# Executes an IPTables command, waiting for the internal lock to ensure only +# one change is made at a time. +# +# Parameters: +# cmd The command to execute +# +# Returns: +# Status of command +#------------------------------------------------------------------------------ + +sub iptables( $ ) +{ + my ($cmd) = @_; + + return system( "$iptables $cmd" ); +} + + +#------------------------------------------------------------------------------ +# sub ipset( cmd ) +# +# Executes an IPSet command. The command is piped to a sub-process running +# ipset, rather than exected separately. This saves the overhead of starting a +# new process for each command. The sub-process is started if it's not already +# running. +# +# Note that the pipe is buffered so commands are not necessarily executed +# immediately. Use ipset_stop() to force commands to be executed. This should +# be done before relying on anything that the ipset commands do, for example +# before referencing the IPSet in an IPTables command. +# +# Parameters: +# cmd The command to execute +#------------------------------------------------------------------------------ + +sub ipset( $ ) +{ + my ($cmd) = @_; + + if (not $ipset_running) + { + local $SIG{PIPE} = 'IGNORE'; + open IPSET, "|-", $ipset, "restore" or die "Can't start ipset: $!"; + $ipset_running = 1; + } + + print IPSET "$cmd\n"; +} + + +#------------------------------------------------------------------------------ +# sub stop_ipset( ) +# +# Stops the ipset sub-process. +# This causes any pending ipset commands to be executed. +#------------------------------------------------------------------------------ + +sub stop_ipset( ) +{ + if ($ipset_running) + { + close IPSET or abort "ipset process died: $! $?"; + $ipset_running = 0; + } +} + + +#------------------------------------------------------------------------------ +# sub abort( message, parameters... ) +# +# Used when aborting the current activity, printing out an error message. +# +# Parameters: +# message Message to be printed +#------------------------------------------------------------------------------ + +sub abort( $ ) +{ + my ($message) = @_; + + log_message( LOG_ERR, $message ); + carp $message; +} + + +#------------------------------------------------------------------------------ +# sub log_message( level, message ) +# +# Logs a message. If the script is run from a terminal messages are also +# output on STDOUT. +# +# Parameters: +# level Severity of message +# message Message to be logged +#------------------------------------------------------------------------------ + +sub log_message( $$ ) +{ + my ($level, $message) = @_; + + print "($level) $message\n" if (-t STDIN); + syslog( $level, $message ); +} + + +#------------------------------------------------------------------------------ +# sub debug( level, message ) +# +# Optionally logs a debug message. If the script is run from a terminal, level +# 1 debug messages are output regardless of the debug setting. +# +# Parameters: +# level Debug level +# message Message to be logged +#------------------------------------------------------------------------------ + +sub debug( $$ ) +{ + my ($level, $message) = @_; + + if (($level <= $settings{'DEBUG'}) or + ($level == 1 and -t STDIN)) + { + log_message LOG_DEBUG, $message; + } +}