[PATCH v2 1/8] ipblacklist: Main script

[Tim FitzGeorge <ipfr@tfitzgeorge.me.uk>]

[-- Attachment #1: Type: text/plain, Size: 42504 bytes --]

Responsible for downloading blacklists and creating/modifying IPSets
Does all work involving creating, deleting and changing IPTables and
IPSets.

Signed-off-by: Tim FitzGeorge <ipfr(a)tfitzgeorge.me.uk>
---
 src/scripts/ipblacklist | 1382 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1382 insertions(+)
 create mode 100755 src/scripts/ipblacklist

diff --git a/src/scripts/ipblacklist b/src/scripts/ipblacklist
new file mode 100755
index 000000000..6f950214c
--- /dev/null
+++ b/src/scripts/ipblacklist
@@ -0,0 +1,1382 @@
+#! /usr/bin/perl
+
+############################################################################
+#                                                                          #
+# IP Address blocklists for IPFire                                         #
+#                                                                          #
+# This is free software; you can redistribute it and/or modify             #
+# it under the terms of the GNU General Public License as published by     #
+# the Free Software Foundation; either version 3 of the License, or        #
+# (at your option) any later version.                                      #
+#                                                                          #
+# This is distributed in the hope that it will be useful,                  #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of           #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
+# GNU General Public License for more details.                             #
+#                                                                          #
+# You should have received a copy of the GNU General Public License        #
+# along with IPFire; if not, write to the Free Software                    #
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA #
+#                                                                          #
+# Copyright (C) 2018 - 2020 The IPFire team                                #
+#                                                                          #
+############################################################################
+#                                                                          #
+# This script uses a file containing blacklist details in                  #
+# /var/ipfire/ipblacklist/sources as well as                               #
+# /var/ipfire/ipblacklistsettings containing an enable/disable flag for    #
+# each source.                                                             #
+#                                                                          #
+# Two IPTables chains are used: BLACKLISTIN and BLACKLISTOUT are inserted  #
+# inserted into the main INPUT, OUTPUT and FORWARD chains; they capture    #
+# packets other than for the ICMP protocol.                                #
+#                                                                          #
+# For each blacklist that is loaded, a chain is created to optionally log  #
+# and then to drop matching packets. An IPSet is created containing the    #
+# addresses or networks blocked by the blacklist, and then rules are added #
+# to the BLACKLISTIN and BLACKLISTOUT chains to jump to this chain if a    #
+# packet list matches the set.                                             #
+#                                                                          #
+# When checking for updates, the modification time is used for each source #
+# and if necessary the list is downloaded.  The downloaded list is         #
+# compared to the existing IPSet contents and entries created or deleted   #
+# as necessary.                                                            #
+#                                                                          #
+############################################################################
+
+use strict;
+#use warnings;
+
+use Carp;
+use Sys::Syslog qw(:standard :macros);
+use HTTP::Request;
+use LWP::UserAgent;
+
+require "/var/ipfire/general-functions.pl";
+
+############################################################################
+# Configuration variables
+#
+# These variables give the locations of various files used by this script
+############################################################################
+
+my $settingsdir    = "/var/ipfire/ipblacklist";
+my $savedir        = "/var/lib/ipblacklist";
+my $tmpdir         = "/var/tmp";
+
+my $settings       = "$settingsdir/settings";
+my $sources        = "$settingsdir/sources";
+my $checked        = "$settingsdir/checked";
+my $modified       = "$settingsdir/modified";
+my $iptables_list  = "/var/tmp/iptables.txt";
+my $getipstat      = "/usr/local/bin/getipstat";
+my $iptables       = "/sbin/iptables";
+my $ipset          = "/usr/sbin/ipset";
+my $fcrontab       = "/usr/bin/fcrontab";
+my $lockfile       = "/var/run/ipblacklist.pid";
+my $proxy_settings = "${General::swroot}/proxy/settings";
+my $red_setting    = "/var/ipfire/red/iface";
+my $detailed_log   = "$tmpdir/ipblacklist_log.txt";
+my $active         = "/var/ipfire/red/active";
+
+# Other configuration items
+
+my $margin            = 30;         # Scheduling allowance for run time etc in seconds
+my $count             = 30;         # Maximum time to wait for another instance (300s)
+my $max_dl_fails      = 3;          # Ignore check rate limit for this number of failures
+my $max_size_fraction = 0.7;        # Maximum fill fraction of IPSet before enlarging.
+my $min_ipset_entries = 1024;       # The minimum size of an IPSet.
+my $max_dl_bytes      = 10_485_760; # Maximum number of bytes to download.
+my %parsers           = ( 'ip-or-net-list' => \&parse_ip_or_net_list,
+                          'dshield'        => \&parse_dshield );
+
+############################################################################
+# Default settings
+# Should be overwritten by reading settings files
+############################################################################
+
+my %sources        = ( );
+
+my %settings       = ( 'DEBUG'           => 0,
+                       'LOGGING'         => 'on',
+                       'ENABLE'          => 'off' );
+
+my %proxy_settings = ( 'UPSTREAM_PROXY' => '' );         # No Proxy in use
+
+############################################################################
+# Function prototypes
+############################################################################
+
+sub abort( $ );
+sub create_list( $ );
+sub create_ipset( $$$ );
+sub debug( $$ );
+sub delete_list( $ );
+sub disable_logging();
+sub disable_updates();
+sub do_delete();
+sub do_start();
+sub do_stop();
+sub do_update();
+sub download_list( $$$ );
+sub download_check_header_time( $$$ );
+sub download_wget( $$$ );
+sub enable_logging();
+sub enable_updates();
+sub get_ipsets();
+sub get_rate_seconds( $ );
+sub iptables( $ );
+sub ipset( $ );
+sub stop_ipset();
+sub is_connected();
+sub log_message( $$ );
+sub parse_dshield( $ );
+sub parse_ip_or_net_list( $ );
+sub read_ipset( $$$$ );
+sub update_list( $$$ );
+
+############################################################################
+# Variables
+############################################################################
+
+my %chains;                # The Blacklist IPSets already loaded
+my %old_blacklist;         # Already blocked IP Addresses and/or networks
+                           # downloaded for current blacklist
+my $update_status = 0;     # Set to 1 to update status file
+my $ipset_running = 0;     # Set to 1 if IPSet process is running
+my %status;                # Status information
+my %checked;               # Time blacklists last changed
+my %modified;              # Time blacklists last modified
+my $red_iface;             # The name of the red interface
+
+############################################################################
+# Synchronise runs
+############################################################################
+
+# This script can be triggered either by cron or the WUI.  If another
+# instance is running, wait for it to finish or timeout.
+
+while (-r $lockfile and $count > 0)
+{
+  open LOCKFILE, '<', $lockfile or (abort "Can't open lockfile", last);
+  my $pid = <LOCKFILE>;
+  close LOCKFILE;
+
+  chomp $pid;
+
+  last unless (-e "/proc/$pid");
+
+  sleep 10;
+  $count--;
+}
+
+# Create pid file before starting main processing
+
+open LOCKFILE, '>', '/var/run/ipblacklist.pid' or abort "Can't open PID file: $!";
+print LOCKFILE "$$\n";
+close LOCKFILE;
+
+############################################################################
+# Set up for update
+############################################################################
+
+mkdir $settingsdir unless (-d $settingsdir);
+
+# Connect to the system log
+
+openlog( "ipblacklist", "nofatal", LOG_USER);
+log_message LOG_INFO, "Starting IP Blacklist processing";
+
+# Read settings
+
+General::readhash( $settings, \%settings )             if (-e $settings);
+General::readhash( $checked,  \%checked )              if (-e $checked);
+General::readhash( $modified, \%modified )             if (-e $modified);
+General::readhash( $proxy_settings, \%proxy_settings ) if (-e $proxy_settings);
+
+if (-r $sources)
+{
+  debug 1, "Reading sources file";
+
+  eval qx|/bin/cat $sources|;
+}
+
+if (-r $red_setting)
+{
+  open REDIF, '<', $red_setting or (abort "Can't open red interface file", exit);
+  $red_iface = <REDIF>;
+  chomp $red_iface;
+  close REDIF;
+}
+
+if (@ARGV)
+{
+  foreach my $cmd (@ARGV)
+  {
+    if ('update' =~ m/^$cmd/i)
+    {
+      # Called hourly when enabled and on setting changes.
+      # Update the blacklists.
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        do_update();
+      }
+    }
+    elsif ('start' =~ m/^$cmd/i)
+    {
+      # Called during system startup.
+      # Restore saved blacklists.
+      # Don't do an update since can take too long.
+
+      do_start() if ($settings{'ENABLE'} eq 'on');
+    }
+    elsif ('stop' =~ m/^$cmd/i)
+    {
+      # Called when shutting down.
+      # Delete IPSets and IPTables chains
+
+      do_stop();
+    }
+    elsif ('restore' =~ m/^$cmd/i)
+    {
+      # Called after restoring backup.
+      # Delete IPSets and IPTables chains, then re-create them with the new
+      # (restored) settings and make sure updates are enabled.
+
+      do_stop();
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        do_start();
+        enable_updates();
+      }
+      else
+      {
+        disable_updates();
+      }
+    }
+    elsif ('log-on' =~ m/^$cmd/i)
+    {
+      # Called from WUI.
+      # Create entries in IPTables chains to log dropped packets.
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        enable_logging();
+      }
+    }
+    elsif ('log-off' =~ m/^$cmd/i)
+    {
+      # Called from WUI
+      # Delete entries in IPTables chains to log dropped packets.
+
+      disable_logging();
+    }
+    elsif ('enable' =~ m/^$cmd/i)
+    {
+      # Called from WUI to enable blacklists
+      # Do an update and then enable automatic updates
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        do_update();
+        enable_updates();
+      }
+    }
+    elsif ('disable' =~ m/^$cmd/i)
+    {
+      # Called from WUI to disable blacklists.
+      # Disable updates, delete IPSets, IPTables chains and save files.
+
+      disable_updates();
+      do_delete();
+    }
+    else
+    {
+      print "Usage: $0 [update|start|stop|restart|log-on|log-off|enable|disable]\n";
+    }
+  }
+}
+elsif ($settings{'ENABLE'} eq 'on') # Default action if none specified
+{
+  do_update();
+}
+
+stop_ipset();
+
+if ($update_status)
+{
+  debug 1, "Writing updated status file";
+
+  General::writehash( $checked,  \%checked );
+  General::writehash( $modified, \%modified );
+}
+
+# Remove the pid file
+
+unlink $lockfile;
+
+log_message LOG_INFO, "Finished IP Blacklist processing";
+closelog();
+
+
+#------------------------------------------------------------------------------
+# sub do_stop
+#
+# Deletes all the IPTables chains and the IPSets
+#------------------------------------------------------------------------------
+
+sub do_stop()
+{
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Stopping IP Blacklists";
+
+  foreach my $list ( sort keys %sources )
+  {
+    delete_list( $list ) if (exists $chains{$list});
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_start
+#
+# Recreates the IPTables chains and the IPSets from the saved values
+#------------------------------------------------------------------------------
+
+sub do_start()
+{
+  log_message LOG_NOTICE, "Starting IP Blacklists";
+
+  foreach my $list ( sort keys %sources )
+  {
+    delete_list( $list ) if (exists $chains{$list});     # Make sure OK to start
+
+    if ((-e "$savedir/$list.conf") and ($red_iface))
+    {
+      log_message LOG_INFO, "Restoring blacklist $list";
+      system( "$ipset restore -f $savedir/$list.conf" ); # Can't use the ipset
+                                                         # function to do this
+
+      create_list( $list );
+    }
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_delete
+#
+# Deletes the IPTables chains, the IPSets and the saved values.
+#------------------------------------------------------------------------------
+
+sub do_delete()
+{
+  # Get the list of current ipsets
+
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Deleting IP Blacklists";
+
+  foreach my $source ( sort keys %sources )
+  {
+    if (exists $chains{$source})
+    {
+      delete_list( $source );
+    }
+
+    if (-e "$savedir/$source.conf")
+    {
+      unlink "$savedir/$source.conf";
+    }
+  }
+
+  %modified      = ();
+  $update_status = 1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_update
+#
+# Updates all the blacklists.
+# Creates or deletes the blacklist firewall rules as necessary and checks for
+# updates to the blacklists.  Each blacklist has its own minimum elapsed time
+# between updates, which is specified in the sources file, so the time of each
+# check is stored.
+#------------------------------------------------------------------------------
+
+sub do_update()
+{
+  return unless ($red_iface);
+
+  # Get the list of current ipsets
+
+  get_ipsets();
+
+  # Check sources
+
+  debug 1, "Checking blacklist sources";
+
+  LIST:
+  foreach my $list ( sort keys %sources )
+  {
+    my @new_blacklist = ();
+    my $name          = $sources{$list}{'name'};
+    my $last_checked  = $checked{$list} || 0;
+    my $failures      = $checked{"${list}_failures"} || 0;
+    my $enabled       = 0;
+
+    if (exists $settings{$list})
+    {
+      $enabled = $settings{$list} eq 'on';
+    }
+
+    if ($enabled and is_connected())
+    {
+      debug 1, "Checking blacklist source: $name";
+
+      # Calculate the per list rate
+
+      my $rate = get_rate_seconds( $sources{$list}{'rate'} );
+
+      # Has enough time passed since the last time we checked the list?
+      # Ignore the limit if the last download failed
+
+      if (($last_checked + $rate) < (time() + $margin) or
+          ($failures > 0 and $failures < $max_dl_fails))
+      {
+        my $type = 'hash:ip';
+
+        download_list( $list, \@new_blacklist, \$type );
+
+        next LIST unless (@new_blacklist);
+
+        if (not exists $chains{$list})
+        {
+          # Doesn't currently exist: Create it.
+
+          create_ipset( $list, $type, scalar @new_blacklist );
+          create_list( $list );
+        }
+
+        update_list( $list, \@new_blacklist, $type );
+      }
+    }
+    elsif (exists $chains{$list})
+    {
+      # Exists, but not enabled:  Delete it.
+
+      delete_list( $list );
+
+      # Delete the save file
+      # Don't delete the checked time from the status, in case the list is
+      # re-enabled quickly - don't want to exceed maximum allowed download
+      # rate.
+
+      unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+      delete $modified{$list}             if (exists $modified{$list});
+      delete $checked{"${list}_failures"} if (exists $checked{"${list}_failures"});
+      $update_status  = 1;
+    }
+  }
+
+  # Check for any lists that don't exist any more
+
+  foreach my $list (keys %modified)
+  {
+    next if (exists $sources{$list});
+
+    delete_list( $list );
+
+    # Delete the save file
+
+    unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+    # Delete from the status
+
+    delete $modified{$list} if (exists $modified{$list});
+    $update_status  = 1;
+  }
+
+  foreach my $list (keys %checked)
+  {
+    next if ($list =~ m/_failures/);
+    next if (exists $sources{$list});
+
+    delete $checked{$list};
+    delete $checked{"${list}_failures"};
+    delete $settings{$list} if (exists $settings{$list});
+    $update_status  = 1;
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub get_rate_seconds( text )
+#
+# Converts a check rate into seconds.  A sanity check is made on the coverted
+# value.
+#
+# Parameters:
+#   text  The value to convert in the form nnnu, where nnn is a number and u
+#         is either m (minutes), h (hours) or d (days).  Hours is assumed if
+#         not specified and everything after the first letter is ignored.
+#------------------------------------------------------------------------------
+
+sub get_rate_seconds( $ )
+{
+  my ($text) = @_;
+
+  my ($value, $unit) = (uc $text) =~ m/(\d+)([DHM]?)/;
+
+  if ($unit eq 'D')    # Days
+  {
+    $value *= 60 * 60 * 24;
+  }
+  elsif ($unit eq 'M') # Minutes
+  {
+    $value *= 60;
+  }
+  else                 # Everything else - assume hours
+  {
+    $value *= 60 * 60;
+  }
+
+  # Sanity check - limit to range 5 min .. 1 week
+
+  #        d    h    m    s
+  $value =           5 * 60 if ($value < 5 * 60);
+  $value = 7 * 24 * 60 * 60 if ($value > 7 * 24 * 60 * 60);
+
+  return $value;
+}
+
+
+#------------------------------------------------------------------------------
+# sub is_connected()
+#
+# Checks that the system is connected to the internet.
+#
+# This looks for a file created by IPFire when connected to the internet
+#------------------------------------------------------------------------------
+
+sub is_connected()
+{
+  return (-e $active);
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_list( list )
+#
+# Creates a new IPTables chain for a blacklist source.
+# The set must be created before calling this function.  Two rules are added to
+# the chain:
+# (optional) 1 Log the packet
+#            2 Drop the packet
+#
+# The log rule is only added when logging is enabled by the WUI.
+#
+# Rules are then added to the BLACKLISTIN and BLACKLISTOUT chains that check
+# the packet's IP address against the IPSet and then jump to the newly created
+# chain.
+#
+# Parameters:
+#   list  The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub create_list( $ )
+{
+  my ($list) = @_;
+
+  log_message LOG_INFO, "Create IPTables chains for blacklist $list";
+
+  # Create new chain in filter table
+
+  iptables( "-N ${list}_DROP" ) == 0 or
+    ( abort "Could not create IPTables chain ${list}_DROP", return );
+
+  # Add the logging and drop rules
+
+  if ($settings{'LOGGING'} eq 'on')
+  {
+    iptables( "-A ${list}_DROP -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" ) == 0 or
+      ( abort "Could not create IPTables chain $list LOG rule", return );
+  }
+
+  iptables( "-A ${list}_DROP -j DROP" ) == 0 or
+    ( abort "Could not create IPTables chain $list drop rule", return );
+
+  # Add the rules to check against the set
+
+  iptables( "-A BLACKLISTIN -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" );
+  iptables( "-A BLACKLISTOUT -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub delete_list( $list )
+#
+# Deletes an IPTables chain when a blacklist source is disabled. Also flushes
+# and destroys the IPSet.
+#
+# Parameters:
+#   list  The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub delete_list( $ )
+{
+  my ($list) = @_;
+
+  log_message LOG_INFO, "Delete IPTables chains for blacklist $list";
+
+  # Remove the blacklist chains from the main INPUT and OUTPUT chains
+
+  iptables( "-D BLACKLISTIN -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" );
+  iptables( "-D BLACKLISTOUT -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" );
+
+  # Flush and delete the chain
+
+  iptables( "-F ${list}_DROP" );
+  iptables( "-X ${list}_DROP" );
+
+  # Flush and delete the set
+
+  ipset( "flush $list" );
+  ipset( "destroy $list" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_list( list, ref_list, ref_type )
+#
+# Downloads the IP Addresses for a blacklist.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+#   list      The name of the blacklist
+#   ref_list  A reference to an array to store the downloaded blacklist
+#   ref_type  A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_list( $$$ )
+{
+  my ($list, $new_blacklist, $type) = @_;
+
+  $checked{$list} = time();  # Record that the list has been checked
+  $update_status  = 1;
+
+  # Check the parser for the blacklist
+
+  if (not exists $parsers{ $sources{$list}{'parser'} })
+  {
+    log_message LOG_ERR, "Can't find parser $sources{$list}{'parser'} for $list blacklist";
+    return;
+  }
+
+  # Add alternative download mechanisms here
+
+  download_check_header_time( $list, $new_blacklist, $type );
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_check_header_time( list, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist.  The If-Modified-Since header is
+# specified in the request so that only updated lists are downloaded (providing
+# that the server supports this functionality).
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+#   list      The name of the blacklist
+#   ref_list  A reference to an array to store the downloaded blacklist
+#   ref_type  A reference to store the type of the blacklist
+#
+# Returns:
+#   The list type: 'hash:ip' or 'hash:net'
+#------------------------------------------------------------------------------
+
+sub download_check_header_time( $$$ )
+{
+  my ($list, $new_blacklist, $type) = @_;
+  my $found_ip                      = 0;
+  my $found_net                     = 0;
+
+  # Get the parser for the blacklist
+
+  my $parser = $parsers{ $sources{$list}{'parser'} };
+
+  debug 1, "Checking for blacklist $list updates with LWP";
+
+  # Create a user agent for downloading the blacklist
+  # Limit the download size for safety
+
+  my $ua = LWP::UserAgent->new( max_size => $max_dl_bytes );
+
+  # Get the Proxy settings
+
+  if ($proxy_settings{'UPSTREAM_PROXY'})
+  {
+    if ($proxy_settings{'UPSTREAM_USER'})
+    {
+      $ua->proxy( [["http", "https"] => "http://$proxy_settings{'UPSTREAM_USER'}:$proxy_settings{'UPSTREAM_PASSWORD'}\@$proxy_settings{'UPSTREAM_PROXY'}/"] );
+    }
+    else
+    {
+      $ua->proxy( [["http", "https"]  => "http://$proxy_settings{'UPSTREAM_PROXY'}/"] );
+    }
+  }
+
+  # Get the last modified time
+
+  my $modified = gmtime( $modified{$list} || 0 );
+
+  # Download the blacklist
+
+  my $response = $ua->get( $sources{$list}{'url'}, 'If-Modified-Since' => $modified );
+
+  if (not $response->is_success)
+  {
+    if ($response->code == 304)
+    {
+      # Not an error - list has not been modified
+      debug 1, "Blacklist $list not modified";
+
+      return;
+    }
+
+    log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: ". $response->status_line;
+    $checked{"${list}_failures"}++;
+
+    return;
+  }
+
+  $modified{$list}             = $response->last_modified;
+  $checked{"${list}_failures"} = 0;
+
+  # Parse the downloaded list, checking if it's a list of addresses or nets
+
+  foreach my $line (split /[\r\n]+/, $response->content)
+  {
+    chomp $line;
+
+    my $address = &$parser( $line );
+
+    next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/);
+
+    if ($address =~ m|/32|)
+    {
+      $address =~ s|/32||;
+      $found_ip  = 1;
+    }
+    elsif ($address =~ m|/\d+|)
+    {
+      $found_net = 1;
+    }
+    else
+    {
+      $found_ip  = 1;
+    }
+
+    push @{ $new_blacklist }, $address;
+  }
+
+  if ($found_net and $found_ip)
+  {
+    # Convert mixed addresses and networks to all networks
+
+    foreach my $address (@{ $new_blacklist })
+    {
+      $address .= '/32' unless ($address =~ m|/\d+|);
+    }
+
+    $found_ip = 0;
+  }
+
+  $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub read_ipset( list, old, type, maxelem )
+#
+# Reads the existing contents and type of the set.
+#
+# Parameters:
+#   list    The name of the blacklist
+#   old     Reference to array to contain blacklist
+#   type    Reference to type
+#   maxelem Reference to maximum number of elements
+#------------------------------------------------------------------------------
+
+sub read_ipset( $$$$ )
+{
+  my ($list, $old, $type, $maxelem) = @_;
+  my $found_net                     = 0;
+  my $found_ip                      = 0;
+
+  debug 2, "Reading existing ipset for blacklist $list";
+
+  foreach my $line (qx/$ipset list $list/)
+  {
+    if ($line =~ m|Header:.*maxelem (\d+)|)
+    {
+      $$maxelem = $1;
+      next;
+    }
+
+    next unless ($line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|);
+
+    my $address = $1;
+
+    if ($address =~ m|/32|)
+    {
+      $found_ip  = 1;
+      $address   =~ s|/32$||;
+    }
+    elsif ($address =~ m|/\d+$|)
+    {
+      $found_net = 1;
+    }
+    else
+    {
+      $found_ip  = 1;
+    }
+
+    $$old{$address} = 1;
+  }
+
+  if ($found_ip and $found_net)
+  {
+    # Convert mixed addresses and networks to all networks
+
+    my @ads_list = keys %{ $old };
+
+    foreach my $address (@ads_list)
+    {
+      unless ($address =~ m|/\d+|)
+      {
+        delete $$old{$address};
+        $$old{"$address/32"} = 1;
+      }
+    }
+
+    $found_ip = 0;
+  }
+
+  $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub update_list( list, new, new_type )
+#
+# Updates the IP Addresses for a blacklist
+#
+# The new list is compared to the existing list and new entries added or old
+# entries deleted as necessary.  If the list type ('hash:ip' or 'hash:net') has
+# changed then the IPSet is deleted and re-created with the new type.
+#
+# Parameters:
+#   list      The name of the blacklist
+#   new       Reference to array of new blacklist entries
+#   new_type  The type of the updated list (hash:ip or hash:net)
+#------------------------------------------------------------------------------
+
+sub update_list( $$$ )
+{
+  my ($list, $new, $new_type) = @_;
+  my %old;
+  my $old_type;
+  my $changes = 0;
+  my $maxelem = 0;
+
+  debug 1, "Checking for $list blacklist update from $sources{$list}{'url'}";
+
+  if (exists $chains{$list} )
+  {
+    my $recreate_ipset = 0;
+
+    read_ipset( $list, \%old, \$old_type, \$maxelem );
+
+    # Check the IPSet type hasn't changed
+
+    if ($new_type ne $old_type)
+    {
+      log_message LOG_NOTICE, "Blacklist $list changed type from $old_type to $new_type";
+      $recreate_ipset = 1;
+    }
+
+    if ($max_size_fraction * $maxelem < scalar @{ $new } )
+    {
+      log_message LOG_NOTICE, "Blacklist $list changed size from $maxelem";
+      $recreate_ipset = 1;
+    }
+
+    if ($recreate_ipset)
+    {
+      # Change the IPSet type and/or size.  This requires removing references
+      # to it first. We could delete and then create the chain, but doing it
+      # like this keeps the statistics.
+
+      # Remove the IPSet from the IPTables chains
+
+      iptables( "-D 'BLACKLISTIN' -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ) == 0 or
+        log_message LOG_ERR, "Could not remove ${list} from BLACKLISTIN chain";
+
+      iptables( "-D 'BLACKLISTOUT' -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ) == 0 or
+        log_message LOG_ERR, "Could not remove ${list} from BLACKLISTOUT chain";
+
+      # Flush and delete the old set
+
+      ipset( "flush $list" );
+      ipset( "destroy $list" );
+
+      %old = ();  # Since we've deleted the old set it can't have any entries.
+
+      # Create the new ipset
+
+      create_ipset( $list, $new_type, scalar @{ $new } );
+
+      # Add the rules to check against the set
+
+      iptables( "-A 'BLACKLISTIN' -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ) == 0 or
+        log_message LOG_ERR, "Could not add IPSet $list to BLACKLISTIN chain";
+
+      iptables( "-A 'BLACKLISTOUT' -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ) == 0 or
+        log_message LOG_ERR, "Could not add IPSet $list to BLACKLISTOUT chain";
+    }
+  }
+
+  # Process the blacklist
+
+  foreach my $address ( @{ $new } )
+  {
+    # We've got an address.  Add to IPSet if it's new
+
+    if (exists $old{$address})
+    {
+      delete $old{$address};  # Not new - don't delete from chain later
+    }
+    else
+    {
+      ipset( "add $list $address -exist" ); # New - add it
+
+      $changes++;
+    }
+
+    debug 3, "Add net $address to blacklist $list";
+  }
+
+  # Delete old entries that aren't needed any more
+
+  debug 2, "Removing deleted rules from IPTables chain for blacklist $list";
+
+  foreach my $address ( keys %old )
+  {
+    ipset( "del $list $address" );
+
+    $changes++;
+
+    debug 3, "Delete old net $address from blacklist $list";
+  }
+
+  log_message LOG_INFO, "Updated $list blacklist with $changes changes";
+
+  # Save the blacklist for the next reboot
+
+  mkdir "$savedir" unless (-d "$savedir" );
+
+  ipset( "save $list -file $savedir/$list.conf" ) if ($changes > 0);
+
+  stop_ipset();
+}
+
+
+#------------------------------------------------------------------------------
+# sub get_ipsets( )
+#
+# Gets a list of the current IPSets
+#------------------------------------------------------------------------------
+
+sub get_ipsets( )
+{
+  debug 1, "Reading list of existing ipsets";
+
+  my @sets = qx($ipset -n list);
+
+  # Parse the tables
+
+  foreach my $line (@sets)
+  {
+    chomp $line;
+
+    next unless ($line);
+
+    $chains{$line} = 1;
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_ipset( list, type, size )
+#
+# Creates a new IPSet.  The current size of the set is determined by taking the
+# next power of two greater than the number of entries; the maximum size is set
+# to double this, subject to a minimum size.  This allows for future expansion.
+#
+# Parameters:
+#   list  The name of the blacklist
+#   type  The type of the blacklist (hash:ip or hash:net)
+#   size  The number of entries in the lsit
+#------------------------------------------------------------------------------
+
+sub create_ipset( $$$ )
+{
+  my ($list, $type, $size) = @_;
+
+  my $hashsize = 1;
+  $hashsize  <<= 1 while ($hashsize < $size);
+  my $maxsize  = $hashsize * 2;
+  $maxsize     = $min_ipset_entries if ($maxsize < $min_ipset_entries);
+
+  # Create the new ipset
+  ipset( "create $list $type hashsize $hashsize maxelem $maxsize" );
+  stop_ipset(); # Need to do this to action the IPSet commands
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_logging()
+#
+# Enable logging of packets dropped by IP Blacklist rules.
+# This adds a rule to log the packet to each lists' IPTables chain.
+#------------------------------------------------------------------------------
+
+sub enable_logging()
+{
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Enabling IP Blacklist logging";
+
+  foreach my $list ( sort keys %sources )
+  {
+    if (exists $chains{$list})
+    {
+      iptables( "-I ${list}_DROP 1 -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" );
+    }
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_logging()
+#
+# Disable logging of packets dropped by IP Blacklist rules.
+# This deletes a rule to log the packet from each lists' IPTables chain.
+#------------------------------------------------------------------------------
+
+sub disable_logging()
+{
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Disabling IP Blacklist logging";
+
+  foreach my $list ( sort keys %sources )
+  {
+    if (exists $chains{$list})
+    {
+      iptables( "-D ${list}_DROP -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" );
+    }
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_updates()
+#
+# Adds a command to the fcrontab to run the update hourly.
+# If there is a command already in the fcrontab to do this it will be
+# uncommented, otherwise a new line is added.
+#
+# The update is executed at an offset from the hour so that all the users don't
+# try to download the updates at exactly the same time - the blacklists are
+# provided free, so it's good manners to spread the load on the servers.  The
+# offset is initialised to a random number that avoids running on the hour
+# (when a lot of other things happen), and every fifteen minutes thereafter.
+#------------------------------------------------------------------------------
+
+sub enable_updates()
+{
+  my @lines = qx/$fcrontab -l/;
+  my $found = 0;
+
+  # Check for an existing fcrontab entry.
+
+  foreach my $line (@lines)
+  {
+    if ($line =~ m|/usr/local/bin/ipblacklist|)
+    {
+      return if ($line !~ m/^#/); # Already enabled - do nothing
+
+      # Already in fcrontab - uncomment the line
+
+      $line  =~ s/^#+//;
+      $found = 1;
+      log_message LOG_INFO, "Enable IP Address Blacklist update in crontab";
+      last;
+    }
+  }
+
+  if (not $found)
+  {
+    # Add a new entry to fcrontab
+
+    my $start = int( rand(13) ) + 1;
+
+    my $times = $start;
+
+    for (my $offset = $times+15 ; $offset < 60 ; $offset += 15)
+    {
+      $times .= ",$offset";
+    }
+
+    push @lines, "\n";
+    push @lines, "# IP Blacklist update\n";
+    push @lines, "\%nice(1) $times * * * * /usr/local/bin/ipblacklist\n";
+    log_message LOG_INFO, "Add IP Address Blacklist update to crontab";
+  }
+
+  open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+  print FCRONTAB @lines;
+  close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_updates()
+#
+# Comments out the entry in the fcrontab that runs the updates.
+#------------------------------------------------------------------------------
+
+sub disable_updates()
+{
+  my @lines = qx/$fcrontab -l/;
+  my $found = 0;
+
+  foreach my $line (@lines)
+  {
+    if ($line =~ m|/usr/local/bin/ipblacklist|)
+    {
+      return if ($line =~ m/^#/); # Already disabled - do nothing
+
+      # In fcrontab - comment the line
+
+      $line  =~ s/^#*/#/;
+      $found = 1;
+      log_message LOG_INFO, "Disable IP Address Blacklist updates";
+      last;
+    }
+  }
+
+  return unless ($found); # Don't update crontab unnecessarily
+
+  open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+  print FCRONTAB @lines;
+  close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_ip_or_net_list( line )
+#
+# Parses an input line, looking for lines starting with an IP Address or
+# Network specification.
+#
+# Parameters:
+#   line  The line to parse
+#
+# Returns:
+#   Either an IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_ip_or_net_list( $ )
+{
+  my ($line) = @_;
+
+  $line =~ m|^(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|;
+
+  return $1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_dshield( line )
+#
+# Parses an input line removing comments.
+#
+# The format is:
+# Start Addrs   End Addrs   Netmask   Nb Attacks   Network Name   Country   email
+# We're only interested in the start address and netmask.
+#
+# Parameters:
+#   line  The line to parse
+#
+# Returns:
+#   Either and IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_dshield( $ )
+{
+  my ($line) = @_;
+
+  return "" if ($line =~ m/^\s*#/);
+
+  $line =~ s/#.*$//;
+
+  #          |Start addrs                |   |End Addrs                |   |Mask
+  $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d+)?\s+(\d+)|;
+
+  return unless ($1);
+  return "$1/32" unless ($2);
+
+  return "$1/$2";
+}
+
+
+#------------------------------------------------------------------------------
+# sub iptables( cmd )
+#
+# Executes an IPTables command, waiting for the internal lock to ensure only
+# one change is made at a time.
+#
+# Parameters:
+#   cmd  The command to execute
+#
+# Returns:
+#   Status of command
+#------------------------------------------------------------------------------
+
+sub iptables( $ )
+{
+  my ($cmd) = @_;
+
+  return system( "$iptables $cmd" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub ipset( cmd )
+#
+# Executes an IPSet command.  The command is piped to a sub-process running
+# ipset, rather than exected separately.  This saves the overhead of starting a
+# new process for each command.  The sub-process is started if it's not already
+# running.
+#
+# Note that the pipe is buffered so commands are not necessarily executed
+# immediately.  Use ipset_stop() to force commands to be executed.  This should
+# be done before relying on anything that the ipset commands do, for example
+# before referencing the IPSet in an IPTables command.
+#
+# Parameters:
+#   cmd  The command to execute
+#------------------------------------------------------------------------------
+
+sub ipset( $ )
+{
+  my ($cmd) = @_;
+
+  if (not $ipset_running)
+  {
+    local $SIG{PIPE} = 'IGNORE';
+    open IPSET, "|-", $ipset, "restore" or die "Can't start ipset: $!";
+    $ipset_running = 1;
+  }
+
+  print IPSET "$cmd\n";
+}
+
+
+#------------------------------------------------------------------------------
+# sub stop_ipset( )
+#
+# Stops the ipset sub-process.
+# This causes any pending ipset commands to be executed.
+#------------------------------------------------------------------------------
+
+sub stop_ipset( )
+{
+  if ($ipset_running)
+  {
+    close IPSET or abort "ipset process died: $! $?";
+    $ipset_running = 0;
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub abort( message, parameters... )
+#
+# Used when aborting the current activity, printing out an error message.
+#
+# Parameters:
+#   message     Message to be printed
+#------------------------------------------------------------------------------
+
+sub abort( $ )
+{
+  my ($message) = @_;
+
+  log_message( LOG_ERR, $message );
+  carp $message;
+}
+
+
+#------------------------------------------------------------------------------
+# sub log_message( level, message )
+#
+# Logs a message.  If the script is run from a terminal messages are also
+# output on STDOUT.
+#
+# Parameters:
+#   level   Severity of message
+#   message Message to be logged
+#------------------------------------------------------------------------------
+
+sub log_message( $$ )
+{
+  my ($level, $message) = @_;
+
+  print "($level) $message\n" if (-t STDIN);
+  syslog( $level, $message );
+}
+
+
+#------------------------------------------------------------------------------
+# sub debug( level, message )
+#
+# Optionally logs a debug message.  If the script is run from a terminal, level
+# 1 debug messages are output regardless of the debug setting.
+#
+# Parameters:
+#   level   Debug level
+#   message Message to be logged
+#------------------------------------------------------------------------------
+
+sub debug( $$ )
+{
+  my ($level, $message) = @_;
+
+  if (($level <= $settings{'DEBUG'}) or
+      ($level == 1 and -t STDIN))
+  {
+    log_message LOG_DEBUG, $message;
+  }
+}
-- 
2.16.4