[PATCH 1/5] ipblacklist: Main script

[Tim FitzGeorge <ipfr@tfitzgeorge.me.uk>]

[-- Attachment #1: Type: text/plain, Size: 47086 bytes --]

Responsible for downloading blacklists and creating/modifying IPSets
Does all work involving creating, deleting and chaging IPTables and
IPSets.

Signed-off-by: Tim FitzGeorge <ipfr(a)tfitzgeorge.me.uk>
---
 src/scripts/ipblacklist | 1558 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1558 insertions(+)
 create mode 100755 src/scripts/ipblacklist

diff --git a/src/scripts/ipblacklist b/src/scripts/ipblacklist
new file mode 100755
index 000000000..b3f8048d9
--- /dev/null
+++ b/src/scripts/ipblacklist
@@ -0,0 +1,1558 @@
+#! /usr/bin/perl
+
+############################################################################
+#                                                                          #
+# IP Address blocklists for IPFire                                         #
+#                                                                          #
+# This is free software; you can redistribute it and/or modify             #
+# it under the terms of the GNU General Public License as published by     #
+# the Free Software Foundation; either version 3 of the License, or        #
+# (at your option) any later version.                                      #
+#                                                                          #
+# This is distributed in the hope that it will be useful,                  #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of           #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
+# GNU General Public License for more details.                             #
+#                                                                          #
+# You should have received a copy of the GNU General Public License        #
+# along with IPFire; if not, write to the Free Software                    #
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA #
+#                                                                          #
+# Copyright (C) 2018 - 2019 The IPFire team                                #
+#                                                                          #
+############################################################################
+#                                                                          #
+# This script use a file containing blacklist details in                   #
+# /var/ipfire/ipblacklist/sources as well as                               #
+# /var/ipfire/ipblacklistsettings containing an enable/disable flag for    #
+# each source.                                                             #
+#                                                                          #
+# Two IPTables chains are used, IPBLACKLISTREDIN and IPBLACKLISTREDOUT,    #
+# which are inserted into the main INPUT, OUTPUT and FORWARD chains.       #
+#                                                                          #
+# For each blacklist that is loaded, a chain is created to optionally log  #
+# and then to drop matching packets. An IPSet is created containing the    #
+# addresses or networks blocked by the blacklist, and then rules are added #
+# to the IPBLACKLISTREDIN and IPBLACKLISTREDOUT chains to jump to this     #
+# chain if appropriate packet list matches in the set.                     #
+#                                                                          #
+# When checking for updates, the modification time is read for each source #
+# and if necessary the list is downloaded.  The downloaded list is         #
+# compared to the existing IPSet contents and entries created or deleted   #
+# as necessary.                                                            #
+#                                                                          #
+############################################################################
+
+use strict;
+use warnings;
+
+use Carp;
+use Sys::Syslog qw(:standard :macros);
+use HTTP::Request;
+use LWP::UserAgent;
+
+require "/var/ipfire/general-functions.pl";
+
+############################################################################
+# Configuration variables
+#
+# These variables give the locations of various files used by this script
+############################################################################
+
+my $settingsdir    = "/var/ipfire/ipblacklist";
+my $savedir        = "/var/lib/ipblacklist";
+my $tmpdir         = "/var/tmp";
+
+my $settings       = "$settingsdir/settings";
+my $sources        = "$settingsdir/sources";
+my $checked        = "$settingsdir/checked";
+my $modified       = "$settingsdir/modified";
+my $iptables_list  = "/var/tmp/iptables.txt";
+my $getipstat      = "/usr/local/bin/getipstat";
+my $iptables       = "/sbin/iptables";
+my $ipset          = "/usr/sbin/ipset";
+my $fcrontab       = "/usr/bin/fcrontab";
+my $lockfile       = "/var/run/ipblacklist.pid";
+my $proxy_settings = "${General::swroot}/proxy/settings";
+my $red_setting    = "/var/ipfire/red/iface";
+my $detailed_log   = "$tmpdir/ipblacklist_log.txt";
+my $autoblacklist  = 'AUTOBLACKLIST';
+
+my %parsers  = ( 'text-with-hash-comments'      => \&parse_text_with_hash_comments,
+                 'text-with-semicolon-comments' => \&parse_text_with_semicolon_comments,
+                 'dshield'                      => \&parse_dshield );
+
+############################################################################
+# Default settings
+# Should be overwritten by reading settings files
+############################################################################
+
+my %sources  = ( );
+
+my %settings = ( 'DEBUG'           => 0,
+                 'LOGGING'         => 'on',
+                 'RATE'            => 24,
+                 'ENABLE'          => 'off' );
+
+my %proxy_settings = ( 'UPSTREAM_PROXY' => '' );         # No Proxy in use
+
+############################################################################
+# Function prototypes
+############################################################################
+
+sub abort( $ );
+sub autoblacklist_update();
+sub autoblacklist_clear();
+sub create_autoblacklist();
+sub create_list( $ );
+sub create_ipset( $$$ );
+sub debug( $$ );
+sub delete_autoblacklist();
+sub delete_list( $ );
+sub disable_logging();
+sub disable_updates();
+sub do_delete();
+sub do_start();
+sub do_stop();
+sub do_update();
+sub download_list( $$$ );
+sub download_check_header_time( $$$ );
+sub download_wget( $$$ );
+sub enable_logging();
+sub enable_updates();
+sub get_ipsets();
+sub iptables( $ );
+sub ipset( $ );
+sub stop_ipset();
+sub is_connected();
+sub log_message( $$ );
+sub parse_dshield( $ );
+sub parse_text_with_hash_comments( $ );
+sub parse_text_with_semicolon_comments( $ );
+sub read_ipset( $$$ );
+sub update_list( $$$ );
+
+############################################################################
+# Variables
+############################################################################
+
+my %chains;                # The Blacklist IPSets already loaded
+my %old_blacklist;         # Already blocked IP Addresses and/or networks
+                           # downloaded for current blacklist
+my $update_status = 0;     # Set to 1 to update status file
+my $ipset_running = 0;     # Set to 1 if IPSet process is running
+my %status;                # Status information
+my %checked;               # Time blacklists last changed
+my %modified;              # Time blacklists last modified
+my $red_iface;             # Name of red interface
+my $hours          = 3600; # One hour in seconds
+my $margin         = 600;  # Allowance for run time etc
+my $count          = 30;   # Maximum time to wait for another instance (300s)
+my @wget_status    = ( 'Success', 'Error', 'Parse Error', 'File I/O Error',
+                       'Network Error', 'SSL Verification Error',
+                       'Authentication Error', 'Protocol Error', 'Server Error' );
+
+
+############################################################################
+# Synchronise runs
+############################################################################
+
+# This script can be triggered either by cron or the WUI.  If another
+# instance is running, wait for it to finish.
+
+while (-r $lockfile and $count > 0)
+{
+  open LOCKFILE, '<', $lockfile or die "Can't open lockfile";
+  my $pid = <LOCKFILE>;
+  close LOCKFILE;
+
+  chomp $pid;
+
+  last unless (-e "/proc/$pid");
+
+  sleep 10;
+  $count--;
+}
+
+# Create pid file before starting main processing
+
+open LOCKFILE, '>', '/var/run/ipblacklist.pid' or die "Can't open PID file: $!";
+print LOCKFILE "$$\n";
+close LOCKFILE;
+
+############################################################################
+# Set up for update
+############################################################################
+
+mkdir $settingsdir unless (-d $settingsdir);
+
+# Connect to the system log
+
+openlog( "ipblacklist", "nofatal", LOG_USER);
+log_message LOG_INFO, "Starting IP Blacklist processing";
+
+# Read settings
+
+General::readhash( $settings, \%settings )             if (-e $settings);
+General::readhash( $checked,  \%checked )              if (-e $checked);
+General::readhash( $modified, \%modified )             if (-e $modified);
+General::readhash( $proxy_settings, \%proxy_settings ) if (-e $proxy_settings);
+
+if (-r $sources)
+{
+  debug 1, "Reading sources file";
+
+  eval qx|/bin/cat $sources|;
+}
+
+# Find out the red interface name
+
+if (-r $red_setting)
+{
+  open IN, '<', $red_setting or die "Can't open red interface name file: $!";
+
+  $red_iface = <IN>;
+  chomp $red_iface;
+
+  close IN;
+}
+
+if (@ARGV)
+{
+  foreach my $cmd (@ARGV)
+  {
+    if ('update' =~ m/^$cmd/i)
+    {
+      # Called hourly when enabled and on setting changes.
+      # Update the blacklists.
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        do_update();
+      }
+    }
+    elsif ('start' =~ m/^$cmd/i)
+    {
+      # Called during system startup.
+      # Restore saved blacklists.
+      # Don't do an update since that takes too long.
+
+      do_start() if ($settings{'ENABLE'} eq 'on');
+    }
+    elsif ('stop' =~ m/^$cmd/i)
+    {
+      # Called when shutting down.
+      # Delete IPSets and IPTables chains
+
+      do_stop();
+    }
+    elsif ('restore' =~ m/^$cmd/i)
+    {
+      # Called after restoring backup.
+      # Delete IPSets and IPTables chains, then re-create them with the new
+      # (restored) settings and make sure updates are enabled.
+
+      do_stop();
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        do_start();
+        enable_updates();
+      }
+      else
+      {
+        disable_updates();
+      }
+    }
+    elsif ('log-on' =~ m/^$cmd/i)
+    {
+      # Called from WUI.
+      # Create entries in IPTables chains to log dropped packets.
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        enable_logging();
+      }
+    }
+    elsif ('log-off' =~ m/^$cmd/i)
+    {
+      # Called from WUI
+      # Delete entries in IPTables chains to log dropped packets.
+
+      disable_logging();
+    }
+    elsif ('enable' =~ m/^$cmd/i)
+    {
+      # Called from WUI to enable blacklists
+      # Do an update and then enable automatic updates
+
+      if ($settings{'ENABLE'} eq 'on')
+      {
+        do_update();
+        enable_updates();
+      }
+    }
+    elsif ('disable' =~ m/^$cmd/i)
+    {
+      # Called from WUI to disable blacklists.
+      # Disable updates, delete IPSets, IPTables chains and save files.
+
+      disable_updates();
+      do_delete();
+    }
+    elsif ('autoblacklist-update' =~ m/^$cmd/i)
+    {
+      # Updates AUTOBLACKLIST options
+
+      autoblacklist_update();
+    }
+    elsif ('autoblacklist-clear' =~ m/^$cmd/i)
+    {
+      # Clears AUTOBLACKLIST contents
+
+      autoblacklist_clear();
+    }
+    else
+    {
+      print "Usage: $0 [update|start|stop|restart|log-on|log-off|enable|disable|autoblacklist-update|autoblacklist-clear]\n";
+    }
+  }
+}
+elsif ($settings{'ENABLE'} eq 'on')
+{
+  do_update();
+}
+
+stop_ipset();
+
+if ($update_status)
+{
+  debug 1, "Writing updated status file";
+
+  General::writehash( $checked,  \%checked );
+  General::writehash( $modified, \%modified );
+}
+
+# Remove the pid file
+
+unlink $lockfile;
+
+log_message LOG_INFO, "Finished IP Blacklist processing";
+closelog();
+
+
+#------------------------------------------------------------------------------
+# sub do_stop
+#
+# Deletes all the IPTables chains and the IPSets
+#------------------------------------------------------------------------------
+
+sub do_stop()
+{
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Stopping IP Blacklists";
+
+  foreach my $list ( $autoblacklist, sort keys %sources )
+  {
+    if (exists $chains{$list})
+    {
+      delete_list( $list );
+    }
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_start
+#
+# Recreates the IPTables chains and the IPSets from the saved values
+#------------------------------------------------------------------------------
+
+sub do_start()
+{
+  log_message LOG_NOTICE, "Starting IP Blacklists";
+
+  foreach my $list ( sort keys %sources )
+  {
+    if (-e "$savedir/$list.conf")
+    {
+      log_message LOG_INFO, "Restoring blacklist $list";
+      system( "$ipset restore -f $savedir/$list.conf" );
+
+      create_list( $list );
+    }
+  }
+
+  if ($settings{$autoblacklist} eq 'on')
+  {
+    create_autoblacklist();
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_delete
+#
+# Deletes the IPTables chains, the IPSets and the saved values.
+#------------------------------------------------------------------------------
+
+sub do_delete()
+{
+  # Get the list of current ipsets
+
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Deleting IP Blacklists";
+
+  foreach my $source ( sort keys %sources )
+  {
+    if (exists $chains{$source})
+    {
+      delete_list( $source );
+    }
+
+    if (-e "$savedir/$source.conf")
+    {
+      unlink "$savedir/$source.conf";
+    }
+  }
+
+  if ($settings{$autoblacklist} eq 'on')
+  {
+    delete_autoblacklist();
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_update
+#
+# Updates all the blacklists.
+# Creates or deletes the blacklist firewall rules as necessary and checks for
+# updates to the blacklists.
+#------------------------------------------------------------------------------
+
+sub do_update()
+{
+  return unless (is_connected());
+
+  my $type = 'hash:ip';
+
+  # Get the list of current ipsets
+
+  get_ipsets();
+
+  # Check sources
+
+  debug 1, "Checking blacklist sources";
+
+  foreach my $list ( sort keys %sources )
+  {
+    my @new_blacklist = ();
+    my $name          = $sources{$list}{'name'};
+    my $rate          = $sources{$list}{'rate'};
+    my $last_checked  = $checked{$list} || 0;
+    my $enabled       = 0;
+
+    if (exists $modified{$list})
+    {
+      # Limit the check rate to the minimum defined in the WUI, unless we're
+      # creating the list
+
+      $rate = $settings{'RATE'} if ($settings{'RATE'} > $rate);
+    }
+
+    if (exists $settings{$list})
+    {
+      $enabled = $settings{$list} eq 'on';
+    }
+
+    debug 1, "Checking blacklist source: $name";
+
+    if ($enabled)
+    {
+      # Has enough time passed since the last time we checked the list?
+
+      if (($last_checked + $rate * $hours) < (time() + $margin))
+      {
+        download_list( $list, \@new_blacklist, \$type );
+
+        next unless (@new_blacklist);
+
+        if (not exists $chains{$list})
+        {
+          # Doesn't currently exist: Create it.
+
+          create_ipset( $list, $type, scalar @new_blacklist );
+          create_list( $list );
+        }
+
+        update_list( $list, \@new_blacklist, $type );
+      }
+    }
+    elsif (exists $chains{$list})
+    {
+      # Exists, but not enabled:  Delete it.
+
+      delete_list( $list );
+
+      # Delete the save file
+      # Don't delete the checked time from the status, in case the list is
+      # re-enabled quickly - don't want to exceed maximum allowed download
+      # rate.
+
+      unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+      delete $modified{$list} if (exists $modified{$list});
+      $update_status  = 1;
+    }
+  }
+
+  # Check for any deleted lists
+
+  foreach my $list (keys %sources)
+  {
+    if (not exists $sources{$list})
+    {
+      delete_list( $list );
+
+      # Delete the save file
+
+      unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+      # Delete from the status
+
+      delete $modified{$list} if (exists $modified{$list});
+      delete $checked{$list}  if (exists $checked{$list});
+      $update_status  = 1;
+    }
+  }
+
+  if ($settings{$autoblacklist} eq 'on')
+  {
+    create_autoblacklist() if (not exists $chains{$autoblacklist});
+  }
+  else
+  {
+    delete_autoblacklist() if (exists $chains{$autoblacklist});
+  }
+
+  log_message LOG_INFO, "Completed IP Blacklist update";
+}
+
+
+#------------------------------------------------------------------------------
+# sub autoblacklist_update()
+#
+# Updates the settings for the AUTOBLACKLIST
+#------------------------------------------------------------------------------
+
+sub autoblacklist_update()
+{
+  # Get the list of current ipsets
+
+  get_ipsets();
+
+  # Delete the existing AUTOBLACKLIST, if it currently exists.
+
+  delete_autoblacklist() if (exists $chains{$autoblacklist});
+
+  # Re-create the AUTOBLACKLIST with the correct parameters.
+
+  create_autoblacklist() if ($settings{$autoblacklist} eq 'on');
+}
+
+
+#------------------------------------------------------------------------------
+# sub autoblacklist_clear()
+#
+# Clears the contents of the AUTOBLACKLIST
+#------------------------------------------------------------------------------
+
+sub autoblacklist_clear()
+{
+  log_message LOG_INFO, "Flush Automatic blacklist";
+  ipset( "flush $autoblacklist" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub is_connected()
+#
+# Checks that the system is connected to the internet.
+#
+# This looks for a file created by IPFire when connected to the internet
+#------------------------------------------------------------------------------
+
+sub is_connected()
+{
+  return (-e "${General::swroot}/red/active");
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_list( list )
+#
+# Creates a new IPTables chain for a blacklist source.
+# The set must be created before calling this function.
+#
+# Parameters:
+#   list  The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub create_list( $ )
+{
+  my ($list) = @_;
+
+  log_message LOG_INFO, "Create IPTables chains for blacklist $list";
+
+  # Create new chain in filter table
+
+  iptables( " -N ${list}_BLOCK" ) == 0 or
+    ( abort "Could not create IPTables chain ${list}_BLOCK", return );
+
+  # Add the logging and drop rules
+
+  if ($settings{'LOGGING'} eq 'on')
+  {
+    iptables( "-A ${list}_BLOCK -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" ) == 0 or
+      ( abort "Could not create IPTables chain $list LOG rule", return );
+  }
+
+  iptables( "-A ${list}_BLOCK -j DROP" ) == 0 or
+    ( abort "Could not create IPTables chain $list drop rule", return );
+
+  # Add the rules to check against the set
+
+  iptables( "-A IPBLACKLISTREDIN -p ALL -m set --match-set $list src -j ${list}_BLOCK" );
+  iptables( "-A IPBLACKLISTREDOUT -p ALL -m set --match-set $list dst -j ${list}_BLOCK" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_autoblacklist()
+#
+# Creates a new IPTables chain for the AUTOBLACKLIST.  This also creates the
+# IPSet with the correct timeout.
+#------------------------------------------------------------------------------
+
+sub create_autoblacklist()
+{
+  return unless ($red_iface);  # Can't add rule to policy unless this is set
+
+  # Create the set for the AUTOBLACKLIST
+
+  ipset( "create $autoblacklist hash:ip timeout $settings{BLOCK_PERIOD}" );
+
+  # Create new chain in filter table
+
+  create_list( $autoblacklist );
+
+  # For the AUTOBLACKLIST there are extra rules to reset the timeout on the
+  # blockled addresses
+
+  iptables( "-I ${autoblacklist}_BLOCK -m set --match-set $autoblacklist src -j SET --add-set $autoblacklist src --exist" );
+  iptables( "-I ${autoblacklist}_BLOCK -m set --match-set $autoblacklist dst -j SET --add-set $autoblacklist dst --exist" );
+
+  # For the AUTOBLACKLIST there is an extra rule to add an entry to the list
+  # of blocked addresses.  This is added to the input policy chain.
+
+  iptables( "-I POLICYIN 1 -i $red_iface -m hashlimit --hashlimit-mode srcip --hashlimit-above $settings{BLOCK_THRESHOLD}/hour --hashlimit-name $autoblacklist -j SET --add-set $autoblacklist src" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub delete_list( $list )
+#
+# Deletes an IPTables chain when a blacklist source is disabled. Also flushes
+# and destroys the IPSet.
+#
+# Parameters:
+#   list  The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub delete_list( $ )
+{
+  my ($list) = @_;
+
+  log_message LOG_INFO, "Delete IPTables chains for blacklist $list";
+
+  # Remove the blacklist chains from the main INPUT and OUTPUT chains
+
+  iptables( "-D IPBLACKLISTREDIN -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or
+    log_message LOG_ERR, "Could not remove IPSet $list from IPBLACKLISTREDIN chain";
+
+  iptables( "-D IPBLACKLISTREDOUT -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or
+    log_message LOG_ERR, "Could not remove IPSet $list from IPBLACKLISTREDOUT chain";
+
+  # Flush and delete the chain
+
+  iptables( "-F ${list}_BLOCK" ) == 0 or
+    log_message LOG_ERR, "Could not flush IPTables chain ${list}_BLOCK";
+
+  iptables( "-X ${list}_BLOCK" ) == 0 or
+    log_message LOG_ERR, "Could not delete IPTables chain ${list}_BLOCK";
+
+  # Flush and delete the set
+
+  ipset( "flush $list" );
+
+  ipset( "destroy $list" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub delete_autoblacklist()
+#
+# Deletes the autoblacklist IPTables chain when it is disabled. Also flushes
+# and destroys the IPSet.
+#------------------------------------------------------------------------------
+
+sub delete_autoblacklist()
+{
+  # For the AUTOBLACKLIST there is an extra rule to remove
+
+  unless ($red_iface)
+  {
+    iptables( "-D POLICYIN -i $red_iface -m hashlimit --hashlimit-mode srcip --hashlimit-above $settings{BLOCK_THRESHOLD}/hour --hashlimit-name $autoblacklist -j SET --add-set $autoblacklist src" );
+  }
+
+  # Now do a normal delete
+
+  delete_list( $autoblacklist );
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_list( chain, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist.  Depending on the blacklist one of
+# two methods are used:
+#
+# - For some lists the header is downloaded and the modification date checked.
+#   If newer than the existing list, the update is downloaded.
+# - For other lists this is not supported,so the whole file has to be
+#   downloaded regardless.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+#   list      The name of the blacklist
+#   ref_list  A reference to an array to store the downloaded blacklist
+#   ref_type  A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_list( $$$ )
+{
+  my ($list, $new_blacklist, $type) = @_;
+
+  $checked{$list}            = time();
+  $update_status             = 1;
+
+  # Check the parser for the blacklist
+
+  if (not exists $parsers{ $sources{$list}{'parser'} })
+  {
+    log_message LOG_ERR, "Can't find parser $sources{$list}{'parser'} for $list blacklist";
+    return;
+  }
+
+  if ($sources{$list}{'method'} eq 'check-header-time')
+  {
+    download_check_header_time( $list, $new_blacklist, $type );
+  }
+  else
+  {
+    download_wget( $list, $new_blacklist, $type );
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_check_header_time( chain, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist.  The header is downloaded and the
+# modification date checked. If newer than the existing list, the update is
+# downloaded.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+#   list      The name of the blacklist
+#   ref_list  A reference to an array to store the downloaded blacklist
+#   ref_type  A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_check_header_time( $$$ )
+{
+  my ($list, $new_blacklist, $type) = @_;
+  my $found_ip                      = 0;
+  my $found_net                     = 0;
+
+  # Get the parser for the blacklist
+
+  my $parser = $parsers{ $sources{$list}{'parser'} };
+
+  log_message LOG_INFO, "Checking modification time for blacklist $list update with LWP";
+
+  # Create a user agent for downloading the blacklist
+  # Limit the download size for safety (10 MiB)
+
+  my $ua = LWP::UserAgent->new( max_size => 10485760 );
+
+  # Get the Proxy settings
+
+  if ($proxy_settings{'UPSTREAM_PROXY'})
+  {
+    if ($proxy_settings{'UPSTREAM_USER'})
+    {
+      $ua->proxy("http"  => "http://$proxy_settings{'UPSTREAM_USER'}:$proxy_settings{'UPSTREAM_PASSWORD'}\@$proxy_settings{'UPSTREAM_PROXY'}/");
+      $ua->proxy("https" => "http://$proxy_settings{'UPSTREAM_USER'}:$proxy_settings{'UPSTREAM_PASSWORD'}\@$proxy_settings{'UPSTREAM_PROXY'}/");
+    }
+    else
+    {
+      $ua->proxy("http"  => "http://$proxy_settings{'UPSTREAM_PROXY'}/");
+      $ua->proxy("https" => "http://$proxy_settings{'UPSTREAM_PROXY'}/");
+    }
+  }
+
+  # Get the blacklist modification time from the internet
+
+  my $request  = HTTP::Request->new( HEAD => $sources{$list}{'url'} );
+
+  my $response = $ua->request( $request );
+
+  if (not $response->is_success)
+  {
+    log_message LOG_WARNING, "Failed to download $list header $sources{$list}{'url'}: ". $response->status_line;
+
+    return;
+  }
+
+  # Has the blacklist been modified since we last read it?
+
+  if (exists $modified{$list} and $modified{$list} >= $response->last_modified)
+  {
+    # We've already got this version of the blacklist
+
+    debug 1, "Blacklist $list not modified";
+    return;
+  }
+
+  debug 1, "Blacklist $list Modification times: old " . $modified{$list} . ", new " . $response->last_modified if (exists $modified{$list});
+  log_message LOG_INFO, "Downloading blacklist $list with LWP";
+
+  # Download the blacklist
+
+  $request = HTTP::Request->new( GET => $sources{$list}{'url'} );
+  $response = $ua->request($request);
+
+  if (not $response->is_success)
+  {
+    log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: ". $response->status_line;
+
+    return;
+  }
+
+  $modified{$list} = $response->last_modified;
+
+  foreach my $line (split /[\r\n]+/, $response->content)
+  {
+    chomp $line;
+
+    my $address = &$parser( $line );
+
+    next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/);
+
+    if ($address =~ m|/\d+|)
+    {
+      $found_net = 1;
+    }
+    else
+    {
+      $found_ip  = 1;
+    }
+
+    push @{ $new_blacklist }, $address;
+  }
+
+  if ($found_net and $found_ip)
+  {
+    # Convert mixed address and network set to all network
+
+    foreach my $address (@{ $new_blacklist })
+    {
+      $address .= '/32' unless ($address =~ m|/\d+|);
+    }
+
+    $found_ip = 0;
+  }
+
+  $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_wget( chain, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist.  The whole file is download with
+# wget and then the modification time compared with the stored modification
+# time.  If the update is newer then the downloaded list is parsed.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+#   list      The name of the blacklist
+#   ref_list  A reference to an array to store the downloaded blacklist
+#   ref_type  A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_wget( $$$ )
+{
+  my ($list, $new_blacklist, $type) = @_;
+  my $wget_proxy                    = '';
+  my $found_ip                      = 0;
+  my $found_net                     = 0;
+
+  my $parser = $parsers{ $sources{$list}{'parser'} };
+
+  log_message LOG_INFO, "Downloading blacklist $list update with wget";
+
+  # Get the Proxy settings
+
+  if ($proxy_settings{'UPSTREAM_PROXY'})
+  {
+    if ($proxy_settings{'UPSTREAM_USER'})
+    {
+      $wget_proxy = "--proxy=on --proxy-user=$proxy_settings{'UPSTREAM_USER'} --proxy-passwd=$proxy_settings{'UPSTREAM_PASSWORD'} -e http_proxy=http://$proxy_settings{'UPSTREAM_PROXY'}/";
+    }
+    else
+    {
+      $wget_proxy = "--proxy=on -e http_proxy=http://$proxy_settings{'UPSTREAM_PROXY'}/";
+    }
+  }
+
+  my $retv = system( "wget $wget_proxy --no-show-progress -o $detailed_log -O $tmpdir/ipblacklist_$list $sources{$list}{'url'}" );
+
+  if ($retv != 0)
+  {
+    my $error = $wget_status[ $retv/256 ];
+    log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: $error";
+    return;
+  }
+
+  my @file_info = stat( "$tmpdir/ipblacklist_$list" );
+
+  if (exists $modified{$list} and $modified{$list} >= $file_info[9])
+  {
+    # We've already got this version of the blocklist
+
+    debug 1, "Blacklist $list not modified";
+    unlink "$tmpdir/ipblacklist_$list";
+    return;
+  }
+
+  open LIST, '<', "$tmpdir/ipblacklist_$list" or (abort "Can't open downloaded blacklist for $list: $!", return);
+
+  $modified{$list} = $file_info[9];
+
+  foreach my $line (<LIST>)
+  {
+    chomp $line;
+
+    my $address = &$parser( $line );
+
+    next unless ($address);
+    next unless ($address =~ m|\d+\.\d+\.\d+\.\d+|);
+
+    if ($address =~ m|/\d+|)
+    {
+      $found_net = 1;
+    }
+    else
+    {
+      $found_ip  = 1;
+    }
+
+    push @{ $new_blacklist }, $address;
+  }
+
+  close LIST;
+
+  unlink "$tmpdir/ipblacklist_$list";
+
+  if ($found_net and $found_ip)
+  {
+    # Convert mixed address and network set to all network
+
+    foreach my $address (@{ $new_blacklist })
+    {
+      $address .= '/32' unless ($address =~ m|/\d+|);
+    }
+
+    $found_ip = 0;
+  }
+
+  $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub read_ipset( list, old, type )
+#
+# Reads the existing contents of the set
+#
+# Parameters:
+#   chain  The name of the blacklist
+#   old    Reference to array to contain blacklist
+#   type   Reference to type
+#------------------------------------------------------------------------------
+
+sub read_ipset( $$$ )
+{
+  my ($list, $old, $type) = @_;
+  my $found_net            = 0;
+  my $found_ip             = 0;
+
+  debug 2, "Reading existing ipset for blacklist $list";
+
+  foreach my $line (qx/$ipset list $list/)
+  {
+    next unless ($line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|);
+
+    my $address = $1;
+
+    if (($address =~ m|/\d+$|) and ($address !~ m|/32$|))
+    {
+      $found_net = 1;
+    }
+    else
+    {
+      $found_ip  = 1;
+      $address   =~ s|/32$||;
+    }
+
+    $$old{$address} = 1;
+  }
+
+  if ($found_ip and $found_net)
+  {
+    # Convert mixed address and network set to all network
+
+    my @ads_list = keys %{ $old };
+
+    foreach my $address (@ads_list)
+    {
+      unless ($address =~ m|/\d+|)
+      {
+        delete $$old{$address};
+        $$old{"$address/32"} = 1;
+      }
+    }
+
+    $found_ip = 0;
+  }
+
+  $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub update_list( chain, new, new_type )
+#
+# Updates the IP Addresses for a blacklist
+#
+# The new list is compared to the existing list and new entries added or old
+# entries deleted as necessary.
+#
+# Parameters:
+#   list      The name of the blacklist
+#   new       Reference to array of new blacklist entries
+#   new_type  The type of the updated list (hash:ip or hash:net)
+#------------------------------------------------------------------------------
+
+sub update_list( $$$ )
+{
+  my ($list, $new, $new_type) = @_;
+  my %old;
+  my $old_type;
+  my $changes = 0;
+
+  debug 2, "Checking for $list blacklist update from $sources{$list}{'url'}";
+
+  log_message LOG_INFO, "Updating $list blacklist";
+
+  read_ipset( $list, \%old, \$old_type );
+
+  # Check the IPSet type hasn't changed
+
+  if ($new_type ne $old_type)
+  {
+    # Change the IPSet type.  This requires removing references to it first.
+    # We could delete and then create the chain, but doing it like this keeps
+    # the statistics.
+
+    log_message LOG_NOTICE, "Blacklist $list changed type from $old_type to $new_type";
+
+    # Remove the IPSet from the IPTables chains
+
+    iptables( "-D 'IPBLACKLISTREDIN' -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or
+      log_message LOG_ERR, "Could not remove ${list} from IPBLACKLISTREDIN chain";
+
+    iptables( "-D 'IPBLACKLISTREDOUT' -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or
+      log_message LOG_ERR, "Could not remove ${list} from IPBLACKLISTREDOUT chain";
+
+    # Flush and delete the old set
+
+    ipset( "flush $list" );
+    ipset( "destroy $list" );
+
+    %old = ();
+
+    # Create the new ipset
+
+    create_ipset( $list, $new_type, scalar @{ $new } );
+
+    # Add the rules to check against the set
+
+    iptables( "-A 'IPBLACKLISTREDIN' -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or
+      log_message LOG_ERR, "Could not add IPSet $list to IPBLACKLISTREDIN chain";
+
+    iptables( "-A 'IPBLACKLISTREDOUT' -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or
+      log_message LOG_ERR, "Could not add IPSet $list to IPBLACKLISTREDOUT chain";
+  }
+
+  # Process the blacklist
+
+  foreach my $address ( @{ $new } )
+  {
+    # We've got an address.  Add to IPSet if it's new
+
+    if (exists $old{$address})
+    {
+      delete $old{$address};  # Not new
+    }
+    else
+    {
+      ipset( "add $list $address -exist" );
+
+      $changes++;
+    }
+
+    debug 3, "Add net $address to blacklist $list";
+  }
+
+  # Delete old entries that aren't needed any more
+
+  debug 2, "Removing deleted rules from IPTables chain for blacklist $list";
+
+  foreach my $address ( keys %old )
+  {
+    ipset( "del $list $address" );
+
+    $changes++;
+
+    debug 3, "Delete old net $address from blacklist $list";
+  }
+
+
+  log_message LOG_INFO, "Finished updating $list blacklist with $changes changes";
+
+  # Save the blacklist for the next reboot
+
+  mkdir "$savedir" unless (-d "$savedir" );
+
+  ipset( "save $list -file $savedir/$list.conf" );
+
+  stop_ipset();
+}
+
+
+#------------------------------------------------------------------------------
+# sub get_ipsets( )
+#
+# Gets a list of the current IPSets
+#------------------------------------------------------------------------------
+
+sub get_ipsets( )
+{
+  debug 1, "Reading list of existing ipsets";
+
+  my @sets = qx($ipset -n list);
+
+  # Parse the tables
+
+  foreach my $line (@sets)
+  {
+    chomp $line;
+
+    next unless ($line);
+
+    $chains{$line} = 1;
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_ipset( name, type, size )
+#
+# Creates a new IPSet.  The current and maximum size of the set are determined
+# by taking the next power of two greater than the numer of entries, subject to
+# a minimum size.  This allows for future expansion.
+#
+# Parameters:
+#   name  The name of the blacklist
+#   type  The type of the blacklist (hash:ip or hash:net)
+#   size  The number of entries in the lsit
+#------------------------------------------------------------------------------
+
+sub create_ipset( $$$ )
+{
+  my ($name, $type, $size) = @_;
+
+  my $hashsize = 1;
+  $hashsize  <<= 1 while ($hashsize < $size);
+  my $maxsize  = ($hashsize < 16384) ? 32768 : $hashsize * 2;
+
+  # Create the new ipset
+  ipset( "create $name $type hashsize $hashsize maxelem $maxsize" );
+  stop_ipset(); # Need to do this to action the IPSet commands
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_logging()
+#
+# Enable logging of packets dropped by IP Blacklist rules.
+#------------------------------------------------------------------------------
+
+sub enable_logging()
+{
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Enabling IP Blacklist logging";
+
+  foreach my $list ( sort keys %sources )
+  {
+    if (exists $chains{$list})
+    {
+      iptables( "-I ${list}_BLOCK 1 -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" );
+    }
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_logging()
+#
+# Disable logging of packets dropped by IP Blacklist rules.
+#------------------------------------------------------------------------------
+
+sub disable_logging()
+{
+  get_ipsets();
+
+  log_message LOG_NOTICE, "Disabling IP Blacklist logging";
+
+  foreach my $list ( sort keys %sources )
+  {
+    if (exists $chains{$list})
+    {
+      iptables( "-D ${list}_BLOCK -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" );
+    }
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_updates()
+#
+# Adds a command to the fcrontab to run the update hourly.
+# The update is executed at an offset from the hour so that all the users don't
+# try to download the updates at exactly the same time - the blacklists are
+# provided free, so it's good manners to spread the load on the servers.
+#------------------------------------------------------------------------------
+
+sub enable_updates()
+{
+  my @lines = qx/$fcrontab -l/;
+  my $found = 0;
+
+  # Check for an existing fcrontab entry.
+
+  foreach my $line (@lines)
+  {
+    if ($line =~ m|/usr/local/bin/ipblacklist|)
+    {
+      return if ($line !~ m/^#/); # Already enabled
+
+      # Found - uncomment the line
+
+      $line  =~ s/^#+//;
+      $found = 1;
+      log_message LOG_INFO, "Enable IP Address Blacklist update in crontab";
+      last;
+    }
+  }
+
+  if (not $found)
+  {
+    # Add a new entry
+
+    my $start = int( rand(50) ) + 5;
+
+    push @lines, "\n";
+    push @lines, "# IP Blacklist update\n";
+    push @lines, "\%hourly,nice(1),random,serial $start /usr/local/bin/ipblacklist\n";
+    log_message LOG_INFO, "Add IP Address Blacklist update to crontab";
+  }
+
+  open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+  print FCRONTAB @lines;
+  close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_updates()
+#
+# Comments out the entry in the fcrontab that runs the updates.
+#------------------------------------------------------------------------------
+
+sub disable_updates()
+{
+  my @lines = qx/$fcrontab -l/;
+  my $found = 0;
+
+  foreach my $line (@lines)
+  {
+    if ($line =~ m|/usr/local/bin/ipblacklist|)
+    {
+      return if ($line =~ m/^#/); # Already disabled
+
+      # Found - comment the line
+
+      $line  =~ s/^#*/#/;
+      $found = 1;
+      log_message LOG_INFO, "Disable IP Address Blacklist updates";
+      last;
+    }
+  }
+
+  return unless ($found); # Don't update crontab unnecessarily
+
+  open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+  print FCRONTAB @lines;
+  close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_text_with_hash_comments( line )
+#
+# Parses an input line removing comments.
+#
+# Parameters:
+#   line  The line to parse
+#
+# Returns:
+#   Either an IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_text_with_hash_comments( $ )
+{
+  my ($line) = @_;
+
+  return "" if ($line =~ m/^\s*#/);
+
+  $line =~ s/#.*$//;
+
+  $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|;
+
+  return $1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_text_with_semicolon_comments( line )
+#
+# Parses an input line removing comments.
+#
+# Parameters:
+#   line  The line to parse
+#
+# Returns:
+#   Either and IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_text_with_semicolon_comments( $ )
+{
+  my ($line) = @_;
+
+  return "" if ($line =~ m/^\s*;/);
+
+  $line =~ s/;.*$//;
+
+  $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|;
+
+  return $1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_dshield( line )
+#
+# Parses an input line removing comments.
+#
+# The format is:
+# Start Addrs   End Addrs   Netmask   Nb Attacks   Network Name   Country   email
+# We're only interested in the start address and netmask.
+#
+# Parameters:
+#   line  The line to parse
+#
+# Returns:
+#   Either and IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_dshield( $ )
+{
+  my ($line) = @_;
+
+  return "" if ($line =~ m/^\s*#/);
+
+  $line =~ s/#.*$//;
+
+  $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d+)?\s+(\d+)|;
+
+  return unless ($1);
+  return "$1/32" unless ($2);
+
+  return "$1/$2";
+}
+
+
+#------------------------------------------------------------------------------
+# sub iptables( cmd )
+#
+# Executes an IPTables command, waiting for the lock to ensure only one change
+# is made at a time.
+#
+# Parameters:
+#   cmd  The command to execute
+#
+# Returns:
+#   Status of command
+#------------------------------------------------------------------------------
+
+sub iptables( $ )
+{
+  my ($cmd) = @_;
+
+  return system( "$iptables $cmd" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub ipset( cmd )
+#
+# Executes an IPSet command.  The command is piped to a sub-process running
+# ipset, rather than exected separately.  This saves the overhead of starting a
+# new process for each command.  The sub-process is started if it's not already
+# running.
+#
+# Parameters:
+#   cmd  The command to execute
+#------------------------------------------------------------------------------
+
+sub ipset( $ )
+{
+  my ($cmd) = @_;
+
+  if (not $ipset_running)
+  {
+    local $SIG{PIPE} = 'IGNORE';
+    open IPSET, "|-", $ipset, "restore" or die "Can't start ipset: $!";
+    $ipset_running = 1;
+  }
+
+  print IPSET "$cmd\n";
+}
+
+
+#------------------------------------------------------------------------------
+# sub stop_ipset( )
+#
+# Stops the ipset sub-process.
+#------------------------------------------------------------------------------
+
+sub stop_ipset( )
+{
+  if ($ipset_running)
+  {
+    close IPSET or abort "ipset process died: $! $?";
+    $ipset_running = 0;
+  }
+}
+
+
+#------------------------------------------------------------------------------
+# sub abort( message, parameters... )
+#
+# Aborts the current activity, printing out an error message.
+#
+# Parameters:
+#   message     Message to be printed
+#------------------------------------------------------------------------------
+
+sub abort( $ )
+{
+  my ($message) = @_;
+
+  log_message( LOG_ERR, $message );
+  carp $message;
+}
+
+
+#------------------------------------------------------------------------------
+# sub log_message( level, message )
+#
+# Logs a message.  If the script is run from a terminal messages are also
+# output on STDOUT.
+#
+# Parameters:
+#   level   Severity of message
+#   message Message to be logged
+#------------------------------------------------------------------------------
+
+sub log_message( $$ )
+{
+  my ($level, $message) = @_;
+
+  print "($level) $message\n" if (-t STDIN);
+  syslog( $level, $message );
+}
+
+
+#------------------------------------------------------------------------------
+# sub debug( level, message )
+#
+# Optionally logs a debug message.  If the script is run from a terminal, level
+# 1 debug messages are output regardless of the debug setting.
+#
+# Parameters:
+#   level   Debug level
+#   message Message to be logged
+#------------------------------------------------------------------------------
+
+sub debug( $$ )
+{
+  my ($level, $message) = @_;
+
+  if (($level <= $settings{'DEBUG'}) or
+      ($level == 1 and -t STDIN))
+  {
+    log_message LOG_DEBUG, $message;
+  }
+}
-- 
2.16.4