[PATCH 1/2] rules.pl: Allow dynamic destory of loaded but unused ipset sets.

[PATCH 1/2] rules.pl: Allow dynamic destory of loaded but unused ipset sets.

[Stefan Schantl]

[-- Attachment #1: Type: text/plain, Size: 4687 bytes --]

Instead of stupidly destroying all ipsets, we now grab the already loaded sets
and compare them with the loaded sets during runtime of the script.

So we are now able to determine which sets are not longer required and
safely can destroy (unload) at a later time.

This saves us from taking care about dropping/flushing rules which are
based on ipset before we can destroy them - because only unused sets are
affected.

Signed-off-by: Stefan Schantl <stefan.schantl(a)ipfire.org>
Inspired-by: Tim FitzGeorge <ipfr(a)tfitzgeorge.me.uk>
---
 config/firewall/rules.pl | 68 ++++++++++++++++++++++++++++++----------
 1 file changed, 52 insertions(+), 16 deletions(-)

diff --git a/config/firewall/rules.pl b/config/firewall/rules.pl
index 927c1f2ba..7a7c8ed31 100644
--- a/config/firewall/rules.pl
+++ b/config/firewall/rules.pl
@@ -70,7 +70,8 @@ my %confignatfw=();
 my %locationsettings = (
 	"LOCATIONBLOCK_ENABLED" => "off"
 );
-my %loaded_ipset_lists=();
+my %ipset_loaded_sets = ();
+my @ipset_used_sets = ();
 
 my $configfwdfw		= "${General::swroot}/firewall/config";
 my $configinput	    = "${General::swroot}/firewall/input";
@@ -114,12 +115,12 @@ undef (@dummy);
 &main();
 
 sub main {
+	# Get currently used ipset sets.
+	&ipset_get_sets();
+
 	# Flush all chains.
 	&flush();
 
-	# Destroy all existing ipsets.
-	run("$IPSET destroy");
-
 	# Prepare firewall rules.
 	if (! -z  "${General::swroot}/firewall/input"){
 		&buildrules(\%configinputfw);
@@ -137,6 +138,9 @@ sub main {
 	# Reload firewall policy.
 	run("/usr/sbin/firewall-policy");
 
+	# Cleanup not longer needed ipset sets.
+	&ipset_cleanup();
+
 	#Reload firewall.local if present
 	if ( -f '/etc/sysconfig/firewall.local'){
 		run("/etc/sysconfig/firewall.local reload");
@@ -189,9 +193,6 @@ sub flush {
 	run("$IPTABLES -t nat -F $CHAIN_NAT_SOURCE");
 	run("$IPTABLES -t nat -F $CHAIN_NAT_DESTINATION");
 	run("$IPTABLES -t mangle -F $CHAIN_MANGLE_NAT_DESTINATION_FIX");
-
-	# Flush LOCATIONBLOCK chain.
-	run("$IPTABLES -F LOCATIONBLOCK");
 }
 
 sub buildrules {
@@ -639,7 +640,8 @@ sub time_convert_to_minutes {
 }
 
 sub locationblock {
-	# The LOCATIONBLOCK chain now gets flushed by the flush() function.
+	# Flush LOCATIONBLOCK chain.
+	run("$IPTABLES -F LOCATIONBLOCK");
 
 	# If location blocking is not enabled, we are finished here.
 	if ($locationsettings{'LOCATIONBLOCK_ENABLED'} ne "on") {
@@ -669,7 +671,7 @@ sub locationblock {
 			&ipset_restore($location);
 
 			# Call iptables and create rule to use the loaded ipset list.
-			run("$IPTABLES -A LOCATIONBLOCK -m set --match-set CC_$location src -j DROP");
+			run("$IPTABLES -A LOCATIONBLOCK -m set --match-set $location src -j DROP");
 		}
 	}
 }
@@ -887,24 +889,58 @@ sub firewall_is_in_subnet {
 	return 0;
 }
 
+sub ipset_get_sets () {
+	# Get all currently used ipset lists and store them in an array.
+	my @output = `$IPSET -n list`;
+
+	# Loop through the temporary array.
+	foreach my $set (@output) {
+		# Remove any newlines.
+		chomp($set);
+
+		# Add the set the array of used sets.
+		push(@ipset_used_sets, $set);
+	}
+
+	# Display used sets in debug mode.
+	if($DEBUG) {
+		print "Used ipset sets:\n";
+		print "@ipset_used_sets\n\n";
+	}
+}
+
 sub ipset_restore ($) {
-	my ($list) = @_;
+	my ($set) = @_;
 
 	my $file_prefix = "ipset4";
-	my $db_file = "$Location::Functions::ipset_db_directory/$list.$file_prefix";
+	my $db_file = "$Location::Functions::ipset_db_directory/$set.$file_prefix";
 
-	# Check if the network list already has been loaded.
-	if($loaded_ipset_lists{$list}) {
+	# Check if the set already has been loaded.
+	if($ipset_loaded_sets{$set}) {
 		# It already has been loaded - so there is nothing to do.
 		return;
 	}
 
 	# Check if the generated file exists.
 	if (-f $db_file) {
-		# Run ipset and restore the list of the given country code.
+		# Run ipset and restore the given set.
 		run("$IPSET restore < $db_file");
 
-		# Store the restored list name to the hash to prevent from loading it again.
-		$loaded_ipset_lists{$list} = "1";
+		# Store the restored set to the hash to prevent from loading it again.
+		$ipset_loaded_sets{$set} = "1";
+	}
+}
+
+sub ipset_cleanup () {
+	# Loop through the array of used sets.
+	foreach my $set (@ipset_used_sets) {
+		# Check if this set is still in use.
+		#
+		# In this case an entry in the loaded sets hash exists.
+		unless($ipset_loaded_sets{$set}) {
+			# Entry does not exist, so this set is not longer
+			# used and can be destroyed.
+			run("$IPSET destroy $set");
+		}
 	}
 }
-- 
2.30.2

[PATCH 2/2] firewall: Move dropping hostile networks to rules.pl.

[Stefan Schantl]

[-- Attachment #1: Type: text/plain, Size: 3972 bytes --]

Signed-off-by: Stefan Schantl <stefan.schantl(a)ipfire.org>
---
 config/firewall/rules.pl        | 33 +++++++++++++++++++++++++++++++++
 src/initscripts/system/firewall | 23 ++++++++---------------
 2 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/config/firewall/rules.pl b/config/firewall/rules.pl
index 7a7c8ed31..b12764d18 100644
--- a/config/firewall/rules.pl
+++ b/config/firewall/rules.pl
@@ -59,6 +59,9 @@ my @PRIVATE_NETWORKS = (
 # MARK masks
 my $NAT_MASK = 0x0f000000;
 
+# Country code, which is used to mark hostile networks.
+my $HOSTILE_CCODE = "XD";
+
 my %fwdfwsettings=();
 my %fwoptions = ();
 my %defaultNetworks=();
@@ -97,6 +100,9 @@ if (-e "$locationfile") {
 # Get all available locations.
 my @locations = &Location::Functions::get_locations();
 
+# Name or the RED interface.
+my $RED_DEV = &General::get_red_interface();
+
 my @log_limit_options = &make_log_limit_options();
 
 my $POLICY_INPUT_ALLOWED   = 0;
@@ -135,6 +141,9 @@ sub main {
 	# Load Location block rules.
 	&locationblock();
 
+	# Load rules to block hostile networks.
+	&drop_hostile_networks();
+
 	# Reload firewall policy.
 	run("/usr/sbin/firewall-policy");
 
@@ -676,6 +685,30 @@ sub locationblock {
 	}
 }
 
+sub drop_hostile_networks () {
+	# Flush the HOSTILE firewall chain.
+	run("$IPTABLES -F HOSTILE");
+
+	# If dropping hostile networks is not enabled, we are finished here.
+	if ($fwoptions{'DROPHOSTILE'} ne "on") {
+		# Exit function.
+		return;
+	}
+
+	# Call function to load the network list of hostile networks.
+	&ipset_restore($HOSTILE_CCODE);
+
+	# Setup rules to pass traffic which does not belong to a hostile network.
+	run("$IPTABLES -A HOSTILE -i $RED_DEV -m set ! --match-set $HOSTILE_CCODE src -j RETURN");
+	run("$IPTABLES -A HOSTILE -o $RED_DEV -m set ! --match-set $HOSTILE_CCODE dst -j RETURN");
+
+	# Setup logging.
+	run("$IPTABLES -A HOSTILE -m limit --limit 10/second -j LOG  --log-prefix \"DROP_HOSTILE \"");
+
+	# Drop traffic from/to hostile network.
+        run("$IPTABLES -A HOSTILE -j DROP -m comment --comment \"DROP_HOSTILE\"");
+}
+
 sub get_protocols {
 	my $hash = shift;
 	my $key = shift;
diff --git a/src/initscripts/system/firewall b/src/initscripts/system/firewall
index 22e3fae59..2c4d3163b 100644
--- a/src/initscripts/system/firewall
+++ b/src/initscripts/system/firewall
@@ -169,21 +169,6 @@ iptables_init() {
 	iptables -t nat -N CUSTOMPOSTROUTING
 	iptables -t nat -A POSTROUTING -j CUSTOMPOSTROUTING
 
-	# Log and drop any traffic from and to networks known as being hostile, posing
-	# a technical threat to our users (i. e. listed at Spamhaus DROP et al.)
-	iptables -N HOSTILE
-	if [ "$DROPHOSTILE" == "on" ]; then
-		# Call ipset and load the list which contains the hostile networks.
-		ipset restore < $IPSET_DB_DIR/CC_XD.ipset4
-
-		iptables -A HOSTILE -m limit --limit 10/second -j LOG  --log-prefix "DROP_HOSTILE "
-		iptables -A INPUT   -i $IFACE -m set --match-set CC_XD src -j HOSTILE
-		iptables -A FORWARD -i $IFACE -m set --match-set CC_XD src -j HOSTILE
-		iptables -A FORWARD -o $IFACE -m set --match-set CC_XD dst -j HOSTILE
-		iptables -A OUTPUT  -o $IFACE -m set --match-set CC_XD src -j HOSTILE
-	fi
-	iptables -A HOSTILE -j DROP -m comment --comment "DROP_HOSTILE"
-
 	# IPS (Guardian) chains
 	iptables -N GUARDIAN
 	iptables -A INPUT -j GUARDIAN
@@ -274,6 +259,14 @@ iptables_init() {
 		iptables -A OUTPUT -o "${BLUE_DEV}" -j DHCPBLUEOUTPUT
 	fi
 
+	# Chains for networks known as being hostile, posing a technical threat to our users
+	# (i. e. listed at Spamhaus DROP et al.)
+	iptables -N HOSTILE
+	iptables -A INPUT -i $IFACE -j HOSTILE
+	iptables -A FORWARD -i $IFACE -j HOSTILE
+	iptables -A FORWARD -o $IFACE -j HOSTILE
+	iptables -A OUTPUT -o $IFACE -j HOSTILE
+
 	# Tor (inbound)
 	iptables -N TOR_INPUT
 	iptables -A INPUT -j TOR_INPUT
-- 
2.30.2

Re: [PATCH 1/2] rules.pl: Allow dynamic destory of loaded but unused ipset sets.

[Michael Tremer]

[-- Attachment #1: Type: text/plain, Size: 5158 bytes --]

Hello,

> On 27 Feb 2022, at 13:49, Stefan Schantl <stefan.schantl(a)ipfire.org> wrote:
> 
> Instead of stupidly destroying all ipsets, we now grab the already loaded sets
> and compare them with the loaded sets during runtime of the script.
> 
> So we are now able to determine which sets are not longer required and
> safely can destroy (unload) at a later time.
> 
> This saves us from taking care about dropping/flushing rules which are
> based on ipset before we can destroy them - because only unused sets are
> affected.
> 
> Signed-off-by: Stefan Schantl <stefan.schantl(a)ipfire.org>
> Inspired-by: Tim FitzGeorge <ipfr(a)tfitzgeorge.me.uk>

LOL. This isn’t an official tag :)

But nevertheless:

Reviewed-by: Michael Tremer <michael.tremer(a)ipfire.org>

> ---
> config/firewall/rules.pl | 68 ++++++++++++++++++++++++++++++----------
> 1 file changed, 52 insertions(+), 16 deletions(-)
> 
> diff --git a/config/firewall/rules.pl b/config/firewall/rules.pl
> index 927c1f2ba..7a7c8ed31 100644
> --- a/config/firewall/rules.pl
> +++ b/config/firewall/rules.pl
> @@ -70,7 +70,8 @@ my %confignatfw=();
> my %locationsettings = (
> 	"LOCATIONBLOCK_ENABLED" => "off"
> );
> -my %loaded_ipset_lists=();
> +my %ipset_loaded_sets = ();
> +my @ipset_used_sets = ();
> 
> my $configfwdfw		= "${General::swroot}/firewall/config";
> my $configinput	    = "${General::swroot}/firewall/input";
> @@ -114,12 +115,12 @@ undef (@dummy);
> &main();
> 
> sub main {
> +	# Get currently used ipset sets.
> +	&ipset_get_sets();
> +
> 	# Flush all chains.
> 	&flush();
> 
> -	# Destroy all existing ipsets.
> -	run("$IPSET destroy");
> -
> 	# Prepare firewall rules.
> 	if (! -z  "${General::swroot}/firewall/input"){
> 		&buildrules(\%configinputfw);
> @@ -137,6 +138,9 @@ sub main {
> 	# Reload firewall policy.
> 	run("/usr/sbin/firewall-policy");
> 
> +	# Cleanup not longer needed ipset sets.
> +	&ipset_cleanup();
> +
> 	#Reload firewall.local if present
> 	if ( -f '/etc/sysconfig/firewall.local'){
> 		run("/etc/sysconfig/firewall.local reload");
> @@ -189,9 +193,6 @@ sub flush {
> 	run("$IPTABLES -t nat -F $CHAIN_NAT_SOURCE");
> 	run("$IPTABLES -t nat -F $CHAIN_NAT_DESTINATION");
> 	run("$IPTABLES -t mangle -F $CHAIN_MANGLE_NAT_DESTINATION_FIX");
> -
> -	# Flush LOCATIONBLOCK chain.
> -	run("$IPTABLES -F LOCATIONBLOCK");
> }
> 
> sub buildrules {
> @@ -639,7 +640,8 @@ sub time_convert_to_minutes {
> }
> 
> sub locationblock {
> -	# The LOCATIONBLOCK chain now gets flushed by the flush() function.
> +	# Flush LOCATIONBLOCK chain.
> +	run("$IPTABLES -F LOCATIONBLOCK");
> 
> 	# If location blocking is not enabled, we are finished here.
> 	if ($locationsettings{'LOCATIONBLOCK_ENABLED'} ne "on") {
> @@ -669,7 +671,7 @@ sub locationblock {
> 			&ipset_restore($location);
> 
> 			# Call iptables and create rule to use the loaded ipset list.
> -			run("$IPTABLES -A LOCATIONBLOCK -m set --match-set CC_$location src -j DROP");
> +			run("$IPTABLES -A LOCATIONBLOCK -m set --match-set $location src -j DROP");
> 		}
> 	}
> }
> @@ -887,24 +889,58 @@ sub firewall_is_in_subnet {
> 	return 0;
> }
> 
> +sub ipset_get_sets () {
> +	# Get all currently used ipset lists and store them in an array.
> +	my @output = `$IPSET -n list`;
> +
> +	# Loop through the temporary array.
> +	foreach my $set (@output) {
> +		# Remove any newlines.
> +		chomp($set);
> +
> +		# Add the set the array of used sets.
> +		push(@ipset_used_sets, $set);
> +	}
> +
> +	# Display used sets in debug mode.
> +	if($DEBUG) {
> +		print "Used ipset sets:\n";
> +		print "@ipset_used_sets\n\n";
> +	}
> +}
> +
> sub ipset_restore ($) {
> -	my ($list) = @_;
> +	my ($set) = @_;
> 
> 	my $file_prefix = "ipset4";
> -	my $db_file = "$Location::Functions::ipset_db_directory/$list.$file_prefix";
> +	my $db_file = "$Location::Functions::ipset_db_directory/$set.$file_prefix";
> 
> -	# Check if the network list already has been loaded.
> -	if($loaded_ipset_lists{$list}) {
> +	# Check if the set already has been loaded.
> +	if($ipset_loaded_sets{$set}) {
> 		# It already has been loaded - so there is nothing to do.
> 		return;
> 	}
> 
> 	# Check if the generated file exists.
> 	if (-f $db_file) {
> -		# Run ipset and restore the list of the given country code.
> +		# Run ipset and restore the given set.
> 		run("$IPSET restore < $db_file");
> 
> -		# Store the restored list name to the hash to prevent from loading it again.
> -		$loaded_ipset_lists{$list} = "1";
> +		# Store the restored set to the hash to prevent from loading it again.
> +		$ipset_loaded_sets{$set} = "1";
> +	}
> +}
> +
> +sub ipset_cleanup () {
> +	# Loop through the array of used sets.
> +	foreach my $set (@ipset_used_sets) {
> +		# Check if this set is still in use.
> +		#
> +		# In this case an entry in the loaded sets hash exists.
> +		unless($ipset_loaded_sets{$set}) {
> +			# Entry does not exist, so this set is not longer
> +			# used and can be destroyed.
> +			run("$IPSET destroy $set");
> +		}
> 	}
> }
> -- 
> 2.30.2
> 

Re: [PATCH 2/2] firewall: Move dropping hostile networks to rules.pl.

[Michael Tremer]

[-- Attachment #1: Type: text/plain, Size: 4311 bytes --]

Reviewed-by: Michael Tremer <michael.tremer(a)ipfire.org>

> On 27 Feb 2022, at 13:49, Stefan Schantl <stefan.schantl(a)ipfire.org> wrote:
> 
> Signed-off-by: Stefan Schantl <stefan.schantl(a)ipfire.org>
> ---
> config/firewall/rules.pl        | 33 +++++++++++++++++++++++++++++++++
> src/initscripts/system/firewall | 23 ++++++++---------------
> 2 files changed, 41 insertions(+), 15 deletions(-)
> 
> diff --git a/config/firewall/rules.pl b/config/firewall/rules.pl
> index 7a7c8ed31..b12764d18 100644
> --- a/config/firewall/rules.pl
> +++ b/config/firewall/rules.pl
> @@ -59,6 +59,9 @@ my @PRIVATE_NETWORKS = (
> # MARK masks
> my $NAT_MASK = 0x0f000000;
> 
> +# Country code, which is used to mark hostile networks.
> +my $HOSTILE_CCODE = "XD";
> +
> my %fwdfwsettings=();
> my %fwoptions = ();
> my %defaultNetworks=();
> @@ -97,6 +100,9 @@ if (-e "$locationfile") {
> # Get all available locations.
> my @locations = &Location::Functions::get_locations();
> 
> +# Name or the RED interface.
> +my $RED_DEV = &General::get_red_interface();
> +
> my @log_limit_options = &make_log_limit_options();
> 
> my $POLICY_INPUT_ALLOWED   = 0;
> @@ -135,6 +141,9 @@ sub main {
> 	# Load Location block rules.
> 	&locationblock();
> 
> +	# Load rules to block hostile networks.
> +	&drop_hostile_networks();
> +
> 	# Reload firewall policy.
> 	run("/usr/sbin/firewall-policy");
> 
> @@ -676,6 +685,30 @@ sub locationblock {
> 	}
> }
> 
> +sub drop_hostile_networks () {
> +	# Flush the HOSTILE firewall chain.
> +	run("$IPTABLES -F HOSTILE");
> +
> +	# If dropping hostile networks is not enabled, we are finished here.
> +	if ($fwoptions{'DROPHOSTILE'} ne "on") {
> +		# Exit function.
> +		return;
> +	}
> +
> +	# Call function to load the network list of hostile networks.
> +	&ipset_restore($HOSTILE_CCODE);
> +
> +	# Setup rules to pass traffic which does not belong to a hostile network.
> +	run("$IPTABLES -A HOSTILE -i $RED_DEV -m set ! --match-set $HOSTILE_CCODE src -j RETURN");
> +	run("$IPTABLES -A HOSTILE -o $RED_DEV -m set ! --match-set $HOSTILE_CCODE dst -j RETURN");
> +
> +	# Setup logging.
> +	run("$IPTABLES -A HOSTILE -m limit --limit 10/second -j LOG  --log-prefix \"DROP_HOSTILE \"");
> +
> +	# Drop traffic from/to hostile network.
> +        run("$IPTABLES -A HOSTILE -j DROP -m comment --comment \"DROP_HOSTILE\"");
> +}
> +
> sub get_protocols {
> 	my $hash = shift;
> 	my $key = shift;
> diff --git a/src/initscripts/system/firewall b/src/initscripts/system/firewall
> index 22e3fae59..2c4d3163b 100644
> --- a/src/initscripts/system/firewall
> +++ b/src/initscripts/system/firewall
> @@ -169,21 +169,6 @@ iptables_init() {
> 	iptables -t nat -N CUSTOMPOSTROUTING
> 	iptables -t nat -A POSTROUTING -j CUSTOMPOSTROUTING
> 
> -	# Log and drop any traffic from and to networks known as being hostile, posing
> -	# a technical threat to our users (i. e. listed at Spamhaus DROP et al.)
> -	iptables -N HOSTILE
> -	if [ "$DROPHOSTILE" == "on" ]; then
> -		# Call ipset and load the list which contains the hostile networks.
> -		ipset restore < $IPSET_DB_DIR/CC_XD.ipset4
> -
> -		iptables -A HOSTILE -m limit --limit 10/second -j LOG  --log-prefix "DROP_HOSTILE "
> -		iptables -A INPUT   -i $IFACE -m set --match-set CC_XD src -j HOSTILE
> -		iptables -A FORWARD -i $IFACE -m set --match-set CC_XD src -j HOSTILE
> -		iptables -A FORWARD -o $IFACE -m set --match-set CC_XD dst -j HOSTILE
> -		iptables -A OUTPUT  -o $IFACE -m set --match-set CC_XD src -j HOSTILE
> -	fi
> -	iptables -A HOSTILE -j DROP -m comment --comment "DROP_HOSTILE"
> -
> 	# IPS (Guardian) chains
> 	iptables -N GUARDIAN
> 	iptables -A INPUT -j GUARDIAN
> @@ -274,6 +259,14 @@ iptables_init() {
> 		iptables -A OUTPUT -o "${BLUE_DEV}" -j DHCPBLUEOUTPUT
> 	fi
> 
> +	# Chains for networks known as being hostile, posing a technical threat to our users
> +	# (i. e. listed at Spamhaus DROP et al.)
> +	iptables -N HOSTILE
> +	iptables -A INPUT -i $IFACE -j HOSTILE
> +	iptables -A FORWARD -i $IFACE -j HOSTILE
> +	iptables -A FORWARD -o $IFACE -j HOSTILE
> +	iptables -A OUTPUT -o $IFACE -j HOSTILE
> +
> 	# Tor (inbound)
> 	iptables -N TOR_INPUT
> 	iptables -A INPUT -j TOR_INPUT
> -- 
> 2.30.2
>