Re: [PATCH v2 2/3] proxy.cgi: Implement proactive Fast Flux detection and detection for selectively announced destinations

[Michael Tremer <michael.tremer@ipfire.org>]

[-- Attachment #1: Type: text/plain, Size: 8466 bytes --]

Looks okay to me now. Thank you for working on this.

Reviewed-by: Michael Tremer <michael.tremer(a)ipfire.org>

> On 10 Oct 2021, at 18:43, Peter Müller <peter.mueller(a)ipfire.org> wrote:
> 
> This patch adds two new features to IPFire's web proxy:
> 
> (a) Proactive Fast Flux detection
>    FQDNs are resolved to their IP addresses, which are then resolved to
>    corresponding Autonomous System Numbers using IPFire's location
>    database. Most destinations will scatter across a very low number of
>    ASNs (not to be confused with IP addresses!). FQDNs hosted on Fast
>    Flux setups have a significantly higher ASN diversity (5 is usually
>    a good threshold), so they can be proactively detected.
> 
> (b) Detection for selectively announced destinations
>    Especially in targeted operations, miscreants host FQDNs for
>    exfiltrating data or malware distributions on ASNs not announced
>    globally, but only to the intended victim or it's upstream ISPs.
> 
>    That way, security researchers located in other parts of the
>    internet have no insights into these attacks, hence not being able
>    to publish listings or send take down notices for the domains used.
> 
>    While RPKI made this attack harder, it can still be observed every
>    now and then.
> 
>    This feature also protects against accessing FQDNs resolving to IP
>    addresses not being globally routeable, hence providing a trivial
>    mitigation for so-called "rebound attacks" - which we cannot filter
>    at DNS level currently.
> 
> The second version of this patch consumes the user-defined whitelist for
> the URL filter (if present and populated) for the ASNBL helper as well,
> to make exceptions for funny destinations such as fedoraproject.org
> possible. In addition, the ASNBL helper's sanity tests no longer include
> publicly routable IP addresses, so failures on location01 cannot brick
> IPFire installations in the field.
> 
> Thanks to Michael Tremer and Adolf Belka for these suggestions.
> 
> Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org>
> ---
> html/cgi-bin/proxy.cgi | 98 ++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 98 insertions(+)
> 
> diff --git a/html/cgi-bin/proxy.cgi b/html/cgi-bin/proxy.cgi
> index 966593e4d..202a8f3bc 100644
> --- a/html/cgi-bin/proxy.cgi
> +++ b/html/cgi-bin/proxy.cgi
> @@ -21,6 +21,7 @@
> 
> use strict;
> use Apache::Htpasswd;
> +use Scalar::Util qw(looks_like_number);
> 
> # enable only the following on debugging purpose
> #use warnings;
> @@ -229,6 +230,9 @@ $proxysettings{'THROTTLING_GREEN_TOTAL'} = 'unlimited';
> $proxysettings{'THROTTLING_GREEN_HOST'} = 'unlimited';
> $proxysettings{'THROTTLING_BLUE_TOTAL'} = 'unlimited';
> $proxysettings{'THROTTLING_BLUE_HOST'} = 'unlimited';
> +$proxysettings{'ASNBL_FASTFLUX_DETECTION'} = 'off';
> +$proxysettings{'ASNBL_FASTFLUX_THRESHOLD'} = '5';
> +$proxysettings{'ASNBL_SELECANN_DETECTION'} = 'off';
> $proxysettings{'ENABLE_MIME_FILTER'} = 'off';
> $proxysettings{'AUTH_METHOD'} = 'none';
> $proxysettings{'AUTH_REALM'} = '';
> @@ -418,6 +422,21 @@ if (($proxysettings{'ACTION'} eq $Lang::tr{'save'}) || ($proxysettings{'ACTION'}
> 		$errormessage = $Lang::tr{'invalid maximum incoming size'};
> 		goto ERROR;
> 	}
> +	if (($proxysettings{'ASNBL_FASTFLUX_DETECTION'} eq 'on') || ($proxysettings{'ASNBL_SELECANN_DETECTION'} eq 'on'))
> +	{
> +		if (-z $proxysettings{'ASNBL_FASTFLUX_THRESHOLD'}) {
> +			$errormessage = $Lang::tr{'advproxy fastflux no threshold given'};
> +			goto ERROR;
> +		}
> +		if (! looks_like_number($proxysettings{'ASNBL_FASTFLUX_THRESHOLD'})) {
> +			$errormessage = $Lang::tr{'advproxy fastflux threshold invalid'};
> +			goto ERROR;
> +		}
> +		if (($proxysettings{'ASNBL_FASTFLUX_THRESHOLD'} < 2) || ($proxysettings{'ASNBL_FASTFLUX_THRESHOLD'} > 10)) {
> +			$errormessage = $Lang::tr{'advproxy fastflux threshold out of bounds'};
> +			goto ERROR;
> +		}
> +	}
> 	if (!($proxysettings{'AUTH_METHOD'} eq 'none'))
> 	{
> 		unless (($proxysettings{'AUTH_METHOD'} eq 'ident') &&
> @@ -801,6 +820,14 @@ $selected{'THROTTLING_GREEN_HOST'}{$proxysettings{'THROTTLING_GREEN_HOST'}} = "s
> $selected{'THROTTLING_BLUE_TOTAL'}{$proxysettings{'THROTTLING_BLUE_TOTAL'}} = "selected='selected'";
> $selected{'THROTTLING_BLUE_HOST'}{$proxysettings{'THROTTLING_BLUE_HOST'}} = "selected='selected'";
> 
> +$checked{'ASNBL_FASTFLUX_DETECTION'}{'off'} = '';
> +$checked{'ASNBL_FASTFLUX_DETECTION'}{'on'} = '';
> +$checked{'ASNBL_FASTFLUX_DETECTION'}{$proxysettings{'ASNBL_FASTFLUX_DETECTION'}} = "checked='checked'";
> +
> +$checked{'ASNBL_SELECANN_DETECTION'}{'off'} = '';
> +$checked{'ASNBL_SELECANN_DETECTION'}{'on'} = '';
> +$checked{'ASNBL_SELECANN_DETECTION'}{$proxysettings{'ASNBL_SELECANN_DETECTION'}} = "checked='checked'";
> +
> $checked{'ENABLE_MIME_FILTER'}{'off'} = '';
> $checked{'ENABLE_MIME_FILTER'}{'on'} = '';
> $checked{'ENABLE_MIME_FILTER'}{$proxysettings{'ENABLE_MIME_FILTER'}} = "checked='checked'";
> @@ -1633,6 +1660,24 @@ END
> print <<END
> </table>
> 
> +<hr size='1'>
> +
> +<table width='100%'>
> +<tr>
> +       <td><b>$Lang::tr{'advproxy asbased anomaly detection'}</b></td>
> +</tr>
> +<tr>
> +       <td class='base'>$Lang::tr{'advproxy fastflux detection'}:</td>
> +       <td><input type='checkbox' name='ASNBL_FASTFLUX_DETECTION' $checked{'ASNBL_FASTFLUX_DETECTION'}{'on'} /></td>
> +       <td class='base'>$Lang::tr{'advproxy fastflux detection threshold'}:</td>
> +       <td><input type='text' name='ASNBL_FASTFLUX_THRESHOLD' value='$proxysettings{'ASNBL_FASTFLUX_THRESHOLD'}' size=2 /></td>
> +</tr>
> +<tr>
> +       <td class='base'>$Lang::tr{'advproxy selectively announcements detection'}:</td>
> +       <td colspan='3'><input type='checkbox' name='ASNBL_SELECANN_DETECTION' $checked{'ASNBL_SELECANN_DETECTION'}{'on'} /></td>
> +</tr>
> +</table>
> +
> <hr size='1'>
> END
> ;
> @@ -3525,6 +3570,59 @@ if (@ssl_ports) {
> 	print FILE "http_access deny  CONNECT !SSL_ports\n";
> }
> 
> +	if ((($proxysettings{'ASNBL_FASTFLUX_DETECTION'} eq 'on') && (!-z $proxysettings{'ASNBL_FASTFLUX_THRESHOLD'})) || ($proxysettings{'ASNBL_SELECANN_DETECTION'} eq 'on')) {
> +		print FILE "external_acl_type asnblhelper children-max=10 children-startup=2 ttl=86400 %DST /usr/bin/asnbl-helper.py ${General::swroot}/proxy/asnbl-helper.conf\n";
> +		print FILE "acl asnbl external asnblhelper\n";
> +
> +		# Use the user-defined URL filter whitelist (if present and populated) for the ASNBL helper as well
> +		# Necessary for destinations such as fedoraproject.org, but we do not want to maintain a dedicated
> +		# or hardcoded list for such FQDNs.
> +		if ((-e "${General::swroot}/urlfilter/blacklists/custom/allowed/domains") && (!-z "${General::swroot}/urlfilter/blacklists/custom/allowed/domains")) {
> +			print FILE "acl asnbl_whitelisted_destinations dstdomain \"${General::swroot}/urlfilter/blacklists/custom/allowed/domains\"\n";
> +			print FILE "http_access deny asnbl !asnbl_whitelisted_destinations\n\n";
> +		} else {
> +			print FILE "http_access deny asnbl\n\n";
> +		}
> +
> +		# Write ASNBL helper configuration file...
> +		open(ASNBLFILE, ">${General::swroot}/proxy/asnbl-helper.conf");
> +		flock(ASNBLFILE, 2);
> +
> +		print ASNBLFILE<<END
> +#
> +# This file has been automatically generated. Manual changes will be overwritten.
> +#
> +
> +[GENERAL]
> +LOGLEVEL = INFO
> +ASNDB_PATH = /var/lib/location/database.db
> +USE_REPLYMAP = no
> +END
> +;
> +
> +		print ASNBLFILE "AS_DIVERSITY_THRESHOLD = $proxysettings{'ASNBL_FASTFLUX_THRESHOLD'}\n";
> +
> +		if ($proxysettings{'ASNBL_SELECANN_DETECTION'} eq 'on') {
> +			print ASNBLFILE "BLOCK_SUSPECTED_SELECTIVE_ANNOUNCEMENTS = yes\n";
> +		} else {
> +			print ASNBLFILE "BLOCK_SUSPECTED_SELECTIVE_ANNOUNCEMENTS = no\n";
> +		}
> +
> +		if ($proxysettings{'ASNBL_FASTFLUX_DETECTION'} eq 'on') {
> +			print ASNBLFILE "BLOCK_DIVERSITY_EXCEEDING_DESTINATIONS = yes\n";
> +		} else {
> +			print ASNBLFILE "BLOCK_DIVERSITY_EXCEEDING_DESTINATIONS = no\n";
> +		}
> +
> +		print ASNBLFILE<<END
> +TESTDATA = (10.0.0.1, 0) (127.0.0.1, 0) (fe80::1, 0)
> +ACTIVE_ASNBLS = 
> +END
> +;
> +
> +		close ASNBLFILE;
> +    }
> +
> if ($proxysettings{'AUTH_METHOD'} eq 'ident')
> {
> print FILE "#Set ident ACLs\n";
> -- 
> 2.26.2