Re: [PATCH] Implement an additional flag for hostile networks safe to drop

[Michael Tremer <michael.tremer@ipfire.org>]

[-- Attachment #1: Type: text/plain, Size: 12644 bytes --]

Hello,

Thank you. Great work. Merged.

-Michael

> On 4 Jun 2021, at 16:57, Peter Müller <peter.mueller(a)ipfire.org> wrote:
> 
> This patch implements an additional flag intended for networks and
> Autonomous Systems being considered hostile. While libloc does not and
> should not be an opinionated database, reality shows it is being used
> this way.
> 
> Hereby, we assign "XD" (drop) as a custom country code for networks
> being flagged this way. According to ISO, "XA" to "XZ" are reserved for
> "user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html),
> so this is a safe thing to do.
> 
> This patch does not interfere with "A1" to "A3", which we currently
> assign outside standardised country code ranges for historical reasons.
> 
> Neither does it specify any policy or source for tagging networks with a
> "drop" flag. Doing so is beyond the scope of this - technical -
> approach.
> 
> To avoid confusions with the SQL "DROP" command, "is_drop" will be used
> as a column name for database operations.
> 
> Thanks to Michael for his remarks and ideas during the run-up.
> 
> Cc: Michael Tremer <michael.tremer(a)ipfire.org>
> Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org>
> ---
> man/location.txt                |  4 ++--
> po/de.po                        |  3 +++
> src/loc/network.h               |  3 ++-
> src/perl/Location.xs            |  2 ++
> src/python/export.py            |  3 ++-
> src/python/location-importer.in | 32 +++++++++++++++++++++++++++-----
> src/python/location.in          | 19 ++++++++++++++++---
> src/python/locationmodule.c     |  5 ++++-
> 8 files changed, 58 insertions(+), 13 deletions(-)
> 
> diff --git a/man/location.txt b/man/location.txt
> index df1da53..b38f21c 100644
> --- a/man/location.txt
> +++ b/man/location.txt
> @@ -10,7 +10,7 @@ location - Query the location database
> `location list-countries [--show-name] [--show-continent]`
> `location list-networks-by-as ASN`
> `location list-networks-by-cc COUNTRY_CODE`
> -`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
> +`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]`
> `location lookup ADDRESS [ADDRESS...]`
> `location search-as STRING`
> `location update [--cron=daily|weekly|monthly]`
> @@ -73,7 +73,7 @@ or countries.
> 	+
> 	See above for usage of the '--family' and '--format' parameters.
> 
> -'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast]'::
> +'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]'::
> 	Lists all networks that have a certain flag.
> 	+
> 	See above for usage of the '--family' and '--format' parameters.
> diff --git a/po/de.po b/po/de.po
> index 3b073d6..3cbcdd7 100644
> --- a/po/de.po
> +++ b/po/de.po
> @@ -152,6 +152,9 @@ msgstr ""
> msgid "Anycasts"
> msgstr ""
> 
> +msgid "Hostile Networks safe to drop"
> +msgstr ""
> +
> msgid "Lists all countries"
> msgstr ""
> 
> diff --git a/src/loc/network.h b/src/loc/network.h
> index af3dafd..a30f653 100644
> --- a/src/loc/network.h
> +++ b/src/loc/network.h
> @@ -1,7 +1,7 @@
> /*
> 	libloc - A library to determine the location of someone on the Internet
> 
> -	Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org>
> +	Copyright (C) 2017-2021 IPFire Development Team <info(a)ipfire.org>
> 
> 	This library is free software; you can redistribute it and/or
> 	modify it under the terms of the GNU Lesser General Public
> @@ -27,6 +27,7 @@ enum loc_network_flags {
> 	LOC_NETWORK_FLAG_ANONYMOUS_PROXY    = (1 << 0), // A1
> 	LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2
> 	LOC_NETWORK_FLAG_ANYCAST            = (1 << 2), // A3
> +	LOC_NETWORK_FLAG_DROP               = (1 << 3), // XD
> };
> 
> struct loc_network;
> diff --git a/src/perl/Location.xs b/src/perl/Location.xs
> index b7676d2..73f85b4 100644
> --- a/src/perl/Location.xs
> +++ b/src/perl/Location.xs
> @@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag)
> 			iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER;
> 		else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0)
> 			iv |= LOC_NETWORK_FLAG_ANYCAST;
> +		else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0)
> +			iv |= LOC_NETWORK_FLAG_DROP;
> 		else
> 			croak("Invalid flag");
> 
> diff --git a/src/python/export.py b/src/python/export.py
> index f0eae26..3b9e1e0 100644
> --- a/src/python/export.py
> +++ b/src/python/export.py
> @@ -3,7 +3,7 @@
> #                                                                             #
> # libloc - A library to determine the location of someone on the Internet     #
> #                                                                             #
> -# Copyright (C) 2020 IPFire Development Team <info(a)ipfire.org>                #
> +# Copyright (C) 2020-2021 IPFire Development Team <info(a)ipfire.org>           #
> #                                                                             #
> # This library is free software; you can redistribute it and/or               #
> # modify it under the terms of the GNU Lesser General Public                  #
> @@ -33,6 +33,7 @@ FLAGS = {
> 	_location.NETWORK_FLAG_ANONYMOUS_PROXY    : "A1",
> 	_location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
> 	_location.NETWORK_FLAG_ANYCAST            : "A3",
> +	_location.NETWORK_FLAG_DROP               : "XD",
> }
> 
> class OutputWriter(object):
> diff --git a/src/python/location-importer.in b/src/python/location-importer.in
> index f796652..ea1e8f5 100644
> --- a/src/python/location-importer.in
> +++ b/src/python/location-importer.in
> @@ -182,6 +182,7 @@ class CLI(object):
> 				);
> 				CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
> 					ON autnum_overrides(number);
> +				ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
> 
> 				CREATE TABLE IF NOT EXISTS network_overrides(
> 					network inet NOT NULL,
> @@ -194,6 +195,7 @@ class CLI(object):
> 					ON network_overrides(network);
> 				CREATE INDEX IF NOT EXISTS network_overrides_search
> 					ON network_overrides USING GIST(network inet_ops);
> +				ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
> 			""")
> 
> 		return db
> @@ -301,7 +303,20 @@ class CLI(object):
> 							WHERE networks.autnum = overrides.number
> 					),
> 					FALSE
> -				) AS is_anycast
> +				) AS is_anycast,
> +				COALESCE(
> +					(
> +						SELECT is_drop FROM network_overrides overrides
> +							WHERE networks.network <<= overrides.network
> +							ORDER BY masklen(overrides.network) DESC
> +							LIMIT 1
> +					),
> +					(
> +						SELECT is_drop FROM autnum_overrides overrides
> +							WHERE networks.autnum = overrides.number
> +					),
> +					FALSE
> +				) AS is_drop
> 			FROM (
> 				SELECT
> 					known_networks.network AS network,
> @@ -350,6 +365,9 @@ class CLI(object):
> 			if row.is_anycast:
> 				network.set_flag(location.NETWORK_FLAG_ANYCAST)
> 
> +			if row.is_drop:
> +				network.set_flag(location.NETWORK_FLAG_DROP)
> +
> 		# Add all countries
> 		log.info("Writing countries...")
> 		rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
> @@ -966,14 +984,16 @@ class CLI(object):
> 									country,
> 									is_anonymous_proxy,
> 									is_satellite_provider,
> -									is_anycast
> -								) VALUES (%s, %s, %s, %s, %s)
> +									is_anycast,
> +									is_drop
> +								) VALUES (%s, %s, %s, %s, %s, %s)
> 								ON CONFLICT (network) DO NOTHING""",
> 								"%s" % network,
> 								block.get("country"),
> 								self._parse_bool(block, "is-anonymous-proxy"),
> 								self._parse_bool(block, "is-satellite-provider"),
> 								self._parse_bool(block, "is-anycast"),
> +								self._parse_bool(block, "drop"),
> 							)
> 
> 						elif type == "aut-num":
> @@ -994,8 +1014,9 @@ class CLI(object):
> 									country,
> 									is_anonymous_proxy,
> 									is_satellite_provider,
> -									is_anycast
> -								) VALUES(%s, %s, %s, %s, %s, %s)
> +									is_anycast,
> +									is_drop
> +								) VALUES(%s, %s, %s, %s, %s, %s, %s)
> 								ON CONFLICT DO NOTHING""",
> 								autnum,
> 								block.get("name"),
> @@ -1003,6 +1024,7 @@ class CLI(object):
> 								self._parse_bool(block, "is-anonymous-proxy"),
> 								self._parse_bool(block, "is-satellite-provider"),
> 								self._parse_bool(block, "is-anycast"),
> +								self._parse_bool(block, "drop"),
> 							)
> 
> 						else:
> diff --git a/src/python/location.in b/src/python/location.in
> index e02b4e8..0c89d75 100644
> --- a/src/python/location.in
> +++ b/src/python/location.in
> @@ -3,7 +3,7 @@
> #                                                                             #
> # libloc - A library to determine the location of someone on the Internet     #
> #                                                                             #
> -# Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org>                #
> +# Copyright (C) 2017-2021 IPFire Development Team <info(a)ipfire.org>           #
> #                                                                             #
> # This library is free software; you can redistribute it and/or               #
> # modify it under the terms of the GNU Lesser General Public                  #
> @@ -146,6 +146,9 @@ class CLI(object):
> 		list_networks_by_flags.add_argument("--anycast",
> 			action="store_true", help=_("Anycasts"),
> 		)
> +		list_networks_by_flags.add_argument("--drop",
> +			action="store_true", help=_("Hostile Networks safe to drop"),
> +		)
> 		list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
> 		list_networks_by_flags.add_argument("--format",
> 			choices=location.export.formats.keys(), default="list")
> @@ -305,6 +308,12 @@ class CLI(object):
> 					_("Anycast"), _("yes"),
> 				))
> 
> +			# Hostile Network
> +			if network.has_flag(location.NETWORK_FLAG_DROP):
> +				print(format % (
> +					_("Hostile Network safe to drop"), _("yes"),
> +				))
> +
> 		return ret
> 
> 	def handle_dump(self, db, ns):
> @@ -346,6 +355,7 @@ class CLI(object):
> 			location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
> 			location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
> 			location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
> +			location.NETWORK_FLAG_DROP               : "drop:",
> 		}
> 
> 		# Iterate over all networks
> @@ -523,6 +533,9 @@ class CLI(object):
> 		if ns.anycast:
> 			flags |= location.NETWORK_FLAG_ANYCAST
> 
> +		if ns.drop:
> +			flags |= location.NETWORK_FLAG_DROP
> +
> 		if not flags:
> 			raise ValueError(_("You must at least pass one flag"))
> 
> @@ -551,7 +564,7 @@ class CLI(object):
> 				asns.append(object)
> 
> 			elif location.country_code_is_valid(object) \
> -					or object in ("A1", "A2", "A3"):
> +					or object in ("A1", "A2", "A3", "XD"):
> 				countries.append(object)
> 
> 			else:
> @@ -560,7 +573,7 @@ class CLI(object):
> 
> 		# Default to exporting all countries
> 		if not countries and not asns:
> -			countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
> +			countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
> 
> 		# Select the output format
> 		writer = self.__get_output_formatter(ns)
> diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c
> index 5b72be9..5dd4ec6 100644
> --- a/src/python/locationmodule.c
> +++ b/src/python/locationmodule.c
> @@ -1,7 +1,7 @@
> /*
> 	libloc - A library to determine the location of someone on the Internet
> 
> -	Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org>
> +	Copyright (C) 2017-2021 IPFire Development Team <info(a)ipfire.org>
> 
> 	This library is free software; you can redistribute it and/or
> 	modify it under the terms of the GNU Lesser General Public
> @@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) {
> 	if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST))
> 		return NULL;
> 
> +	if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP))
> +		return NULL;
> +
> 	// Add latest database version
> 	if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST))
> 		return NULL;
> -- 
> 2.26.2