Re: [PATCH] Implement an additional flag for hostile networks safe to drop

Hello,

Thank you. Great work. Merged.

-Michael

On 4 Jun 2021, at 16:57, Peter Müller peter.mueller@ipfire.org wrote:

This patch implements an additional flag intended for networks and Autonomous Systems being considered hostile. While libloc does not and should not be an opinionated database, reality shows it is being used this way.

Hereby, we assign "XD" (drop) as a custom country code for networks being flagged this way. According to ISO, "XA" to "XZ" are reserved for "user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html), so this is a safe thing to do.

This patch does not interfere with "A1" to "A3", which we currently assign outside standardised country code ranges for historical reasons.

Neither does it specify any policy or source for tagging networks with a "drop" flag. Doing so is beyond the scope of this - technical - approach.

To avoid confusions with the SQL "DROP" command, "is_drop" will be used as a column name for database operations.

Thanks to Michael for his remarks and ideas during the run-up.

Cc: Michael Tremer michael.tremer@ipfire.org Signed-off-by: Peter Müller peter.mueller@ipfire.org

---

man/location.txt | 4 ++-- po/de.po | 3 +++ src/loc/network.h | 3 ++- src/perl/Location.xs | 2 ++ src/python/export.py | 3 ++- src/python/location-importer.in | 32 +++++++++++++++++++++++++++----- src/python/location.in | 19 ++++++++++++++++--- src/python/locationmodule.c | 5 ++++- 8 files changed, 58 insertions(+), 13 deletions(-)

diff --git a/man/location.txt b/man/location.txt index df1da53..b38f21c 100644 --- a/man/location.txt +++ b/man/location.txt @@ -10,7 +10,7 @@ location - Query the location database `location list-countries [--show-name] [--show-continent]` `location list-networks-by-as ASN` `location list-networks-by-cc COUNTRY_CODE` -`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` +`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]` `location lookup ADDRESS [ADDRESS...]` `location search-as STRING` `location update [--cron=daily|weekly|monthly]` @@ -73,7 +73,7 @@ or countries.

See above for usage of the '--family' and '--format' parameters.

-'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast]':: +'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]':: Lists all networks that have a certain flag.

See above for usage of the '--family' and '--format' parameters. diff --git a/po/de.po b/po/de.po index 3b073d6..3cbcdd7 100644 --- a/po/de.po +++ b/po/de.po @@ -152,6 +152,9 @@ msgstr "" msgid "Anycasts" msgstr ""

+msgid "Hostile Networks safe to drop" +msgstr ""

msgid "Lists all countries" msgstr ""

diff --git a/src/loc/network.h b/src/loc/network.h index af3dafd..a30f653 100644 --- a/src/loc/network.h +++ b/src/loc/network.h @@ -1,7 +1,7 @@ /* libloc - A library to determine the location of someone on the Internet

• Copyright (C) 2017 IPFire Development Team info@ipfire.org

• Copyright (C) 2017-2021 IPFire Development Team info@ipfire.org

  This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public

@@ -27,6 +27,7 @@ enum loc_network_flags { LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2 LOC_NETWORK_FLAG_ANYCAST = (1 << 2), // A3

• LOC_NETWORK_FLAG_DROP = (1 << 3), // XD

};

struct loc_network; diff --git a/src/perl/Location.xs b/src/perl/Location.xs index b7676d2..73f85b4 100644 --- a/src/perl/Location.xs +++ b/src/perl/Location.xs @@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag) iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER; else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0) iv |= LOC_NETWORK_FLAG_ANYCAST;

• else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0)
  

• 	iv |= LOC_NETWORK_FLAG_DROP;
  

  else croak("Invalid flag");

diff --git a/src/python/export.py b/src/python/export.py index f0eae26..3b9e1e0 100644 --- a/src/python/export.py +++ b/src/python/export.py @@ -3,7 +3,7 @@ # # # libloc - A library to determine the location of someone on the Internet # # # -# Copyright (C) 2020 IPFire Development Team info@ipfire.org # +# Copyright (C) 2020-2021 IPFire Development Team info@ipfire.org # # # # This library is free software; you can redistribute it and/or # # modify it under the terms of the GNU Lesser General Public # @@ -33,6 +33,7 @@ FLAGS = { _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", _location.NETWORK_FLAG_ANYCAST : "A3",

• _location.NETWORK_FLAG_DROP : "XD",

}

class OutputWriter(object): diff --git a/src/python/location-importer.in b/src/python/location-importer.in index f796652..ea1e8f5 100644 --- a/src/python/location-importer.in +++ b/src/python/location-importer.in @@ -182,6 +182,7 @@ class CLI(object): ); CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number ON autnum_overrides(number);

• 		ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
  
  	CREATE TABLE IF NOT EXISTS network_overrides(
  		network inet NOT NULL,
  

@@ -194,6 +195,7 @@ class CLI(object): ON network_overrides(network); CREATE INDEX IF NOT EXISTS network_overrides_search ON network_overrides USING GIST(network inet_ops);

• 		ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
  """)
  
  

  return db

@@ -301,7 +303,20 @@ class CLI(object): WHERE networks.autnum = overrides.number ), FALSE

• 		) AS is_anycast
  

• 		) AS is_anycast,
  

• 		COALESCE(
  

• 			(
  

• 				SELECT is_drop FROM network_overrides overrides
  

• 					WHERE networks.network <<= overrides.network
  

• 					ORDER BY masklen(overrides.network) DESC
  

• 					LIMIT 1
  

• 			),
  

• 			(
  

• 				SELECT is_drop FROM autnum_overrides overrides
  

• 					WHERE networks.autnum = overrides.number
  

• 			),
  

• 			FALSE
  

• 		) AS is_drop
  FROM (
  	SELECT
  		known_networks.network AS network,
  

@@ -350,6 +365,9 @@ class CLI(object): if row.is_anycast: network.set_flag(location.NETWORK_FLAG_ANYCAST)

• 	if row.is_drop:
  

• 		network.set_flag(location.NETWORK_FLAG_DROP)
  

• # Add all countries log.info("Writing countries...") rows = self.db.query("SELECT * FROM countries ORDER BY country_code")

@@ -966,14 +984,16 @@ class CLI(object): country, is_anonymous_proxy, is_satellite_provider,

• 							is_anycast
  

• 						) VALUES (%s, %s, %s, %s, %s)
  

• 							is_anycast,
  

• 							is_drop
  

• 						) VALUES (%s, %s, %s, %s, %s, %s)
  					ON CONFLICT (network) DO NOTHING""",
  					"%s" % network,
  					block.get("country"),
  					self._parse_bool(block, "is-anonymous-proxy"),
  					self._parse_bool(block, "is-satellite-provider"),
  					self._parse_bool(block, "is-anycast"),
  

• 						self._parse_bool(block, "drop"),
  				)
  
  			elif type == "aut-num":
  

@@ -994,8 +1014,9 @@ class CLI(object): country, is_anonymous_proxy, is_satellite_provider,

• 							is_anycast
  

• 						) VALUES(%s, %s, %s, %s, %s, %s)
  

• 							is_anycast,
  

• 							is_drop
  

• 						) VALUES(%s, %s, %s, %s, %s, %s, %s)
  					ON CONFLICT DO NOTHING""",
  					autnum,
  					block.get("name"),
  

@@ -1003,6 +1024,7 @@ class CLI(object): self._parse_bool(block, "is-anonymous-proxy"), self._parse_bool(block, "is-satellite-provider"), self._parse_bool(block, "is-anycast"),

• 						self._parse_bool(block, "drop"),
  				)
  
  			else:
  

diff --git a/src/python/location.in b/src/python/location.in index e02b4e8..0c89d75 100644 --- a/src/python/location.in +++ b/src/python/location.in @@ -3,7 +3,7 @@ # # # libloc - A library to determine the location of someone on the Internet # # # -# Copyright (C) 2017 IPFire Development Team info@ipfire.org # +# Copyright (C) 2017-2021 IPFire Development Team info@ipfire.org # # # # This library is free software; you can redistribute it and/or # # modify it under the terms of the GNU Lesser General Public # @@ -146,6 +146,9 @@ class CLI(object): list_networks_by_flags.add_argument("--anycast", action="store_true", help=_("Anycasts"), )

• list_networks_by_flags.add_argument("--drop",
  

• 	action="store_true", help=_("Hostile Networks safe to drop"),
  

• )
  

  list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4")) list_networks_by_flags.add_argument("--format", choices=location.export.formats.keys(), default="list")

@@ -305,6 +308,12 @@ class CLI(object): _("Anycast"), _("yes"), ))

• 	# Hostile Network
  

• 	if network.has_flag(location.NETWORK_FLAG_DROP):
  

• 		print(format % (
  

• 			_("Hostile Network safe to drop"), _("yes"),
  

• 		))
  

• return ret

  def handle_dump(self, db, ns):

@@ -346,6 +355,7 @@ class CLI(object): location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:", location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:", location.NETWORK_FLAG_ANYCAST : "is-anycast:",

• 	location.NETWORK_FLAG_DROP               : "drop:",
  

  }

  # Iterate over all networks

@@ -523,6 +533,9 @@ class CLI(object): if ns.anycast: flags |= location.NETWORK_FLAG_ANYCAST

• if ns.drop:
  

• 	flags |= location.NETWORK_FLAG_DROP
  

• if not flags: raise ValueError(_("You must at least pass one flag"))

@@ -551,7 +564,7 @@ class CLI(object): asns.append(object)

	elif location.country_code_is_valid(object) \

• 			or object in ("A1", "A2", "A3"):
  

• 			or object in ("A1", "A2", "A3", "XD"):
  	countries.append(object)
  
  else:
  

@@ -560,7 +573,7 @@ class CLI(object):

# Default to exporting all countries
if not countries and not asns:

• 	countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
  

• 	countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
  
  

  # Select the output format writer = self.__get_output_formatter(ns)

diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c index 5b72be9..5dd4ec6 100644 --- a/src/python/locationmodule.c +++ b/src/python/locationmodule.c @@ -1,7 +1,7 @@ /* libloc - A library to determine the location of someone on the Internet

• Copyright (C) 2017 IPFire Development Team info@ipfire.org

• Copyright (C) 2017-2021 IPFire Development Team info@ipfire.org

  This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public

@@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) { if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST)) return NULL;

• if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP))

• return NULL;
  

• // Add latest database version if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST)) return NULL;

-- 2.26.2