Hello, Thank you. Great work. Merged. -Michael > On 4 Jun 2021, at 16:57, Peter Müller wrote: > > This patch implements an additional flag intended for networks and > Autonomous Systems being considered hostile. While libloc does not and > should not be an opinionated database, reality shows it is being used > this way. > > Hereby, we assign "XD" (drop) as a custom country code for networks > being flagged this way. According to ISO, "XA" to "XZ" are reserved for > "user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html), > so this is a safe thing to do. > > This patch does not interfere with "A1" to "A3", which we currently > assign outside standardised country code ranges for historical reasons. > > Neither does it specify any policy or source for tagging networks with a > "drop" flag. Doing so is beyond the scope of this - technical - > approach. > > To avoid confusions with the SQL "DROP" command, "is_drop" will be used > as a column name for database operations. > > Thanks to Michael for his remarks and ideas during the run-up. > > Cc: Michael Tremer > Signed-off-by: Peter Müller > --- > man/location.txt | 4 ++-- > po/de.po | 3 +++ > src/loc/network.h | 3 ++- > src/perl/Location.xs | 2 ++ > src/python/export.py | 3 ++- > src/python/location-importer.in | 32 +++++++++++++++++++++++++++----- > src/python/location.in | 19 ++++++++++++++++--- > src/python/locationmodule.c | 5 ++++- > 8 files changed, 58 insertions(+), 13 deletions(-) > > diff --git a/man/location.txt b/man/location.txt > index df1da53..b38f21c 100644 > --- a/man/location.txt > +++ b/man/location.txt > @@ -10,7 +10,7 @@ location - Query the location database > `location list-countries [--show-name] [--show-continent]` > `location list-networks-by-as ASN` > `location list-networks-by-cc COUNTRY_CODE` > -`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` > +`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]` > `location lookup ADDRESS [ADDRESS...]` > `location search-as STRING` > `location update [--cron=daily|weekly|monthly]` > @@ -73,7 +73,7 @@ or countries. > + > See above for usage of the '--family' and '--format' parameters. > > -'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast]':: > +'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]':: > Lists all networks that have a certain flag. > + > See above for usage of the '--family' and '--format' parameters. > diff --git a/po/de.po b/po/de.po > index 3b073d6..3cbcdd7 100644 > --- a/po/de.po > +++ b/po/de.po > @@ -152,6 +152,9 @@ msgstr "" > msgid "Anycasts" > msgstr "" > > +msgid "Hostile Networks safe to drop" > +msgstr "" > + > msgid "Lists all countries" > msgstr "" > > diff --git a/src/loc/network.h b/src/loc/network.h > index af3dafd..a30f653 100644 > --- a/src/loc/network.h > +++ b/src/loc/network.h > @@ -1,7 +1,7 @@ > /* > libloc - A library to determine the location of someone on the Internet > > - Copyright (C) 2017 IPFire Development Team > + Copyright (C) 2017-2021 IPFire Development Team > > This library is free software; you can redistribute it and/or > modify it under the terms of the GNU Lesser General Public > @@ -27,6 +27,7 @@ enum loc_network_flags { > LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 > LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2 > LOC_NETWORK_FLAG_ANYCAST = (1 << 2), // A3 > + LOC_NETWORK_FLAG_DROP = (1 << 3), // XD > }; > > struct loc_network; > diff --git a/src/perl/Location.xs b/src/perl/Location.xs > index b7676d2..73f85b4 100644 > --- a/src/perl/Location.xs > +++ b/src/perl/Location.xs > @@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag) > iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER; > else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0) > iv |= LOC_NETWORK_FLAG_ANYCAST; > + else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0) > + iv |= LOC_NETWORK_FLAG_DROP; > else > croak("Invalid flag"); > > diff --git a/src/python/export.py b/src/python/export.py > index f0eae26..3b9e1e0 100644 > --- a/src/python/export.py > +++ b/src/python/export.py > @@ -3,7 +3,7 @@ > # # > # libloc - A library to determine the location of someone on the Internet # > # # > -# Copyright (C) 2020 IPFire Development Team # > +# Copyright (C) 2020-2021 IPFire Development Team # > # # > # This library is free software; you can redistribute it and/or # > # modify it under the terms of the GNU Lesser General Public # > @@ -33,6 +33,7 @@ FLAGS = { > _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", > _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", > _location.NETWORK_FLAG_ANYCAST : "A3", > + _location.NETWORK_FLAG_DROP : "XD", > } > > class OutputWriter(object): > diff --git a/src/python/location-importer.in b/src/python/location-importer.in > index f796652..ea1e8f5 100644 > --- a/src/python/location-importer.in > +++ b/src/python/location-importer.in > @@ -182,6 +182,7 @@ class CLI(object): > ); > CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number > ON autnum_overrides(number); > + ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; > > CREATE TABLE IF NOT EXISTS network_overrides( > network inet NOT NULL, > @@ -194,6 +195,7 @@ class CLI(object): > ON network_overrides(network); > CREATE INDEX IF NOT EXISTS network_overrides_search > ON network_overrides USING GIST(network inet_ops); > + ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; > """) > > return db > @@ -301,7 +303,20 @@ class CLI(object): > WHERE networks.autnum = overrides.number > ), > FALSE > - ) AS is_anycast > + ) AS is_anycast, > + COALESCE( > + ( > + SELECT is_drop FROM network_overrides overrides > + WHERE networks.network <<= overrides.network > + ORDER BY masklen(overrides.network) DESC > + LIMIT 1 > + ), > + ( > + SELECT is_drop FROM autnum_overrides overrides > + WHERE networks.autnum = overrides.number > + ), > + FALSE > + ) AS is_drop > FROM ( > SELECT > known_networks.network AS network, > @@ -350,6 +365,9 @@ class CLI(object): > if row.is_anycast: > network.set_flag(location.NETWORK_FLAG_ANYCAST) > > + if row.is_drop: > + network.set_flag(location.NETWORK_FLAG_DROP) > + > # Add all countries > log.info("Writing countries...") > rows = self.db.query("SELECT * FROM countries ORDER BY country_code") > @@ -966,14 +984,16 @@ class CLI(object): > country, > is_anonymous_proxy, > is_satellite_provider, > - is_anycast > - ) VALUES (%s, %s, %s, %s, %s) > + is_anycast, > + is_drop > + ) VALUES (%s, %s, %s, %s, %s, %s) > ON CONFLICT (network) DO NOTHING""", > "%s" % network, > block.get("country"), > self._parse_bool(block, "is-anonymous-proxy"), > self._parse_bool(block, "is-satellite-provider"), > self._parse_bool(block, "is-anycast"), > + self._parse_bool(block, "drop"), > ) > > elif type == "aut-num": > @@ -994,8 +1014,9 @@ class CLI(object): > country, > is_anonymous_proxy, > is_satellite_provider, > - is_anycast > - ) VALUES(%s, %s, %s, %s, %s, %s) > + is_anycast, > + is_drop > + ) VALUES(%s, %s, %s, %s, %s, %s, %s) > ON CONFLICT DO NOTHING""", > autnum, > block.get("name"), > @@ -1003,6 +1024,7 @@ class CLI(object): > self._parse_bool(block, "is-anonymous-proxy"), > self._parse_bool(block, "is-satellite-provider"), > self._parse_bool(block, "is-anycast"), > + self._parse_bool(block, "drop"), > ) > > else: > diff --git a/src/python/location.in b/src/python/location.in > index e02b4e8..0c89d75 100644 > --- a/src/python/location.in > +++ b/src/python/location.in > @@ -3,7 +3,7 @@ > # # > # libloc - A library to determine the location of someone on the Internet # > # # > -# Copyright (C) 2017 IPFire Development Team # > +# Copyright (C) 2017-2021 IPFire Development Team # > # # > # This library is free software; you can redistribute it and/or # > # modify it under the terms of the GNU Lesser General Public # > @@ -146,6 +146,9 @@ class CLI(object): > list_networks_by_flags.add_argument("--anycast", > action="store_true", help=_("Anycasts"), > ) > + list_networks_by_flags.add_argument("--drop", > + action="store_true", help=_("Hostile Networks safe to drop"), > + ) > list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4")) > list_networks_by_flags.add_argument("--format", > choices=location.export.formats.keys(), default="list") > @@ -305,6 +308,12 @@ class CLI(object): > _("Anycast"), _("yes"), > )) > > + # Hostile Network > + if network.has_flag(location.NETWORK_FLAG_DROP): > + print(format % ( > + _("Hostile Network safe to drop"), _("yes"), > + )) > + > return ret > > def handle_dump(self, db, ns): > @@ -346,6 +355,7 @@ class CLI(object): > location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:", > location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:", > location.NETWORK_FLAG_ANYCAST : "is-anycast:", > + location.NETWORK_FLAG_DROP : "drop:", > } > > # Iterate over all networks > @@ -523,6 +533,9 @@ class CLI(object): > if ns.anycast: > flags |= location.NETWORK_FLAG_ANYCAST > > + if ns.drop: > + flags |= location.NETWORK_FLAG_DROP > + > if not flags: > raise ValueError(_("You must at least pass one flag")) > > @@ -551,7 +564,7 @@ class CLI(object): > asns.append(object) > > elif location.country_code_is_valid(object) \ > - or object in ("A1", "A2", "A3"): > + or object in ("A1", "A2", "A3", "XD"): > countries.append(object) > > else: > @@ -560,7 +573,7 @@ class CLI(object): > > # Default to exporting all countries > if not countries and not asns: > - countries = ["A1", "A2", "A3"] + [country.code for country in db.countries] > + countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries] > > # Select the output format > writer = self.__get_output_formatter(ns) > diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c > index 5b72be9..5dd4ec6 100644 > --- a/src/python/locationmodule.c > +++ b/src/python/locationmodule.c > @@ -1,7 +1,7 @@ > /* > libloc - A library to determine the location of someone on the Internet > > - Copyright (C) 2017 IPFire Development Team > + Copyright (C) 2017-2021 IPFire Development Team > > This library is free software; you can redistribute it and/or > modify it under the terms of the GNU Lesser General Public > @@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) { > if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST)) > return NULL; > > + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP)) > + return NULL; > + > // Add latest database version > if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST)) > return NULL; > -- > 2.26.2