From mboxrd@z Thu Jan 1 00:00:00 1970 From: Michael Tremer To: location@lists.ipfire.org Subject: Re: [PATCH] Implement an additional flag for hostile networks safe to drop Date: Mon, 07 Jun 2021 18:04:16 +0100 Message-ID: <7282D115-0C5D-4D0B-B83D-E48EC43EE501@ipfire.org> In-Reply-To: MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============5309013904601121828==" List-Id: --===============5309013904601121828== Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Hello, Thank you. Great work. Merged. -Michael > On 4 Jun 2021, at 16:57, Peter M=C3=BCller wro= te: >=20 > This patch implements an additional flag intended for networks and > Autonomous Systems being considered hostile. While libloc does not and > should not be an opinionated database, reality shows it is being used > this way. >=20 > Hereby, we assign "XD" (drop) as a custom country code for networks > being flagged this way. According to ISO, "XA" to "XZ" are reserved for > "user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html), > so this is a safe thing to do. >=20 > This patch does not interfere with "A1" to "A3", which we currently > assign outside standardised country code ranges for historical reasons. >=20 > Neither does it specify any policy or source for tagging networks with a > "drop" flag. Doing so is beyond the scope of this - technical - > approach. >=20 > To avoid confusions with the SQL "DROP" command, "is_drop" will be used > as a column name for database operations. >=20 > Thanks to Michael for his remarks and ideas during the run-up. >=20 > Cc: Michael Tremer > Signed-off-by: Peter M=C3=BCller > --- > man/location.txt | 4 ++-- > po/de.po | 3 +++ > src/loc/network.h | 3 ++- > src/perl/Location.xs | 2 ++ > src/python/export.py | 3 ++- > src/python/location-importer.in | 32 +++++++++++++++++++++++++++----- > src/python/location.in | 19 ++++++++++++++++--- > src/python/locationmodule.c | 5 ++++- > 8 files changed, 58 insertions(+), 13 deletions(-) >=20 > diff --git a/man/location.txt b/man/location.txt > index df1da53..b38f21c 100644 > --- a/man/location.txt > +++ b/man/location.txt > @@ -10,7 +10,7 @@ location - Query the location database > `location list-countries [--show-name] [--show-continent]` > `location list-networks-by-as ASN` > `location list-networks-by-cc COUNTRY_CODE` > -`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|-= -anycast]` > +`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|-= -anycast|--drop]` > `location lookup ADDRESS [ADDRESS...]` > `location search-as STRING` > `location update [--cron=3Ddaily|weekly|monthly]` > @@ -73,7 +73,7 @@ or countries. > + > See above for usage of the '--family' and '--format' parameters. >=20 > -'list-networks-by-flags [--family=3D[ipv6|ipv4]] [--format=3DFORMAT] [--an= onymous-proxy|--satellite-provider|--anycast]':: > +'list-networks-by-flags [--family=3D[ipv6|ipv4]] [--format=3DFORMAT] [--an= onymous-proxy|--satellite-provider|--anycast|--drop]':: > Lists all networks that have a certain flag. > + > See above for usage of the '--family' and '--format' parameters. > diff --git a/po/de.po b/po/de.po > index 3b073d6..3cbcdd7 100644 > --- a/po/de.po > +++ b/po/de.po > @@ -152,6 +152,9 @@ msgstr "" > msgid "Anycasts" > msgstr "" >=20 > +msgid "Hostile Networks safe to drop" > +msgstr "" > + > msgid "Lists all countries" > msgstr "" >=20 > diff --git a/src/loc/network.h b/src/loc/network.h > index af3dafd..a30f653 100644 > --- a/src/loc/network.h > +++ b/src/loc/network.h > @@ -1,7 +1,7 @@ > /* > libloc - A library to determine the location of someone on the Internet >=20 > - Copyright (C) 2017 IPFire Development Team > + Copyright (C) 2017-2021 IPFire Development Team >=20 > This library is free software; you can redistribute it and/or > modify it under the terms of the GNU Lesser General Public > @@ -27,6 +27,7 @@ enum loc_network_flags { > LOC_NETWORK_FLAG_ANONYMOUS_PROXY =3D (1 << 0), // A1 > LOC_NETWORK_FLAG_SATELLITE_PROVIDER =3D (1 << 1), // A2 > LOC_NETWORK_FLAG_ANYCAST =3D (1 << 2), // A3 > + LOC_NETWORK_FLAG_DROP =3D (1 << 3), // XD > }; >=20 > struct loc_network; > diff --git a/src/perl/Location.xs b/src/perl/Location.xs > index b7676d2..73f85b4 100644 > --- a/src/perl/Location.xs > +++ b/src/perl/Location.xs > @@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag) > iv |=3D LOC_NETWORK_FLAG_SATELLITE_PROVIDER; > else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) =3D=3D 0) > iv |=3D LOC_NETWORK_FLAG_ANYCAST; > + else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) =3D=3D 0) > + iv |=3D LOC_NETWORK_FLAG_DROP; > else > croak("Invalid flag"); >=20 > diff --git a/src/python/export.py b/src/python/export.py > index f0eae26..3b9e1e0 100644 > --- a/src/python/export.py > +++ b/src/python/export.py > @@ -3,7 +3,7 @@ > # = # > # libloc - A library to determine the location of someone on the Internet = # > # = # > -# Copyright (C) 2020 IPFire Development Team = # > +# Copyright (C) 2020-2021 IPFire Development Team = # > # = # > # This library is free software; you can redistribute it and/or = # > # modify it under the terms of the GNU Lesser General Public = # > @@ -33,6 +33,7 @@ FLAGS =3D { > _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", > _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", > _location.NETWORK_FLAG_ANYCAST : "A3", > + _location.NETWORK_FLAG_DROP : "XD", > } >=20 > class OutputWriter(object): > diff --git a/src/python/location-importer.in b/src/python/location-importer= .in > index f796652..ea1e8f5 100644 > --- a/src/python/location-importer.in > +++ b/src/python/location-importer.in > @@ -182,6 +182,7 @@ class CLI(object): > ); > CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number > ON autnum_overrides(number); > + ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; >=20 > CREATE TABLE IF NOT EXISTS network_overrides( > network inet NOT NULL, > @@ -194,6 +195,7 @@ class CLI(object): > ON network_overrides(network); > CREATE INDEX IF NOT EXISTS network_overrides_search > ON network_overrides USING GIST(network inet_ops); > + ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; > """) >=20 > return db > @@ -301,7 +303,20 @@ class CLI(object): > WHERE networks.autnum =3D overrides.number > ), > FALSE > - ) AS is_anycast > + ) AS is_anycast, > + COALESCE( > + ( > + SELECT is_drop FROM network_overrides overrides > + WHERE networks.network <<=3D overrides.network > + ORDER BY masklen(overrides.network) DESC > + LIMIT 1 > + ), > + ( > + SELECT is_drop FROM autnum_overrides overrides > + WHERE networks.autnum =3D overrides.number > + ), > + FALSE > + ) AS is_drop > FROM ( > SELECT > known_networks.network AS network, > @@ -350,6 +365,9 @@ class CLI(object): > if row.is_anycast: > network.set_flag(location.NETWORK_FLAG_ANYCAST) >=20 > + if row.is_drop: > + network.set_flag(location.NETWORK_FLAG_DROP) > + > # Add all countries > log.info("Writing countries...") > rows =3D self.db.query("SELECT * FROM countries ORDER BY country_code") > @@ -966,14 +984,16 @@ class CLI(object): > country, > is_anonymous_proxy, > is_satellite_provider, > - is_anycast > - ) VALUES (%s, %s, %s, %s, %s) > + is_anycast, > + is_drop > + ) VALUES (%s, %s, %s, %s, %s, %s) > ON CONFLICT (network) DO NOTHING""", > "%s" % network, > block.get("country"), > self._parse_bool(block, "is-anonymous-proxy"), > self._parse_bool(block, "is-satellite-provider"), > self._parse_bool(block, "is-anycast"), > + self._parse_bool(block, "drop"), > ) >=20 > elif type =3D=3D "aut-num": > @@ -994,8 +1014,9 @@ class CLI(object): > country, > is_anonymous_proxy, > is_satellite_provider, > - is_anycast > - ) VALUES(%s, %s, %s, %s, %s, %s) > + is_anycast, > + is_drop > + ) VALUES(%s, %s, %s, %s, %s, %s, %s) > ON CONFLICT DO NOTHING""", > autnum, > block.get("name"), > @@ -1003,6 +1024,7 @@ class CLI(object): > self._parse_bool(block, "is-anonymous-proxy"), > self._parse_bool(block, "is-satellite-provider"), > self._parse_bool(block, "is-anycast"), > + self._parse_bool(block, "drop"), > ) >=20 > else: > diff --git a/src/python/location.in b/src/python/location.in > index e02b4e8..0c89d75 100644 > --- a/src/python/location.in > +++ b/src/python/location.in > @@ -3,7 +3,7 @@ > # = # > # libloc - A library to determine the location of someone on the Internet = # > # = # > -# Copyright (C) 2017 IPFire Development Team = # > +# Copyright (C) 2017-2021 IPFire Development Team = # > # = # > # This library is free software; you can redistribute it and/or = # > # modify it under the terms of the GNU Lesser General Public = # > @@ -146,6 +146,9 @@ class CLI(object): > list_networks_by_flags.add_argument("--anycast", > action=3D"store_true", help=3D_("Anycasts"), > ) > + list_networks_by_flags.add_argument("--drop", > + action=3D"store_true", help=3D_("Hostile Networks safe to drop"), > + ) > list_networks_by_flags.add_argument("--family", choices=3D("ipv6", "ipv4"= )) > list_networks_by_flags.add_argument("--format", > choices=3Dlocation.export.formats.keys(), default=3D"list") > @@ -305,6 +308,12 @@ class CLI(object): > _("Anycast"), _("yes"), > )) >=20 > + # Hostile Network > + if network.has_flag(location.NETWORK_FLAG_DROP): > + print(format % ( > + _("Hostile Network safe to drop"), _("yes"), > + )) > + > return ret >=20 > def handle_dump(self, db, ns): > @@ -346,6 +355,7 @@ class CLI(object): > location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:", > location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:", > location.NETWORK_FLAG_ANYCAST : "is-anycast:", > + location.NETWORK_FLAG_DROP : "drop:", > } >=20 > # Iterate over all networks > @@ -523,6 +533,9 @@ class CLI(object): > if ns.anycast: > flags |=3D location.NETWORK_FLAG_ANYCAST >=20 > + if ns.drop: > + flags |=3D location.NETWORK_FLAG_DROP > + > if not flags: > raise ValueError(_("You must at least pass one flag")) >=20 > @@ -551,7 +564,7 @@ class CLI(object): > asns.append(object) >=20 > elif location.country_code_is_valid(object) \ > - or object in ("A1", "A2", "A3"): > + or object in ("A1", "A2", "A3", "XD"): > countries.append(object) >=20 > else: > @@ -560,7 +573,7 @@ class CLI(object): >=20 > # Default to exporting all countries > if not countries and not asns: > - countries =3D ["A1", "A2", "A3"] + [country.code for country in db.coun= tries] > + countries =3D ["A1", "A2", "A3", "XD"] + [country.code for country in d= b.countries] >=20 > # Select the output format > writer =3D self.__get_output_formatter(ns) > diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c > index 5b72be9..5dd4ec6 100644 > --- a/src/python/locationmodule.c > +++ b/src/python/locationmodule.c > @@ -1,7 +1,7 @@ > /* > libloc - A library to determine the location of someone on the Internet >=20 > - Copyright (C) 2017 IPFire Development Team > + Copyright (C) 2017-2021 IPFire Development Team >=20 > This library is free software; you can redistribute it and/or > modify it under the terms of the GNU Lesser General Public > @@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) { > if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_AN= YCAST)) > return NULL; >=20 > + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP= )) > + return NULL; > + > // Add latest database version > if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VER= SION_LATEST)) > return NULL; > --=20 > 2.26.2 --===============5309013904601121828==--