This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "IPFire 2.x development tree".

The branch, next has been updated via 05db64d0ea36e3f4be816d5818ce755d7e02b9d1 (commit) from e704dbe6bd661385d19bcc874f95b2ba4dbcce46 (commit)

Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.

- Log ----------------------------------------------------------------- commit 05db64d0ea36e3f4be816d5818ce755d7e02b9d1 Author: Michael Tremer michael.tremer@ipfire.org Date: Wed Nov 18 13:30:15 2020 +0000

libloc: Import recent patches from upstream

Signed-off-by: Michael Tremer michael.tremer@ipfire.org

-----------------------------------------------------------------------

Summary of changes: config/rootfiles/common/libloc | 3 + lfs/libloc | 3 + src/patches/libloc-0.9.4-upstream.patch | 7629 +++++++++++++++++++++++++++++++ 3 files changed, 7635 insertions(+) create mode 100644 src/patches/libloc-0.9.4-upstream.patch

Difference in files: diff --git a/config/rootfiles/common/libloc b/config/rootfiles/common/libloc index b12052074..e8776e56e 100644 --- a/config/rootfiles/common/libloc +++ b/config/rootfiles/common/libloc @@ -1,12 +1,15 @@ usr/bin/location #usr/bin/location-importer #usr/include/libloc +#usr/include/libloc/as-list.h #usr/include/libloc/as.h #usr/include/libloc/compat.h +#usr/include/libloc/country-list.h #usr/include/libloc/country.h #usr/include/libloc/database.h #usr/include/libloc/format.h #usr/include/libloc/libloc.h +#usr/include/libloc/network-list.h #usr/include/libloc/network.h #usr/include/libloc/private.h #usr/include/libloc/resolv.h diff --git a/lfs/libloc b/lfs/libloc index da53e5149..65270eb24 100644 --- a/lfs/libloc +++ b/lfs/libloc @@ -78,6 +78,9 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) @$(PREBUILD) @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE)

+ # Import changes from upstream + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.4-upstream.patch + # Add patch for i586 to disable strong stack protector. ifeq "$(BUILD_ARCH)" "i586" cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.3-perl-i586-regular-stack-protector.patch diff --git a/src/patches/libloc-0.9.4-upstream.patch b/src/patches/libloc-0.9.4-upstream.patch new file mode 100644 index 000000000..28d569d93 --- /dev/null +++ b/src/patches/libloc-0.9.4-upstream.patch @@ -0,0 +1,7629 @@ +From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 24 Sep 2020 10:17:58 +0000 +Subject: [PATCH 01/70] Revert "importer: Purge any redundant entries" + +This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6. + +The query stalls the database and therefore the automatic +scripts are no longer able to generate a new version of the +database. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 22 +--------------------- + 1 file changed, 1 insertion(+), 21 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 1467923..e3a07a0 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -374,27 +374,7 @@ class CLI(object): + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums + JOIN _organizations ON _autnums.organization = _organizations.handle +- ON CONFLICT (number) DO UPDATE SET name = excluded.name +- """) +- +- self.db.execute(""" +- --- Purge any redundant entries +- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP +- AS +- SELECT network FROM networks candidates +- WHERE EXISTS ( +- SELECT FROM networks +- WHERE +- networks.network << candidates.network +- AND +- networks.country = candidates.country +- ); +- +- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); +- +- DELETE FROM networks WHERE EXISTS ( +- SELECT FROM _garbage WHERE networks.network = _garbage.network +- ); ++ ON CONFLICT (number) DO UPDATE SET name = excluded.name; + """) + + # Download all extended sources +-- +2.20.1 + +From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 24 Sep 2020 10:18:58 +0000 +Subject: [PATCH 02/70] Revert "importer: Import raw sources for inetnum's + again" + +This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 --------------------------------- + 2 files changed, 7 insertions(+), 70 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..de20f37 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e3a07a0..77952f2 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -393,10 +393,6 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + +- # inetnum +- if line.startswith("inet6num:") or line.startswith("inetnum:"): +- return self._parse_inetnum_block(block) +- + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -426,65 +422,6 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + +- def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") +- +- inetnum = {} +- for line in block: +- logging.debug(line) +- +- # Split line +- key, val = split_line(line) +- +- if key == "inetnum": +- start_address, delim, end_address = val.partition("-") +- +- # Strip any excess space +- start_address, end_address = start_address.rstrip(), end_address.strip() +- +- # Convert to IP address +- try: +- start_address = ipaddress.ip_address(start_address) +- end_address = ipaddress.ip_address(end_address) +- except ValueError: +- logging.warning("Could not parse line: %s" % line) +- return +- +- # Set prefix to default +- prefix = 32 +- +- # Count number of addresses in this subnet +- num_addresses = int(end_address) - int(start_address) +- if num_addresses: +- prefix -= math.log(num_addresses, 2) +- +- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) +- +- elif key == "inet6num": +- inetnum[key] = val +- +- elif key == "country": +- if val == "UNITED STATES": +- val = "US" +- +- inetnum[key] = val.upper() +- +- # Skip empty objects +- if not inetnum: +- return +- +- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) +- +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) +- return +- +- self.db.execute("INSERT INTO networks(network, country) \ +- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", +- "%s" % network, inetnum.get("country"), +- ) +- + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Mon, 12 Oct 2020 20:53:31 +0000 +Subject: [PATCH 03/70] Revert "Revert "importer: Import raw sources for + inetnum's again"" + +This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ + 2 files changed, 70 insertions(+), 7 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index de20f37..f19db4b 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 77952f2..e3a07a0 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -393,6 +393,10 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + ++ # inetnum ++ if line.startswith("inet6num:") or line.startswith("inetnum:"): ++ return self._parse_inetnum_block(block) ++ + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -422,6 +426,65 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + ++ def _parse_inetnum_block(self, block): ++ logging.debug("Parsing inetnum block:") ++ ++ inetnum = {} ++ for line in block: ++ logging.debug(line) ++ ++ # Split line ++ key, val = split_line(line) ++ ++ if key == "inetnum": ++ start_address, delim, end_address = val.partition("-") ++ ++ # Strip any excess space ++ start_address, end_address = start_address.rstrip(), end_address.strip() ++ ++ # Convert to IP address ++ try: ++ start_address = ipaddress.ip_address(start_address) ++ end_address = ipaddress.ip_address(end_address) ++ except ValueError: ++ logging.warning("Could not parse line: %s" % line) ++ return ++ ++ # Set prefix to default ++ prefix = 32 ++ ++ # Count number of addresses in this subnet ++ num_addresses = int(end_address) - int(start_address) ++ if num_addresses: ++ prefix -= math.log(num_addresses, 2) ++ ++ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) ++ ++ elif key == "inet6num": ++ inetnum[key] = val ++ ++ elif key == "country": ++ if val == "UNITED STATES": ++ val = "US" ++ ++ inetnum[key] = val.upper() ++ ++ # Skip empty objects ++ if not inetnum: ++ return ++ ++ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) ++ ++ # Bail out in case we have processed a non-public IP network ++ if network.is_private: ++ logging.warning("Skipping non-globally routable network: %s" % network) ++ return ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", ++ "%s" % network, inetnum.get("country"), ++ ) ++ + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Mon, 12 Oct 2020 20:53:32 +0000 +Subject: [PATCH 04/70] location-importer.in: only import relevant data from + AFRINIC, APNIC and RIPE +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +In contrast to ARIN and LACNIC, we are able to process more detailled +feeds from those RIRs, avoiding storage of obviously unnecessary data. + +Thanks to various SQL optimisations, doing so now takes less time than +the first version of this did. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- + 1 file changed, 87 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e3a07a0..093f325 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -165,6 +165,7 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); ++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); + + -- overrides +@@ -363,6 +364,16 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); ++ ++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) ++ ON COMMIT DROP; ++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); ++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); ++ """) ++ ++ # Remove all previously imported content ++ self.db.execute(""" ++ TRUNCATE TABLE networks; + """) + + for source in location.importer.WHOIS_SOURCES: +@@ -370,6 +381,67 @@ class CLI(object): + for block in f: + self._parse_block(block) + ++ # Process all parsed networks from every RIR we happen to have access to, ++ # insert the largest network chunks into the networks table immediately... ++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") ++ ++ for family in (row.family for row in families): ++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) ++ ++ # ... determine any other prefixes for this network family, ... ++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ ++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) ++ ++ # ... and insert networks with this prefix in case they provide additional ++ # information (i. e. subnet of a larger chunk with a different country) ++ for prefix in (row.prefix for row in prefixes): ++ self.db.execute(""" ++ WITH candidates AS ( ++ SELECT ++ _rirdata.network, ++ _rirdata.country ++ FROM ++ _rirdata ++ WHERE ++ family(_rirdata.network) = %s ++ AND ++ masklen(_rirdata.network) = %s ++ ), ++ filtered AS ( ++ SELECT ++ DISTINCT ON (c.network) ++ c.network, ++ c.country, ++ masklen(networks.network), ++ networks.country AS parent_country ++ FROM ++ candidates c ++ LEFT JOIN ++ networks ++ ON ++ c.network << networks.network ++ ORDER BY ++ c.network, ++ masklen(networks.network) DESC NULLS LAST ++ ) ++ INSERT INTO ++ networks(network, country) ++ SELECT ++ network, ++ country ++ FROM ++ filtered ++ WHERE ++ parent_country IS NULL ++ OR ++ country <> parent_country ++ ON CONFLICT DO NOTHING""", ++ family, prefix, ++ ) ++ + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums +@@ -470,17 +542,30 @@ class CLI(object): + inetnum[key] = val.upper() + + # Skip empty objects +- if not inetnum: ++ if not inetnum or not "country" in inetnum: + return + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + ++ # Bail out in case we have processed a network covering the entire IP range, which ++ # is necessary to work around faulty (?) IPv6 network processing ++ if network.prefixlen == 0: ++ logging.warning("Skipping network covering the entire IP adress range: %s" % network) ++ return ++ ++ # Bail out in case we have processed a network whose prefix length indicates it is ++ # not globally routable (we have decided not to process them at the moment, as they ++ # significantly enlarge our database without providing very helpful additional information) ++ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return ++ + # Bail out in case we have processed a non-public IP network + if network.is_private: + logging.warning("Skipping non-globally routable network: %s" % network) + return + +- self.db.execute("INSERT INTO networks(network, country) \ ++ self.db.execute("INSERT INTO _rirdata(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +-- +2.20.1 + +From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 16 Oct 2020 12:24:58 +0000 +Subject: [PATCH 05/70] location-importer: Include all overridden networks + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 093f325..d249a35 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -240,6 +240,8 @@ class CLI(object): + SELECT network FROM announcements + UNION + SELECT network FROM networks ++ UNION ++ SELECT network FROM network_overrides + ORDER BY network + ) + +-- +2.20.1 + +From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 16 Oct 2020 12:26:38 +0000 +Subject: [PATCH 06/70] Revert "location-importer.in: only import relevant data + from AFRINIC, APNIC and RIPE" + +This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 89 +-------------------------------- + 1 file changed, 2 insertions(+), 87 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index d249a35..b220eaf 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -165,7 +165,6 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); +- CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); + + -- overrides +@@ -366,16 +365,6 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); +- +- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) +- ON COMMIT DROP; +- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); +- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); +- """) +- +- # Remove all previously imported content +- self.db.execute(""" +- TRUNCATE TABLE networks; + """) + + for source in location.importer.WHOIS_SOURCES: +@@ -383,67 +372,6 @@ class CLI(object): + for block in f: + self._parse_block(block) + +- # Process all parsed networks from every RIR we happen to have access to, +- # insert the largest network chunks into the networks table immediately... +- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") +- +- for family in (row.family for row in families): +- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) +- +- self.db.execute("INSERT INTO networks(network, country) \ +- SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) +- +- # ... determine any other prefixes for this network family, ... +- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ +- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) +- +- # ... and insert networks with this prefix in case they provide additional +- # information (i. e. subnet of a larger chunk with a different country) +- for prefix in (row.prefix for row in prefixes): +- self.db.execute(""" +- WITH candidates AS ( +- SELECT +- _rirdata.network, +- _rirdata.country +- FROM +- _rirdata +- WHERE +- family(_rirdata.network) = %s +- AND +- masklen(_rirdata.network) = %s +- ), +- filtered AS ( +- SELECT +- DISTINCT ON (c.network) +- c.network, +- c.country, +- masklen(networks.network), +- networks.country AS parent_country +- FROM +- candidates c +- LEFT JOIN +- networks +- ON +- c.network << networks.network +- ORDER BY +- c.network, +- masklen(networks.network) DESC NULLS LAST +- ) +- INSERT INTO +- networks(network, country) +- SELECT +- network, +- country +- FROM +- filtered +- WHERE +- parent_country IS NULL +- OR +- country <> parent_country +- ON CONFLICT DO NOTHING""", +- family, prefix, +- ) +- + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums +@@ -544,30 +472,17 @@ class CLI(object): + inetnum[key] = val.upper() + + # Skip empty objects +- if not inetnum or not "country" in inetnum: ++ if not inetnum: + return + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + +- # Bail out in case we have processed a network covering the entire IP range, which +- # is necessary to work around faulty (?) IPv6 network processing +- if network.prefixlen == 0: +- logging.warning("Skipping network covering the entire IP adress range: %s" % network) +- return +- +- # Bail out in case we have processed a network whose prefix length indicates it is +- # not globally routable (we have decided not to process them at the moment, as they +- # significantly enlarge our database without providing very helpful additional information) +- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): +- logging.info("Skipping network too small to be publicly announced: %s" % network) +- return +- + # Bail out in case we have processed a non-public IP network + if network.is_private: + logging.warning("Skipping non-globally routable network: %s" % network) + return + +- self.db.execute("INSERT INTO _rirdata(network, country) \ ++ self.db.execute("INSERT INTO networks(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +-- +2.20.1 + +From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 16 Oct 2020 12:26:43 +0000 +Subject: [PATCH 07/70] Revert "Revert "Revert "importer: Import raw sources + for inetnum's again""" + +This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 --------------------------------- + 2 files changed, 7 insertions(+), 70 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..de20f37 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index b220eaf..e87d378 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -395,10 +395,6 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + +- # inetnum +- if line.startswith("inet6num:") or line.startswith("inetnum:"): +- return self._parse_inetnum_block(block) +- + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -428,65 +424,6 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + +- def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") +- +- inetnum = {} +- for line in block: +- logging.debug(line) +- +- # Split line +- key, val = split_line(line) +- +- if key == "inetnum": +- start_address, delim, end_address = val.partition("-") +- +- # Strip any excess space +- start_address, end_address = start_address.rstrip(), end_address.strip() +- +- # Convert to IP address +- try: +- start_address = ipaddress.ip_address(start_address) +- end_address = ipaddress.ip_address(end_address) +- except ValueError: +- logging.warning("Could not parse line: %s" % line) +- return +- +- # Set prefix to default +- prefix = 32 +- +- # Count number of addresses in this subnet +- num_addresses = int(end_address) - int(start_address) +- if num_addresses: +- prefix -= math.log(num_addresses, 2) +- +- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) +- +- elif key == "inet6num": +- inetnum[key] = val +- +- elif key == "country": +- if val == "UNITED STATES": +- val = "US" +- +- inetnum[key] = val.upper() +- +- # Skip empty objects +- if not inetnum: +- return +- +- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) +- +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) +- return +- +- self.db.execute("INSERT INTO networks(network, country) \ +- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", +- "%s" % network, inetnum.get("country"), +- ) +- + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 20 Oct 2020 20:44:43 +0000 +Subject: [PATCH 08/70] as: Fix dereferencing NULL pointer when setting AS name + +Reported-by: Gisle Vanem gisle.vanem@gmail.com +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/as.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/src/as.c b/src/as.c +index e1fbb01..8421ac8 100644 +--- a/src/as.c ++++ b/src/as.c +@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) { + } + + LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) { +- as->name = strdup(name); ++ if (as->name) ++ free(as->name); ++ ++ if (name) ++ as->name = strdup(name); ++ else ++ as->name = NULL; + + return 0; + } +-- +2.20.1 + +From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 09:18:08 +0000 +Subject: [PATCH 09/70] as: Do not attempt to match name when it wasn't set + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/as.c | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/src/as.c b/src/as.c +index 8421ac8..757bf3d 100644 +--- a/src/as.c ++++ b/src/as.c +@@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) { + if (!string) + return 1; + ++ // Cannot match anything when name is not set ++ if (!as->name) ++ return 1; ++ + // Search if string is in name + if (strcasestr(as->name, string) != NULL) + return 1; +-- +2.20.1 + +From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 09:28:39 +0000 +Subject: [PATCH 10/70] writer: Free array with pointer to ASes, too + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/writer.c | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/src/writer.c b/src/writer.c +index 5939cff..160650f 100644 +--- a/src/writer.c ++++ b/src/writer.c +@@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) { + EVP_PKEY_free(writer->private_key2); + + // Unref all AS +- for (unsigned int i = 0; i < writer->as_count; i++) { +- loc_as_unref(writer->as[i]); ++ if (writer->as) { ++ for (unsigned int i = 0; i < writer->as_count; i++) { ++ loc_as_unref(writer->as[i]); ++ } ++ free(writer->as); + } + + // Release network tree +-- +2.20.1 + +From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 09:31:29 +0000 +Subject: [PATCH 11/70] writer: Free countries when the writer is being + destroyed + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/writer.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/src/writer.c b/src/writer.c +index 160650f..2f09b56 100644 +--- a/src/writer.c ++++ b/src/writer.c +@@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) { + free(writer->as); + } + ++ // Unref all countries ++ if (writer->countries) { ++ for (unsigned int i = 0; i < writer->countries_count; i++) { ++ loc_country_unref(writer->countries[i]); ++ } ++ free(writer->countries); ++ } ++ + // Release network tree + if (writer->networks) + loc_network_tree_unref(writer->networks); +-- +2.20.1 + +From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 13:19:44 +0000 +Subject: [PATCH 12/70] tests: Try adding an invalid network + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/test-network.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/src/test-network.c b/src/test-network.c +index d38f13d..e908b57 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -160,6 +160,14 @@ int main(int argc, char** argv) { + // Set ASN + loc_network_set_asn(network4, 1024); + ++ // Try adding an invalid network ++ struct loc_network* network; ++ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +-- +2.20.1 + +From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 13:36:35 +0000 +Subject: [PATCH 13/70] networks: Improve parsing IP addresses + +loc_network_new_from_string() seem to have had some unexpected +behaviour for invalid inputs. + +The function has been tidied up slightly and returns as soon as +some invalid input was detected. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 52 +++++++++++++++++++++++++++++++--------------- + src/test-network.c | 7 +++++++ + 2 files changed, 42 insertions(+), 17 deletions(-) + +diff --git a/src/network.c b/src/network.c +index 366caa2..c112a41 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network + LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, + const char* address_string) { + struct in6_addr first_address; +- unsigned int prefix = 0; + char* prefix_string; +- int r = 1; ++ int r = -EINVAL; ++ ++ DEBUG(ctx, "Attempting to parse network %s\n", address_string); + + // Make a copy of the string to work on it + char* buffer = strdup(address_string); +@@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + // Split address and prefix + address_string = strsep(&prefix_string, "/"); + +- // Did we find a prefix? +- if (prefix_string) { +- // Convert prefix to integer +- prefix = strtol(prefix_string, NULL, 10); ++ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); + +- if (prefix) { +- // Parse the address +- r = loc_parse_address(ctx, address_string, &first_address); ++ // We need to have a prefix ++ if (!prefix_string) { ++ DEBUG(ctx, "No prefix set\n"); ++ goto FAIL; ++ } + +- // Map the prefix to IPv6 if needed +- if (IN6_IS_ADDR_V4MAPPED(&first_address)) +- prefix += 96; +- } ++ // Convert prefix to integer ++ unsigned int prefix = strtol(prefix_string, NULL, 10); ++ ++ // End if the prefix was invalid ++ if (!prefix) { ++ DEBUG(ctx, "The prefix is zero or not a number\n"); ++ goto FAIL; ++ } ++ ++ // Parse the address ++ r = loc_parse_address(ctx, address_string, &first_address); ++ if (r) { ++ DEBUG(ctx, "The address could not be parsed\n"); ++ goto FAIL; + } + ++ // Map the prefix to IPv6 if needed ++ if (IN6_IS_ADDR_V4MAPPED(&first_address)) ++ prefix += 96; ++ ++FAIL: + // Free temporary buffer + free(buffer); + +- if (r == 0) { +- r = loc_network_new(ctx, network, &first_address, prefix); +- } ++ // Exit if the parsing was unsuccessful ++ if (r) ++ return r; ++ ++ DEBUG(ctx, "GOT HERE\n"); + +- return r; ++ // Create a new network ++ return loc_network_new(ctx, network, &first_address, prefix); + } + + LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { +diff --git a/src/test-network.c b/src/test-network.c +index e908b57..85eca00 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -168,6 +168,13 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Try adding a single address ++ err = loc_writer_add_network(writer, &network, "2001:db8::"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +-- +2.20.1 + +From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 13:43:21 +0000 +Subject: [PATCH 14/70] networks: Test if we can add localhost (IPv6) + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/test-network.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/src/test-network.c b/src/test-network.c +index 85eca00..8c7e898 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -175,6 +175,13 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Try adding localhost ++ err = loc_writer_add_network(writer, &network, "::1/128"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +-- +2.20.1 + +From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 13:44:50 +0000 +Subject: [PATCH 15/70] networks: Remove accidentially committed debug line + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/src/network.c b/src/network.c +index c112a41..be88d75 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -208,8 +208,6 @@ FAIL: + if (r) + return r; + +- DEBUG(ctx, "GOT HERE\n"); +- + // Create a new network + return loc_network_new(ctx, network, &first_address, prefix); + } +-- +2.20.1 + +From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 13:53:36 +0000 +Subject: [PATCH 16/70] importer: Add search index to announcements table + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e87d378..d0fe5a6 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -152,6 +152,7 @@ class CLI(object): + last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); + CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); + CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); ++ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); + + -- autnums + CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); +-- +2.20.1 + +From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 13:54:45 +0000 +Subject: [PATCH 17/70] importer: Add search index to network_overrides table + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index d0fe5a6..fe21d73 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -189,6 +189,8 @@ class CLI(object): + ); + CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network + ON network_overrides(network); ++ CREATE INDEX IF NOT EXISTS network_overrides_search ++ ON network_overrides USING GIST(network inet_ops); + """) + + return db +-- +2.20.1 + +From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 21 Oct 2020 16:01:57 +0000 +Subject: [PATCH 18/70] importer: Restructure SQL query to be executed in + parallel + +There are no functional changes, this just runs quicker now. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 67 ++++++++++++++++++--------------- + 1 file changed, 37 insertions(+), 30 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index fe21d73..3c1e5e2 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -237,34 +237,24 @@ class CLI(object): + + # Select all known networks + rows = self.db.query(""" +- -- Get a (sorted) list of all known networks +- WITH known_networks AS ( +- SELECT network FROM announcements +- UNION +- SELECT network FROM networks +- UNION +- SELECT network FROM network_overrides +- ORDER BY network +- ) +- + -- Return a list of those networks enriched with all + -- other information that we store in the database + SELECT +- DISTINCT ON (known_networks.network) +- known_networks.network AS network, +- announcements.autnum AS autnum, ++ DISTINCT ON (network) ++ network, ++ autnum, + + -- Country + COALESCE( + ( + SELECT country FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT country FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + networks.country + ) AS country, +@@ -273,50 +263,67 @@ class CLI(object): + COALESCE( + ( + SELECT is_anonymous_proxy FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anonymous_proxy FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_anonymous_proxy, + COALESCE( + ( + SELECT is_satellite_provider FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_satellite_provider FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_satellite_provider, + COALESCE( + ( + SELECT is_anycast FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anycast FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE +- ) AS is_anycast, +- +- -- Must be part of returned values for ORDER BY clause +- masklen(announcements.network) AS sort_a, +- masklen(networks.network) AS sort_b +- FROM known_networks +- LEFT JOIN announcements ON known_networks.network <<= announcements.network +- LEFT JOIN networks ON known_networks.network <<= networks.network +- ORDER BY known_networks.network, sort_a DESC, sort_b DESC ++ ) AS is_anycast ++ FROM ( ++ SELECT ++ known_networks.network AS network, ++ announcements.autnum AS autnum, ++ networks.country AS country, ++ ++ -- Must be part of returned values for ORDER BY clause ++ masklen(announcements.network) AS sort_a, ++ masklen(networks.network) AS sort_b ++ FROM ( ++ SELECT network FROM announcements ++ UNION ALL ++ SELECT network FROM networks ++ UNION ALL ++ SELECT network FROM network_overrides ++ ) known_networks ++ LEFT JOIN ++ announcements ON known_networks.network <<= announcements.network ++ LEFT JOIN ++ networks ON known_networks.network <<= networks.network ++ ORDER BY ++ known_networks.network, ++ sort_a DESC, ++ sort_b DESC ++ ) networks + """) + + for row in rows: +-- +2.20.1 + +From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 22 Oct 2020 12:24:34 +0000 +Subject: [PATCH 19/70] network: Allow adding single IP addresses and + automatically add the prefix + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 33 +++++++++++++++------------------ + src/test-network.c | 4 ++-- + 2 files changed, 17 insertions(+), 20 deletions(-) + +diff --git a/src/network.c b/src/network.c +index be88d75..d7b1645 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + const char* address_string) { + struct in6_addr first_address; + char* prefix_string; ++ unsigned int prefix = 128; + int r = -EINVAL; + + DEBUG(ctx, "Attempting to parse network %s\n", address_string); +@@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); + +- // We need to have a prefix +- if (!prefix_string) { +- DEBUG(ctx, "No prefix set\n"); +- goto FAIL; +- } +- +- // Convert prefix to integer +- unsigned int prefix = strtol(prefix_string, NULL, 10); +- +- // End if the prefix was invalid +- if (!prefix) { +- DEBUG(ctx, "The prefix is zero or not a number\n"); +- goto FAIL; +- } +- + // Parse the address + r = loc_parse_address(ctx, address_string, &first_address); + if (r) { +@@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + goto FAIL; + } + +- // Map the prefix to IPv6 if needed +- if (IN6_IS_ADDR_V4MAPPED(&first_address)) +- prefix += 96; ++ // If a prefix was given, we will try to parse it ++ if (prefix_string) { ++ // Convert prefix to integer ++ prefix = strtol(prefix_string, NULL, 10); ++ ++ if (!prefix) { ++ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); ++ goto FAIL; ++ } ++ ++ // Map the prefix to IPv6 if needed ++ if (IN6_IS_ADDR_V4MAPPED(&first_address)) ++ prefix += 96; ++ } + + FAIL: + // Free temporary buffer +diff --git a/src/test-network.c b/src/test-network.c +index 8c7e898..b6776b4 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -170,8 +170,8 @@ int main(int argc, char** argv) { + + // Try adding a single address + err = loc_writer_add_network(writer, &network, "2001:db8::"); +- if (err != -EINVAL) { +- fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ if (err) { ++ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); + exit(EXIT_FAILURE); + } + +-- +2.20.1 + +From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Wed, 21 Oct 2020 14:47:36 +0000 +Subject: [PATCH 20/70] Revert "Revert "Revert "Revert "importer: Import raw + sources for inetnum's again"""" +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173. + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/importer.py | 14 ++++---- + src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ + 2 files changed, 70 insertions(+), 7 deletions(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index de20f37..f19db4b 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( + "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", + + # Asia Pacific Network Information Centre +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", +- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", ++ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", + #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", + "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", +@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( + # XXX ??? + + # Réseaux IP Européens +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", +- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", ++ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", +@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( + + EXTENDED_SOURCES = ( + # African Network Information Centre +- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", ++ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", + + # Asia Pacific Network Information Centre +- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", ++ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", + + # American Registry for Internet Numbers + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", +@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( + "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens +- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", ++ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", + ) + + class Downloader(object): +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 3c1e5e2..e8a4fc5 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -405,6 +405,10 @@ class CLI(object): + if line.startswith("aut-num:"): + return self._parse_autnum_block(block) + ++ # inetnum ++ if line.startswith("inet6num:") or line.startswith("inetnum:"): ++ return self._parse_inetnum_block(block) ++ + # organisation + elif line.startswith("organisation:"): + return self._parse_org_block(block) +@@ -434,6 +438,65 @@ class CLI(object): + autnum.get("asn"), autnum.get("org"), + ) + ++ def _parse_inetnum_block(self, block): ++ logging.debug("Parsing inetnum block:") ++ ++ inetnum = {} ++ for line in block: ++ logging.debug(line) ++ ++ # Split line ++ key, val = split_line(line) ++ ++ if key == "inetnum": ++ start_address, delim, end_address = val.partition("-") ++ ++ # Strip any excess space ++ start_address, end_address = start_address.rstrip(), end_address.strip() ++ ++ # Convert to IP address ++ try: ++ start_address = ipaddress.ip_address(start_address) ++ end_address = ipaddress.ip_address(end_address) ++ except ValueError: ++ logging.warning("Could not parse line: %s" % line) ++ return ++ ++ # Set prefix to default ++ prefix = 32 ++ ++ # Count number of addresses in this subnet ++ num_addresses = int(end_address) - int(start_address) ++ if num_addresses: ++ prefix -= math.log(num_addresses, 2) ++ ++ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) ++ ++ elif key == "inet6num": ++ inetnum[key] = val ++ ++ elif key == "country": ++ if val == "UNITED STATES": ++ val = "US" ++ ++ inetnum[key] = val.upper() ++ ++ # Skip empty objects ++ if not inetnum: ++ return ++ ++ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) ++ ++ # Bail out in case we have processed a non-public IP network ++ if network.is_private: ++ logging.warning("Skipping non-globally routable network: %s" % network) ++ return ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", ++ "%s" % network, inetnum.get("country"), ++ ) ++ + def _parse_org_block(self, block): + org = {} + for line in block: +-- +2.20.1 + +From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Wed, 21 Oct 2020 14:47:37 +0000 +Subject: [PATCH 21/70] Revert "Revert "location-importer.in: only import + relevant data from AFRINIC, APNIC and RIPE"" +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc. + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- + 1 file changed, 87 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index e8a4fc5..5656c41 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -166,6 +166,7 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); ++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); + + -- overrides +@@ -375,6 +376,16 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); ++ ++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) ++ ON COMMIT DROP; ++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); ++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); ++ """) ++ ++ # Remove all previously imported content ++ self.db.execute(""" ++ TRUNCATE TABLE networks; + """) + + for source in location.importer.WHOIS_SOURCES: +@@ -382,6 +393,67 @@ class CLI(object): + for block in f: + self._parse_block(block) + ++ # Process all parsed networks from every RIR we happen to have access to, ++ # insert the largest network chunks into the networks table immediately... ++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") ++ ++ for family in (row.family for row in families): ++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) ++ ++ self.db.execute("INSERT INTO networks(network, country) \ ++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) ++ ++ # ... determine any other prefixes for this network family, ... ++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ ++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) ++ ++ # ... and insert networks with this prefix in case they provide additional ++ # information (i. e. subnet of a larger chunk with a different country) ++ for prefix in (row.prefix for row in prefixes): ++ self.db.execute(""" ++ WITH candidates AS ( ++ SELECT ++ _rirdata.network, ++ _rirdata.country ++ FROM ++ _rirdata ++ WHERE ++ family(_rirdata.network) = %s ++ AND ++ masklen(_rirdata.network) = %s ++ ), ++ filtered AS ( ++ SELECT ++ DISTINCT ON (c.network) ++ c.network, ++ c.country, ++ masklen(networks.network), ++ networks.country AS parent_country ++ FROM ++ candidates c ++ LEFT JOIN ++ networks ++ ON ++ c.network << networks.network ++ ORDER BY ++ c.network, ++ masklen(networks.network) DESC NULLS LAST ++ ) ++ INSERT INTO ++ networks(network, country) ++ SELECT ++ network, ++ country ++ FROM ++ filtered ++ WHERE ++ parent_country IS NULL ++ OR ++ country <> parent_country ++ ON CONFLICT DO NOTHING""", ++ family, prefix, ++ ) ++ + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums +@@ -482,17 +554,30 @@ class CLI(object): + inetnum[key] = val.upper() + + # Skip empty objects +- if not inetnum: ++ if not inetnum or not "country" in inetnum: + return + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + ++ # Bail out in case we have processed a network covering the entire IP range, which ++ # is necessary to work around faulty (?) IPv6 network processing ++ if network.prefixlen == 0: ++ logging.warning("Skipping network covering the entire IP adress range: %s" % network) ++ return ++ ++ # Bail out in case we have processed a network whose prefix length indicates it is ++ # not globally routable (we have decided not to process them at the moment, as they ++ # significantly enlarge our database without providing very helpful additional information) ++ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return ++ + # Bail out in case we have processed a non-public IP network + if network.is_private: + logging.warning("Skipping non-globally routable network: %s" % network) + return + +- self.db.execute("INSERT INTO networks(network, country) \ ++ self.db.execute("INSERT INTO _rirdata(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +-- +2.20.1 + +From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Wed, 21 Oct 2020 14:47:38 +0000 +Subject: [PATCH 22/70] export.py: fix exporting IP networks for crappy + xt_geoip module +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +In contrast to the location database itself, the xt_geoip module +consumes a list of IP networks for each country, and returns after the +first match. + +We therefore need to... + +(a) sort IP networks by their size, allow as precise matches as possible +(b) export _any_ IP networks - including inverted subnets - to prevent + undefined overlaps +(c) do the entire thing as fast as possible, consuming as less disk + space as possible, which is why we can't just iterate over all /24 + chunks + +Partially fixes: #12499 + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/export.py | 69 ++++++++++++++++++++++++++++++++++---------- + 1 file changed, 54 insertions(+), 15 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index d15c6f0..5eaf43f 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -39,8 +39,8 @@ class OutputWriter(object): + suffix = "networks" + mode = "w" + +- def __init__(self, f, prefix=None, flatten=True): +- self.f, self.prefix, self.flatten = f, prefix, flatten ++ def __init__(self, db, f, prefix=None, flatten=True): ++ self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten + + # The previously written network + self._last_network = None +@@ -49,13 +49,13 @@ class OutputWriter(object): + self._write_header() + + @classmethod +- def open(cls, filename, **kwargs): ++ def open(cls, db, filename, **kwargs): + """ + Convenience function to open a file + """ + f = open(filename, cls.mode) + +- return cls(f, **kwargs) ++ return cls(db, f, **kwargs) + + def __repr__(self): + return "<%s f=%s>" % (self.__class__.__name__, self.f) +@@ -87,13 +87,31 @@ class OutputWriter(object): + def _write_network(self, network): + self.f.write("%s\n" % network) + +- def write(self, network): ++ def write(self, network, subnets): + if self.flatten and self._flatten(network): + log.debug("Skipping writing network %s" % network) + return + +- # Write the network to file +- self._write_network(network) ++ # Write the network when it has no subnets ++ if not subnets: ++ network = ipaddress.ip_network("%s" % network) ++ return self._write_network(network) ++ ++ # Collect all matching subnets ++ matching_subnets = [] ++ ++ for subnet in sorted(subnets): ++ # Try to find the subnet in the database ++ n = self.db.lookup("%s" % subnet.network_address) ++ ++ # No entry or matching country means those IP addresses belong here ++ if not n or n.country_code == network.country_code: ++ matching_subnets.append(subnet) ++ ++ # Write all networks as compact as possible ++ for network in ipaddress.collapse_addresses(matching_subnets): ++ log.debug("Writing %s to database" % network) ++ self._write_network(network) + + def finish(self): + """ +@@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter): + mode = "wb" + + def _write_network(self, network): +- for address in (network.first_address, network.last_address): ++ for address in (network.network_address, network.broadcast_address): + # Convert this into a string of bits + bytes = socket.inet_pton( +- network.family, address, ++ socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, + ) + + self.f.write(bytes) +@@ -175,7 +193,7 @@ class Exporter(object): + directory, prefix=country_code, suffix=self.writer.suffix, family=family, + ) + +- writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) ++ writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code) + + # Create writers for ASNs + for asn in asns: +@@ -183,22 +201,43 @@ class Exporter(object): + directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, + ) + +- writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) ++ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) + + # Get all networks that match the family + networks = self.db.search_networks(family=family) + ++ # Materialise the generator into a list (uses quite some memory) ++ networks = list(networks) ++ + # Walk through all networks +- for network in networks: ++ for i, network in enumerate(networks): ++ _network = ipaddress.ip_network("%s" % network) ++ ++ # Search for all subnets ++ subnets = set() ++ ++ while i < len(networks): ++ subnet = networks[i+1] ++ ++ if subnet.is_subnet_of(network): ++ _subnet = ipaddress.ip_network("%s" % subnet) ++ ++ subnets.add(_subnet) ++ subnets.update(_network.address_exclude(_subnet)) ++ ++ i += 1 ++ else: ++ break ++ + # Write matching countries + try: +- writers[network.country_code].write(network) ++ writers[network.country_code].write(network, subnets) + except KeyError: + pass + + # Write matching ASNs + try: +- writers[network.asn].write(network) ++ writers[network.asn].write(network, subnets) + except KeyError: + pass + +@@ -209,7 +248,7 @@ class Exporter(object): + country = flags[flag] + + try: +- writers[country].write(network) ++ writers[country].write(network, subnets) + except KeyError: + pass + +-- +2.20.1 + +From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Wed, 21 Oct 2020 14:47:39 +0000 +Subject: [PATCH 23/70] location-importer.in: filter bogus IP networks for both + Whois and extended sources +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Sanity checks for parsed networks have been put into a separate function +to avoid boilerplate code for extended sources. This makes the location +database less vulnerable to garbage written into RIR databases on +purpose or by chance. + +Fixes: #12500 + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 83 ++++++++++++++++++++++++++------- + 1 file changed, 67 insertions(+), 16 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 5656c41..f24d357 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -469,6 +469,69 @@ class CLI(object): + for line in f: + self._parse_line(line) + ++ def _check_parsed_network(self, network): ++ """ ++ Assistive function to detect and subsequently sort out parsed ++ networks from RIR data (both Whois and so-called "extended sources"), ++ which are or have... ++ ++ (a) not globally routable (RFC 1918 space, et al.) ++ (b) covering a too large chunk of the IP address space (prefix length ++ is < 7 for IPv4 networks, and < 10 for IPv6) ++ (c) "0.0.0.0" or "::" as a network address ++ (d) are too small for being publicly announced (we have decided not to ++ process them at the moment, as they significantly enlarge our ++ database without providing very helpful additional information) ++ ++ This unfortunately is necessary due to brain-dead clutter across ++ various RIR databases, causing mismatches and eventually disruptions. ++ ++ We will return False in case a network is not suitable for adding ++ it to our database, and True otherwise. ++ """ ++ ++ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): ++ return False ++ ++ if not network.is_global: ++ logging.warning("Skipping non-globally routable network: %s" % network) ++ return False ++ ++ if network.version == 4: ++ if network.prefixlen < 7: ++ logging.warning("Skipping too big IP chunk: %s" % network) ++ return False ++ ++ if network.prefixlen > 24: ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return False ++ ++ if str(network.network_address) == "0.0.0.0": ++ logging.warning("Skipping network based on 0.0.0.0: %s" % network) ++ return False ++ ++ elif network.version == 6: ++ if network.prefixlen < 10: ++ logging.warning("Skipping too big IP chunk: %s" % network) ++ return False ++ ++ if network.prefixlen > 48: ++ logging.info("Skipping network too small to be publicly announced: %s" % network) ++ return False ++ ++ if str(network.network_address) == "::": ++ logging.warning("Skipping network based on '::': %s" % network) ++ return False ++ ++ else: ++ # This should not happen... ++ logging.warning("Skipping network of unknown family, this should not happen: %s" % network) ++ return False ++ ++ # In case we have made it here, the network is considered to ++ # be suitable for libloc consumption... ++ return True ++ + def _parse_block(self, block): + # Get first line to find out what type of block this is + line = block[0] +@@ -559,22 +622,7 @@ class CLI(object): + + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + +- # Bail out in case we have processed a network covering the entire IP range, which +- # is necessary to work around faulty (?) IPv6 network processing +- if network.prefixlen == 0: +- logging.warning("Skipping network covering the entire IP adress range: %s" % network) +- return +- +- # Bail out in case we have processed a network whose prefix length indicates it is +- # not globally routable (we have decided not to process them at the moment, as they +- # significantly enlarge our database without providing very helpful additional information) +- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): +- logging.info("Skipping network too small to be publicly announced: %s" % network) +- return +- +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) ++ if not self._check_parsed_network(network): + return + + self.db.execute("INSERT INTO _rirdata(network, country) \ +@@ -658,6 +706,9 @@ class CLI(object): + log.warning("Invalid IP address: %s" % address) + return + ++ if not self._check_parsed_network(network): ++ return ++ + self.db.execute("INSERT INTO networks(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO \ + UPDATE SET country = excluded.country", +-- +2.20.1 + +From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Wed, 21 Oct 2020 14:47:40 +0000 +Subject: [PATCH 24/70] importer.py: fetch LACNIC data via HTTPS +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/importer.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..5f46bc3 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -64,7 +64,7 @@ EXTENDED_SOURCES = ( + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", + + # Latin America and Caribbean Network Information Centre +- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", ++ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", + + # Réseaux IP Européens + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", +-- +2.20.1 + +From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Wed, 21 Oct 2020 14:47:41 +0000 +Subject: [PATCH 25/70] location-importer.in: omit historic/orphaned RIR data +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Some RIRs include detailled information regarding networks not managed +by or allocated to themselves, particually APNIC. We need to filter +those networks (they usually have a characteristic network name) in +order to prevent operational quirks or returning wrong country codes. + +Fixes: #12501 +Partially fixes: #12499 + +Cc: Michael Tremer michael.tremer@ipfire.org +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 38 +++++++++++++++++++++------------ + 1 file changed, 24 insertions(+), 14 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index f24d357..a869256 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -494,38 +494,38 @@ class CLI(object): + return False + + if not network.is_global: +- logging.warning("Skipping non-globally routable network: %s" % network) ++ log.warning("Skipping non-globally routable network: %s" % network) + return False + + if network.version == 4: + if network.prefixlen < 7: +- logging.warning("Skipping too big IP chunk: %s" % network) ++ log.warning("Skipping too big IP chunk: %s" % network) + return False + + if network.prefixlen > 24: +- logging.info("Skipping network too small to be publicly announced: %s" % network) ++ log.info("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "0.0.0.0": +- logging.warning("Skipping network based on 0.0.0.0: %s" % network) ++ log.warning("Skipping network based on 0.0.0.0: %s" % network) + return False + + elif network.version == 6: + if network.prefixlen < 10: +- logging.warning("Skipping too big IP chunk: %s" % network) ++ log.warning("Skipping too big IP chunk: %s" % network) + return False + + if network.prefixlen > 48: +- logging.info("Skipping network too small to be publicly announced: %s" % network) ++ log.info("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "::": +- logging.warning("Skipping network based on '::': %s" % network) ++ log.warning("Skipping network based on '::': %s" % network) + return False + + else: + # This should not happen... +- logging.warning("Skipping network of unknown family, this should not happen: %s" % network) ++ log.warning("Skipping network of unknown family, this should not happen: %s" % network) + return False + + # In case we have made it here, the network is considered to +@@ -574,15 +574,22 @@ class CLI(object): + ) + + def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") ++ log.debug("Parsing inetnum block:") + + inetnum = {} + for line in block: +- logging.debug(line) ++ log.debug(line) + + # Split line + key, val = split_line(line) + ++ # Filter any inetnum records which are only referring to IP space ++ # not managed by that specific RIR... ++ if key == "netname": ++ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): ++ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) ++ return ++ + if key == "inetnum": + start_address, delim, end_address = val.partition("-") + +@@ -594,7 +601,7 @@ class CLI(object): + start_address = ipaddress.ip_address(start_address) + end_address = ipaddress.ip_address(end_address) + except ValueError: +- logging.warning("Could not parse line: %s" % line) ++ log.warning("Could not parse line: %s" % line) + return + + # Set prefix to default +@@ -611,15 +618,18 @@ class CLI(object): + inetnum[key] = val + + elif key == "country": +- if val == "UNITED STATES": +- val = "US" +- + inetnum[key] = val.upper() + + # Skip empty objects + if not inetnum or not "country" in inetnum: + return + ++ # Skip objects with bogus country code 'ZZ' ++ if inetnum.get("country") == "ZZ": ++ log.warning("Skipping network with bogus country 'ZZ': %s" % \ ++ (inetnum.get("inet6num") or inetnum.get("inetnum"))) ++ return ++ + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) + + if not self._check_parsed_network(network): +-- +2.20.1 + +From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Wed, 21 Oct 2020 14:47:43 +0000 +Subject: [PATCH 26/70] location-importer.in: avoid log spam for too small + networks +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index a869256..864eab1 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -503,7 +503,7 @@ class CLI(object): + return False + + if network.prefixlen > 24: +- log.info("Skipping network too small to be publicly announced: %s" % network) ++ log.debug("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "0.0.0.0": +@@ -516,7 +516,7 @@ class CLI(object): + return False + + if network.prefixlen > 48: +- log.info("Skipping network too small to be publicly announced: %s" % network) ++ log.debug("Skipping network too small to be publicly announced: %s" % network) + return False + + if str(network.network_address) == "::": +-- +2.20.1 + +From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 27 Oct 2020 17:14:30 +0000 +Subject: [PATCH 27/70] export: Flatten the tree before exporting it + +This patch removes the possibility that any IP address ranges +might show up in multiple exported files. + +If this was content from the database: + + * 10.0.0.0/16 - DE + * 10.0.1.0/24 - FR + +Then the IP address 10.0.1.1 would match for both countries. + +The algorithm will now break the larger /16 subnet apart into +smaller subnets so that 10.0.1.0/24 is no longer overlapped. + +There was some time spent on this to make this as efficient +as possible. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/export.py | 154 ++++++++++++++++++++++++++++++------------- + 1 file changed, 110 insertions(+), 44 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index 5eaf43f..dd44332 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -89,28 +89,55 @@ class OutputWriter(object): + + def write(self, network, subnets): + if self.flatten and self._flatten(network): +- log.debug("Skipping writing network %s" % network) ++ log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) + return + ++ # Convert network into a Python object ++ _network = ipaddress.ip_network("%s" % network) ++ + # Write the network when it has no subnets + if not subnets: +- network = ipaddress.ip_network("%s" % network) +- return self._write_network(network) ++ log.debug("Writing %s to %s" % (_network, self.f)) ++ return self._write_network(_network) ++ ++ # Convert subnets into Python objects ++ _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] ++ ++ # Split the network into smaller bits so that ++ # we can accomodate for any gaps in it later ++ to_check = set() ++ for _subnet in _subnets: ++ to_check.update( ++ _network.address_exclude(_subnet) ++ ) ++ ++ # Clear the list of all subnets ++ subnets = [] ++ ++ # Check if all subnets to not overlap with anything else ++ while to_check: ++ subnet_to_check = to_check.pop() + +- # Collect all matching subnets +- matching_subnets = [] ++ for _subnet in _subnets: ++ # Drop this subnet if it equals one of the subnets ++ # or if it is subnet of one of them ++ if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): ++ break + +- for subnet in sorted(subnets): +- # Try to find the subnet in the database +- n = self.db.lookup("%s" % subnet.network_address) ++ # Break it down if it overlaps ++ if subnet_to_check.overlaps(_subnet): ++ to_check.update( ++ subnet_to_check.address_exclude(_subnet) ++ ) ++ break + +- # No entry or matching country means those IP addresses belong here +- if not n or n.country_code == network.country_code: +- matching_subnets.append(subnet) ++ # Add the subnet again as it passed the check ++ else: ++ subnets.append(subnet_to_check) + + # Write all networks as compact as possible +- for network in ipaddress.collapse_addresses(matching_subnets): +- log.debug("Writing %s to database" % network) ++ for network in ipaddress.collapse_addresses(subnets): ++ log.debug("Writing %s to %s" % (network, self.f)) + self._write_network(network) + + def finish(self): +@@ -206,40 +233,40 @@ class Exporter(object): + # Get all networks that match the family + networks = self.db.search_networks(family=family) + +- # Materialise the generator into a list (uses quite some memory) +- networks = list(networks) ++ # Create a stack with all networks in order where we can put items back ++ # again and retrieve them in the next iteration. ++ networks = BufferedStack(networks) + + # Walk through all networks +- for i, network in enumerate(networks): +- _network = ipaddress.ip_network("%s" % network) +- +- # Search for all subnets +- subnets = set() +- +- while i < len(networks): +- subnet = networks[i+1] +- +- if subnet.is_subnet_of(network): +- _subnet = ipaddress.ip_network("%s" % subnet) +- +- subnets.add(_subnet) +- subnets.update(_network.address_exclude(_subnet)) +- +- i += 1 +- else: ++ for network in networks: ++ # Collect all networks which are a subnet of network ++ subnets = [] ++ for subnet in networks: ++ # If the next subnet was not a subnet, we have to push ++ # it back on the stack and break this loop ++ if not subnet.is_subnet_of(network): ++ networks.push(subnet) + break + ++ subnets.append(subnet) ++ + # Write matching countries +- try: +- writers[network.country_code].write(network, subnets) +- except KeyError: +- pass ++ if network.country_code and network.country_code in writers: ++ # Mismatching subnets ++ gaps = [ ++ subnet for subnet in subnets if not network.country_code == subnet.country_code ++ ] ++ ++ writers[network.country_code].write(network, gaps) + + # Write matching ASNs +- try: +- writers[network.asn].write(network, subnets) +- except KeyError: +- pass ++ if network.asn and network.asn in writers: ++ # Mismatching subnets ++ gaps = [ ++ subnet for subnet in subnets if not network.asn == subnet.asn ++ ] ++ ++ writers[network.asn].write(network, gaps) + + # Handle flags + for flag in flags: +@@ -247,10 +274,19 @@ class Exporter(object): + # Fetch the "fake" country code + country = flags[flag] + +- try: +- writers[country].write(network, subnets) +- except KeyError: +- pass ++ if not country in writers: ++ continue ++ ++ gaps = [ ++ subnet for subnet in subnets ++ if not subnet.has_flag(flag) ++ ] ++ ++ writers[country].write(network, gaps) ++ ++ # Push all subnets back onto the stack ++ for subnet in reversed(subnets): ++ networks.push(subnet) + + # Write everything to the filesystem + for writer in writers.values(): +@@ -262,3 +298,33 @@ class Exporter(object): + ) + + return os.path.join(directory, filename) ++ ++ ++class BufferedStack(object): ++ """ ++ This class takes an iterator and when being iterated ++ over it returns objects from that iterator for as long ++ as there are any. ++ ++ It additionally has a function to put an item back on ++ the back so that it will be returned again at the next ++ iteration. ++ """ ++ def __init__(self, iterator): ++ self.iterator = iterator ++ self.stack = [] ++ ++ def __iter__(self): ++ return self ++ ++ def __next__(self): ++ if self.stack: ++ return self.stack.pop(0) ++ ++ return next(self.iterator) ++ ++ def push(self, elem): ++ """ ++ Takes an element and puts it on the stack ++ """ ++ self.stack.insert(0, elem) +-- +2.20.1 + +From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 28 Oct 2020 09:52:36 +0000 +Subject: [PATCH 28/70] location: Fix Python syntax error in verify() + +The database is now being opened before the requested +method is called and handle_verify() wasn't updated. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location.in | 8 +------- + 1 file changed, 1 insertion(+), 7 deletions(-) + +diff --git a/src/python/location.in b/src/python/location.in +index 44ad726..b5e5758 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -453,13 +453,7 @@ class CLI(object): + + return 0 + +- def handle_verify(self, ns): +- try: +- db = location.Database(ns.database) +- except FileNotFoundError as e: +- log.error("%s: %s" % (ns.database, e)) +- return 127 +- ++ def handle_verify(self, db, ns): + # Verify the database + with open(ns.public_key, "r") as f: + if not db.verify(f): +-- +2.20.1 + +From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Thu, 29 Oct 2020 07:25:53 -0700 +Subject: [PATCH 29/70] location update: Remove double conversion of timestamps +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This caused that the timestamp in DNS was misinterpreted +as local time and often, databases could not be downloaded. + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +--- + src/python/location.in | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +diff --git a/src/python/location.in b/src/python/location.in +index b5e5758..070640c 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -421,11 +421,8 @@ class CLI(object): + # Fetch the timestamp we need from DNS + t = location.discover_latest_version() + +- # Parse timestamp into datetime format +- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None +- + # Check the version of the local database +- if db and timestamp and db.created_at >= timestamp.timestamp(): ++ if db and t and db.created_at >= t: + log.info("Already on the latest version") + return + +-- +2.20.1 + +From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Thu, 29 Oct 2020 07:36:46 -0700 +Subject: [PATCH 30/70] location.in: do not confuse UTC with local time zones +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +--- + src/python/location.in | 12 +++++------- + 1 file changed, 5 insertions(+), 7 deletions(-) + +diff --git a/src/python/location.in b/src/python/location.in +index 070640c..0d09210 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -398,10 +398,7 @@ class CLI(object): + + def handle_update(self, db, ns): + if ns.cron and db: +- now = datetime.datetime.utcnow() +- +- # Parse the database timestamp +- t = datetime.datetime.utcfromtimestamp(db.created_at) ++ now = time.time() + + if ns.cron == "daily": + delta = datetime.timedelta(days=1) +@@ -410,11 +407,12 @@ class CLI(object): + elif ns.cron == "monthly": + delta = datetime.timedelta(days=30) + ++ delta = delta.total_seconds() ++ + # Check if the database has recently been updated +- if t >= (now - delta): ++ if db.created_at >= (now - delta): + log.info( +- _("The database has been updated recently (%s)") % \ +- format_timedelta(now - t), ++ _("The database has been updated recently"), + ) + return 3 + +-- +2.20.1 + +From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org +Date: Tue, 3 Nov 2020 15:31:08 +0000 +Subject: [PATCH 31/70] location-importer.in: always convert organisation + handles into upper cases +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fixes: #12523 + +Signed-off-by: Peter Müller peter.mueller@ipfire.org +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location-importer.in | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 864eab1..2dec89e 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -560,7 +560,7 @@ class CLI(object): + autnum["asn"] = m.group(2) + + elif key == "org": +- autnum[key] = val ++ autnum[key] = val.upper() + + # Skip empty objects + if not autnum: +@@ -646,7 +646,9 @@ class CLI(object): + # Split line + key, val = split_line(line) + +- if key in ("organisation", "org-name"): ++ if key == "organisation": ++ org[key] = val.upper() ++ elif key == "org-name": + org[key] = val + + # Skip empty objects +-- +2.20.1 + +From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 11 Nov 2020 21:16:45 +0000 +Subject: [PATCH 32/70] test: Add tests for database enumerator + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 53 insertions(+) + +diff --git a/src/test-database.c b/src/test-database.c +index b4a75c4..4aef94e 100644 +--- a/src/test-database.c ++++ b/src/test-database.c +@@ -38,6 +38,14 @@ const char* DESCRIPTION = + "Maecenas ut venenatis nunc."; + const char* LICENSE = "CC"; + ++const char* networks[] = { ++ "2001:db8::/32", ++ "2001:db8:1000::/48", ++ "2001:db8:2000::/48", ++ "2001:db8:2020::/48", ++ NULL, ++}; ++ + static int attempt_to_open(struct loc_ctx* ctx, char* path) { + FILE* f = fopen(path, "r"); + if (!f) +@@ -139,6 +147,24 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ struct loc_network* network = NULL; ++ ++ // Add some networks ++ const char** n = networks; ++ while (*n) { ++ err = loc_writer_add_network(writer, &network, *n); ++ if (err) { ++ fprintf(stderr, "Could not add network %s\n", *n); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Set a country ++ loc_network_set_country_code(network, "XX"); ++ ++ // Next one ++ n++; ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +@@ -170,6 +196,33 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Enumerator ++ struct loc_database_enumerator* enumerator; ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); ++ if (err) { ++ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Walk through all networks ++ while (1) { ++ err = loc_database_enumerator_next_network(enumerator, &network); ++ if (err) { ++ fprintf(stderr, "Error fetching the next network: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (!network) ++ break; ++ ++ char* s = loc_network_str(network); ++ printf("Got network: %s\n", s); ++ free(s); ++ } ++ ++ // Free the enumerator ++ loc_database_enumerator_unref(enumerator); ++ + // Close the database + loc_database_unref(db); + loc_unref(ctx); +-- +2.20.1 + +From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 11 Nov 2020 23:01:19 +0000 +Subject: [PATCH 33/70] networks: Add list to manage groups of networks + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 11 ++++++ + src/loc/network.h | 12 ++++++ + src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 119 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index b8296eb..6ef2d27 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -98,6 +98,17 @@ global: + loc_network_str; + loc_network_unref; + ++ # Network List ++ loc_network_list_clear; ++ loc_network_list_empty; ++ loc_network_list_get; ++ loc_network_list_new; ++ loc_network_list_pop; ++ loc_network_list_push; ++ loc_network_list_ref; ++ loc_network_list_size; ++ loc_network_list_unref; ++ + # Writer + loc_writer_add_as; + loc_writer_add_country; +diff --git a/src/loc/network.h b/src/loc/network.h +index 70c3803..fd20812 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); + ++// List ++struct loc_network_list; ++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); ++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); ++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); ++size_t loc_network_list_size(struct loc_network_list* list); ++int loc_network_list_empty(struct loc_network_list* list); ++void loc_network_list_clear(struct loc_network_list* list); ++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); ++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); ++struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++ + #ifdef LIBLOC_PRIVATE + + int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); +diff --git a/src/network.c b/src/network.c +index d7b1645..c9e7979 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) + LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } ++ ++// List ++ ++struct loc_network_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_network* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, ++ struct loc_network_list** list) { ++ struct loc_network_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "Network list allocated at %p\n", l); ++ *list = l; ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_network_list_free(struct loc_network_list* list) { ++ DEBUG(list->ctx, "Releasing network list at %p\n", list); ++ ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_network_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ list->size = 0; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_network_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ // Check if we have space left ++ if (list->size == list->max_size) ++ return -ENOMEM; ++ ++ list->list[list->size++] = loc_network_ref(network); ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) ++ return NULL; ++ ++ return list->list[list->size--]; ++} +-- +2.20.1 + +From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 13:57:35 +0000 +Subject: [PATCH 34/70] networks: Add function to dump lists + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 14 ++++++++++++++ + 3 files changed, 16 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index 6ef2d27..a5641c6 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -100,6 +100,7 @@ global: + + # Network List + loc_network_list_clear; ++ loc_network_list_dump; + loc_network_list_empty; + loc_network_list_get; + loc_network_list_new; +diff --git a/src/loc/network.h b/src/loc/network.h +index fd20812..44c50a4 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); + size_t loc_network_list_size(struct loc_network_list* list); + int loc_network_list_empty(struct loc_network_list* list); + void loc_network_list_clear(struct loc_network_list* list); ++void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); +diff --git a/src/network.c b/src/network.c +index c9e7979..0977406 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { + list->size = 0; + } + ++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { ++ struct loc_network* network; ++ char* s; ++ ++ for (unsigned int i = 0; i < list->size; i++) { ++ network = list->list[i]; ++ ++ s = loc_network_str(network); ++ ++ INFO(list->ctx, "%s\n", s); ++ free(s); ++ } ++} ++ + LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { + // Check index + if (index >= list->size) +-- +2.20.1 + +From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 14:18:40 +0000 +Subject: [PATCH 35/70] network: Add functions to break network into subnets + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 5 + + src/loc/network.h | 6 + + src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++- + src/test-network.c | 61 ++++++++++ + 4 files changed, 361 insertions(+), 1 deletion(-) + +diff --git a/src/libloc.sym b/src/libloc.sym +index a5641c6..fcb9ea5 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -80,6 +80,8 @@ global: + + # Network + loc_network_address_family; ++ loc_network_eq; ++ loc_network_exclude; + loc_network_format_first_address; + loc_network_format_last_address; + loc_network_get_asn; +@@ -96,6 +98,7 @@ global: + loc_network_set_country_code; + loc_network_set_flag; + loc_network_str; ++ loc_network_subnets; + loc_network_unref; + + # Network List +@@ -107,7 +110,9 @@ global: + loc_network_list_pop; + loc_network_list_push; + loc_network_list_ref; ++ loc_network_list_reverse; + loc_network_list_size; ++ loc_network_list_sort; + loc_network_list_unref; + + # Writer +diff --git a/src/loc/network.h b/src/loc/network.h +index 44c50a4..4e51a62 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); + int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + ++int loc_network_eq(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); ++struct loc_network_list* loc_network_subnets(struct loc_network* network); ++struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other); + + // List + struct loc_network_list; +@@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++void loc_network_list_sort(struct loc_network_list* list); ++void loc_network_list_reverse(struct loc_network_list* list); + + #ifdef LIBLOC_PRIVATE + +diff --git a/src/network.c b/src/network.c +index 0977406..6c08070 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s + return a; + } + ++static struct in6_addr address_increment(const struct in6_addr* address) { ++ struct in6_addr a = *address; ++ ++ for (int octet = 15; octet >= 0; octet--) { ++ if (a.s6_addr[octet] < 255) { ++ a.s6_addr[octet]++; ++ break; ++ } else { ++ a.s6_addr[octet] = 0; ++ } ++ } ++ ++ return a; ++} ++ + LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* address, unsigned int prefix) { + // Address cannot be unspecified +@@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag + return loc_network_has_flag(network, flag); + } + ++LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { ++#ifdef ENABLE_DEBUG ++ char* n1 = loc_network_str(self); ++ char* n2 = loc_network_str(other); ++ ++ DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); ++ ++ free(n1); ++ free(n2); ++#endif ++ ++ // Family must be the same ++ if (self->family != other->family) { ++ DEBUG(self->ctx, " Family mismatch\n"); ++ ++ return 0; ++ } ++ ++ // The start address must be the same ++ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { ++ DEBUG(self->ctx, " Address mismatch\n"); ++ ++ return 0; ++ } ++ ++ // The prefix length must be the same ++ if (self->prefix != other->prefix) { ++ DEBUG(self->ctx, " Prefix mismatch\n"); ++ return 0; ++ } ++ ++ DEBUG(self->ctx, " Yes!\n"); ++ ++ return 1; ++} ++ ++static int loc_network_gt(struct loc_network* self, struct loc_network* other) { ++ // Families must match ++ if (self->family != other->family) ++ return -1; ++ ++ int r = in6_addr_cmp(&self->first_address, &other->first_address); ++ ++ switch (r) { ++ // Smaller ++ case -1: ++ return 0; ++ ++ // Larger ++ case 1: ++ return 1; ++ ++ default: ++ break; ++ } ++ ++ if (self->prefix > other->prefix) ++ return 1; ++ ++ // Dunno ++ return 0; ++} ++ + LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +@@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net + return 1; + } + ++LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { ++ struct loc_network_list* list; ++ ++ // New prefix length ++ unsigned int prefix = network->prefix + 1; ++ ++ // Check if the new prefix is valid ++ if (valid_prefix(&network->first_address, prefix)) ++ return NULL; ++ ++ // Create a new list with the result ++ int r = loc_network_list_new(network->ctx, &list); ++ if (r) { ++ ERROR(network->ctx, "Could not create network list: %d\n", r); ++ return NULL; ++ } ++ ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; ++ ++ // Create the first half of the network ++ r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix); ++ if (r) ++ goto ERROR; ++ ++ // The next subnet starts after the first one ++ struct in6_addr first_address = address_increment(&subnet1->last_address); ++ ++ // Create the second half of the network ++ r = loc_network_new(network->ctx, &subnet2, &first_address, prefix); ++ if (r) ++ goto ERROR; ++ ++ // Push the both onto the stack (in reverse order) ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; ++ ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; ++ ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); ++ ++ return list; ++ ++ERROR: ++ if (subnet1) ++ loc_network_unref(subnet1); ++ ++ if (subnet2) ++ loc_network_unref(subnet2); ++ ++ if (list) ++ loc_network_list_unref(list); ++ ++ return NULL; ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other) { ++ struct loc_network_list* list; ++ ++#ifdef ENABLE_DEBUG ++ char* n1 = loc_network_str(self); ++ char* n2 = loc_network_str(other); ++ ++ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); ++ ++ free(n1); ++ free(n2); ++#endif ++ ++ // Family must match ++ if (self->family != other->family) { ++ DEBUG(self->ctx, "Family mismatch\n"); ++ ++ return NULL; ++ } ++ ++ // Other must be a subnet of self ++ if (!loc_network_is_subnet_of(other, self)) { ++ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); ++ ++ return NULL; ++ } ++ ++ // We cannot perform this operation if both networks equal ++ if (loc_network_eq(self, other)) { ++ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); ++ ++ return NULL; ++ } ++ ++ // Create a new list with the result ++ int r = loc_network_list_new(self->ctx, &list); ++ if (r) { ++ ERROR(self->ctx, "Could not create network list: %d\n", r); ++ return NULL; ++ } ++ ++ struct loc_network_list* subnets = loc_network_subnets(self); ++ ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; ++ ++ while (subnets) { ++ // Fetch both subnets ++ subnet1 = loc_network_list_get(subnets, 0); ++ subnet2 = loc_network_list_get(subnets, 1); ++ ++ // Free list ++ loc_network_list_unref(subnets); ++ subnets = NULL; ++ ++ if (loc_network_eq(other, subnet1)) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; ++ ++ } else if (loc_network_eq(other, subnet2)) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; ++ ++ } else if (loc_network_is_subnet_of(other, subnet1)) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; ++ ++ subnets = loc_network_subnets(subnet1); ++ ++ } else if (loc_network_is_subnet_of(other, subnet2)) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; ++ ++ subnets = loc_network_subnets(subnet2); ++ ++ } else { ++ ERROR(self->ctx, "We should never get here\n"); ++ goto ERROR; ++ } ++ ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); ++ } ++ ++#ifdef ENABLE_DEBUG ++ loc_network_list_dump(list); ++#endif ++ ++ // Return the result ++ return list; ++ ++ERROR: ++ if (subnet1) ++ loc_network_unref(subnet1); ++ ++ if (subnet2) ++ loc_network_unref(subnet2); ++ ++ if (list) ++ loc_network_list_unref(list); ++ ++ return NULL; ++} ++ + LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { + // Add country code + loc_country_code_copy(dbobj->country_code, network->country_code); +@@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + if (loc_network_list_empty(list)) + return NULL; + +- return list->list[list->size--]; ++ return list->list[--list->size]; ++} ++ ++static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { ++ // Do nothing for invalid indices ++ if (i1 >= list->size || i2 >= list->size) ++ return; ++ ++ DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); ++ ++ struct loc_network* network1 = list->list[i1]; ++ struct loc_network* network2 = list->list[i2]; ++ ++ list->list[i1] = network2; ++ list->list[i2] = network1; ++} ++ ++LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { ++ unsigned int i = 0; ++ unsigned int j = list->size - 1; ++ ++ while (i < j) { ++ loc_network_list_swap(list, i++, j--); ++ } ++} ++ ++LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { ++ unsigned int n = list->size; ++ int swapped; ++ ++ do { ++ swapped = 0; ++ ++ for (unsigned int i = 1; i < n; i++) { ++ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { ++ loc_network_list_swap(list, i-1, i); ++ swapped = 1; ++ } ++ } ++ ++ n--; ++ } while (swapped); + } +diff --git a/src/test-network.c b/src/test-network.c +index b6776b4..af1b2e6 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -118,6 +118,19 @@ int main(int argc, char** argv) { + size_t nodes = loc_network_tree_count_nodes(tree); + printf("The tree has %zu nodes\n", nodes); + ++ // Check equals function ++ err = loc_network_eq(network1, network1); ++ if (!err) { ++ fprintf(stderr, "Network is not equal with itself\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_eq(network1, network2); ++ if (err) { ++ fprintf(stderr, "Networks equal unexpectedly\n"); ++ exit(EXIT_FAILURE); ++ } ++ + // Check subnet function + err = loc_network_is_subnet_of(network1, network2); + if (err != 0) { +@@ -131,6 +144,54 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++ // Make list of subnets ++ struct loc_network_list* subnets = loc_network_subnets(network1); ++ if (!subnets) { ++ fprintf(stderr, "Could not find subnets of network\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_list_dump(subnets); ++ ++ while (!loc_network_list_empty(subnets)) { ++ struct loc_network* subnet = loc_network_list_pop(subnets); ++ if (!subnet) { ++ fprintf(stderr, "Received an empty subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ char* s = loc_network_str(subnet); ++ printf("Received subnet %s\n", s); ++ free(s); ++ ++ if (!loc_network_is_subnet_of(subnet, network1)) { ++ fprintf(stderr, "Not a subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_unref(subnet); ++ } ++ ++ loc_network_list_unref(subnets); ++ ++ struct loc_network_list* excluded = loc_network_exclude(network1, network2); ++ if (!excluded) { ++ fprintf(stderr, "Could not create excluded list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_list_dump(excluded); ++ ++ // Reverse it ++ loc_network_list_reverse(excluded); ++ loc_network_list_dump(excluded); ++ ++ // Sort them and dump them again ++ loc_network_list_sort(excluded); ++ loc_network_list_dump(excluded); ++ ++ loc_network_list_unref(excluded); ++ + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); +-- +2.20.1 + +From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 14:24:58 +0000 +Subject: [PATCH 36/70] networks: Add function to check if two networks overlap + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 16 ++++++++++++++++ + 3 files changed, 18 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index fcb9ea5..0c2835b 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -93,6 +93,7 @@ global: + loc_network_match_flag; + loc_network_new; + loc_network_new_from_string; ++ loc_network_overlaps; + loc_network_ref; + loc_network_set_asn; + loc_network_set_country_code; +diff --git a/src/loc/network.h b/src/loc/network.h +index 4e51a62..ef13756 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_eq(struct loc_network* self, struct loc_network* other); ++int loc_network_overlaps(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); + struct loc_network_list* loc_network_subnets(struct loc_network* network); + struct loc_network_list* loc_network_exclude( +diff --git a/src/network.c b/src/network.c +index 6c08070..d826511 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) { + return 0; + } + ++LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { ++ if (loc_network_match_address(self, &other->first_address) == 0) ++ return 1; ++ ++ if (loc_network_match_address(self, &other->last_address) == 0) ++ return 1; ++ ++ if (loc_network_match_address(other, &self->first_address) == 0) ++ return 1; ++ ++ if (loc_network_match_address(other, &self->last_address) == 0) ++ return 1; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +-- +2.20.1 + +From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 14:28:15 +0000 +Subject: [PATCH 37/70] networks: Add function to check if network is part of a + list + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 9 +++++++++ + 3 files changed, 11 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index 0c2835b..f1b63a2 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -104,6 +104,7 @@ global: + + # Network List + loc_network_list_clear; ++ loc_network_list_contains; + loc_network_list_dump; + loc_network_list_empty; + loc_network_list_get; +diff --git a/src/loc/network.h b/src/loc/network.h +index ef13756..7804512 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); + void loc_network_list_sort(struct loc_network_list* list); + void loc_network_list_reverse(struct loc_network_list* list); + +diff --git a/src/network.c b/src/network.c +index d826511..fcbdc59 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + return list->list[--list->size]; + } + ++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_network_eq(list->list[i], network)) ++ return 1; ++ } ++ ++ return 0; ++} ++ + static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { + // Do nothing for invalid indices + if (i1 >= list->size || i2 >= list->size) +-- +2.20.1 + +From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 14:33:22 +0000 +Subject: [PATCH 38/70] networks: Add function to merge two lists + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 13 +++++++++++++ + 3 files changed, 15 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index f1b63a2..c0b6b1f 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -108,6 +108,7 @@ global: + loc_network_list_dump; + loc_network_list_empty; + loc_network_list_get; ++ loc_network_list_merge; + loc_network_list_new; + loc_network_list_pop; + loc_network_list_push; +diff --git a/src/loc/network.h b/src/loc/network.h +index 7804512..e30d91c 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list); + int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); + void loc_network_list_sort(struct loc_network_list* list); + void loc_network_list_reverse(struct loc_network_list* list); ++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); + + #ifdef LIBLOC_PRIVATE + +diff --git a/src/network.c b/src/network.c +index fcbdc59..541286d 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { + n--; + } while (swapped); + } ++ ++LOC_EXPORT int loc_network_list_merge( ++ struct loc_network_list* self, struct loc_network_list* other) { ++ int r; ++ ++ for (unsigned int i = 0; i < other->size; i++) { ++ r = loc_network_list_push(self, other->list[i]); ++ if (r) ++ return r; ++ } ++ ++ return 0; ++} +-- +2.20.1 + +From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 14:35:43 +0000 +Subject: [PATCH 39/70] network: Make lists unique + +Networks that are in the list won't be added again + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/src/network.c b/src/network.c +index 541286d..44571b3 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis + } + + LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ // Do not add networks that are already on the list ++ if (loc_network_list_contains(list, network)) ++ return 0; ++ + // Check if we have space left + if (list->size == list->max_size) + return -ENOMEM; +-- +2.20.1 + +From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 19:21:13 +0000 +Subject: [PATCH 40/70] database: Pass flag to enumerator to flatten output + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 8 +++++++- + src/loc/database.h | 6 +++++- + src/perl/Location.xs | 2 +- + src/python/database.c | 25 ++++++++++++++++++------- + 4 files changed, 31 insertions(+), 10 deletions(-) + +diff --git a/src/database.c b/src/database.c +index fa1dad0..9baab33 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -104,6 +104,9 @@ struct loc_database_enumerator { + enum loc_network_flags flags; + int family; + ++ // Flatten output? ++ int flatten; ++ + // Index of the AS we are looking at + unsigned int as_index; + +@@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + // Enumerator + + LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode) { ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); + if (!e) + return -ENOMEM; +@@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum + e->mode = mode; + e->refcount = 1; + ++ // Flatten output? ++ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); ++ + // Initialise graph search + //e->network_stack[++e->network_stack_depth] = 0; + e->network_stack_depth = 1; +diff --git a/src/loc/database.h b/src/loc/database.h +index 43173dd..14eb5ea 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -55,9 +55,13 @@ enum loc_database_enumerator_mode { + LOC_DB_ENUMERATE_COUNTRIES = 3, + }; + ++enum loc_database_enumerator_flags { ++ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), ++}; ++ + struct loc_database_enumerator; + int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode); ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); + struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); + struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); + +diff --git a/src/perl/Location.xs b/src/perl/Location.xs +index dcf3f0d..b7676d2 100644 +--- a/src/perl/Location.xs ++++ b/src/perl/Location.xs +@@ -125,7 +125,7 @@ database_countries(db) + PPCODE: + // Create Database enumerator + struct loc_database_enumerator* enumerator; +- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES); ++ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0); + + if (err) { + croak("Could not create a database enumerator\n"); +diff --git a/src/python/database.c b/src/python/database.c +index 1013a58..7f8c2c2 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database + return (PyObject*)self; + } + +-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) { ++static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) { + struct loc_database_enumerator* enumerator; + +- int r = loc_database_enumerator_new(&enumerator, self->db, what); ++ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en + } + + static PyObject* Database_ases(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); + } + + static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { +@@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { + + struct loc_database_enumerator* enumerator; + +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { + } + + static PyObject* Database_networks(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); ++} ++ ++static PyObject* Database_networks_flattened(DatabaseObject *self) { ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +@@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + return NULL; + + struct loc_database_enumerator* enumerator; +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + } + + static PyObject* Database_countries(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); + } + + static struct PyMethodDef Database_methods[] = { +@@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = { + NULL, + NULL, + }, ++ { ++ "networks_flattened", ++ (getter)Database_networks_flattened, ++ NULL, ++ NULL, ++ NULL, ++ }, + { + "vendor", + (getter)Database_get_vendor, +-- +2.20.1 + +From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 19:21:58 +0000 +Subject: [PATCH 41/70] network: Reduce debugging output + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 28 +++------------------------- + 1 file changed, 3 insertions(+), 25 deletions(-) + +diff --git a/src/network.c b/src/network.c +index 44571b3..f7071a6 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag + } + + LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { +-#ifdef ENABLE_DEBUG +- char* n1 = loc_network_str(self); +- char* n2 = loc_network_str(other); +- +- DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); +- +- free(n1); +- free(n2); +-#endif +- + // Family must be the same +- if (self->family != other->family) { +- DEBUG(self->ctx, " Family mismatch\n"); +- ++ if (self->family != other->family) + return 0; +- } + + // The start address must be the same +- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { +- DEBUG(self->ctx, " Address mismatch\n"); +- ++ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) + return 0; +- } + + // The prefix length must be the same +- if (self->prefix != other->prefix) { +- DEBUG(self->ctx, " Prefix mismatch\n"); ++ if (self->prefix != other->prefix) + return 0; +- } +- +- DEBUG(self->ctx, " Yes!\n"); + + return 1; + } +@@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 + if (i1 >= list->size || i2 >= list->size) + return; + +- DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); +- + struct loc_network* network1 = list->list[i1]; + struct loc_network* network2 = list->list[i2]; + +-- +2.20.1 + +From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 19:36:38 +0000 +Subject: [PATCH 42/70] python: Export networks exclude function + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++ + 1 file changed, 39 insertions(+) + +diff --git a/src/python/network.c b/src/python/network.c +index 5496d1e..11f672b 100644 +--- a/src/python/network.c ++++ b/src/python/network.c +@@ -24,6 +24,24 @@ + #include "locationmodule.h" + #include "network.h" + ++static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { ++ PyObject* list = PyList_New(0); ++ if (!networks) ++ return list; ++ ++ while (!loc_network_list_empty(networks)) { ++ struct loc_network* network = loc_network_list_pop(networks); ++ ++ PyObject* n = new_network(&NetworkType, network); ++ PyList_Append(list, n); ++ ++ loc_network_unref(network); ++ Py_DECREF(n); ++ } ++ ++ return list; ++} ++ + PyObject* new_network(PyTypeObject* type, struct loc_network* network) { + NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); + if (self) { +@@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { + Py_RETURN_NONE; + } + ++static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { ++ NetworkObject* other = NULL; ++ ++ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) ++ return NULL; ++ ++ struct loc_network_list* list = loc_network_exclude(self->network, other->network); ++ ++ // Convert to Python objects ++ PyObject* obj = PyList_FromNetworkList(list); ++ loc_network_list_unref(list); ++ ++ return obj; ++} ++ + static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { + NetworkObject* other = NULL; + +@@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) { + } + + static struct PyMethodDef Network_methods[] = { ++ { ++ "exclude", ++ (PyCFunction)Network_exclude, ++ METH_VARARGS, ++ NULL, ++ }, + { + "has_flag", + (PyCFunction)Network_has_flag, +-- +2.20.1 + +From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 19:59:22 +0000 +Subject: [PATCH 43/70] network: Add more debugging output to stacks + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 16 +++++++++++++--- + 1 file changed, 13 insertions(+), 3 deletions(-) + +diff --git a/src/network.c b/src/network.c +index f7071a6..d41e873 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n + return 0; + + // Check if we have space left +- if (list->size == list->max_size) ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); + return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); + + list->list[list->size++] = loc_network_ref(network); + +@@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n + + LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { + // Return nothing when empty +- if (loc_network_list_empty(list)) ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); + return NULL; ++ } + +- return list->list[--list->size]; ++ struct loc_network* network = list->list[--list->size]; ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; + } + + LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { +-- +2.20.1 + +From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 20:00:09 +0000 +Subject: [PATCH 44/70] network: Add new subnet function + +The old one is too difficult to use in terms of order +of input parameters and return value. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 15 +++++++++++++++ + 3 files changed, 17 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index c0b6b1f..5392437 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -87,6 +87,7 @@ global: + loc_network_get_asn; + loc_network_get_country_code; + loc_network_has_flag; ++ loc_network_is_subnet; + loc_network_is_subnet_of; + loc_network_match_asn; + loc_network_match_country_code; +diff --git a/src/loc/network.h b/src/loc/network.h +index e30d91c..2154cdc 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_eq(struct loc_network* self, struct loc_network* other); + int loc_network_overlaps(struct loc_network* self, struct loc_network* other); ++int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); + struct loc_network_list* loc_network_subnets(struct loc_network* network); + struct loc_network_list* loc_network_exclude( +diff --git a/src/network.c b/src/network.c +index d41e873..5719111 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network + return 0; + } + ++LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { ++ // If the start address of the other network is smaller than this network, ++ // it cannot be a subnet. ++ if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) ++ return 0; ++ ++ // If the end address of the other network is greater than this network, ++ // it cannot be a subnet. ++ if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) ++ return 0; ++ ++ return 1; ++} ++ ++// XXX DEPRECATED - I find this too difficult to use + LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +-- +2.20.1 + +From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 20:01:17 +0000 +Subject: [PATCH 45/70] network: Add function to exclude multiple networks at + once + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/libloc.sym | 1 + + src/loc/network.h | 2 ++ + src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 88 insertions(+) + +diff --git a/src/libloc.sym b/src/libloc.sym +index 5392437..bcd11be 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -82,6 +82,7 @@ global: + loc_network_address_family; + loc_network_eq; + loc_network_exclude; ++ loc_network_exclude_list; + loc_network_format_first_address; + loc_network_format_last_address; + loc_network_get_asn; +diff --git a/src/loc/network.h b/src/loc/network.h +index 2154cdc..40712b9 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other + struct loc_network_list* loc_network_subnets(struct loc_network* network); + struct loc_network_list* loc_network_exclude( + struct loc_network* self, struct loc_network* other); ++struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list); + + // List + struct loc_network_list; +diff --git a/src/network.c b/src/network.c +index 5719111..751e8e5 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -677,6 +677,91 @@ ERROR: + return NULL; + } + ++LOC_EXPORT struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list) { ++ struct loc_network_list* to_check; ++ ++ // Create a new list with all networks to look at ++ int r = loc_network_list_new(network->ctx, &to_check); ++ if (r) ++ return NULL; ++ ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets = NULL; ++ ++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Find all excluded networks ++ struct loc_network_list* excluded = loc_network_exclude(network, subnet); ++ if (excluded) { ++ // Add them all to the "to check" list ++ loc_network_list_merge(to_check, excluded); ++ loc_network_list_unref(excluded); ++ } ++ ++ // Cleanup ++ loc_network_unref(subnet); ++ } ++ ++ r = loc_network_list_new(network->ctx, &subnets); ++ if (r) { ++ loc_network_list_unref(to_check); ++ return NULL; ++ } ++ ++ while (!loc_network_list_empty(to_check)) { ++ struct loc_network* subnet_to_check = loc_network_list_pop(to_check); ++ ++ // Marks whether this subnet passed all checks ++ int passed = 1; ++ ++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Drop this subnet if is is already in list ++ if (loc_network_eq(subnet_to_check, subnet)) { ++ passed = 0; ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ // Drop this subnet if is a subnet of another subnet ++ if (loc_network_is_subnet_of(subnet, subnet_to_check)) { ++ passed = 0; ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ // Break it down if it overlaps ++ if (loc_network_overlaps(subnet_to_check, subnet)) { ++ passed = 0; ++ ++ struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet); ++ if (excluded) { ++ loc_network_list_merge(to_check, excluded); ++ loc_network_list_unref(excluded); ++ } ++ ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ loc_network_unref(subnet); ++ } ++ ++ if (passed) { ++ r = loc_network_list_push(subnets, subnet_to_check); ++ } ++ ++ loc_network_unref(subnet_to_check); ++ } ++ ++ loc_network_list_unref(to_check); ++ ++ return subnets; ++} ++ + LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { + // Add country code + loc_country_code_copy(dbobj->country_code, network->country_code); +-- +2.20.1 + +From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 20:02:03 +0000 +Subject: [PATCH 46/70] database: Add option to return networks flattened + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++--------- + 1 file changed, 143 insertions(+), 31 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 9baab33..7a3d1a7 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -118,6 +118,9 @@ struct loc_database_enumerator { + struct loc_node_stack network_stack[MAX_STACK_DEPTH]; + int network_stack_depth; + unsigned int* networks_visited; ++ ++ // For subnet search ++ struct loc_network_list* stack; + }; + + static int loc_database_read_magic(struct loc_database* db) { +@@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + + // Enumerator + ++static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { ++ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); ++ ++ // Release all references ++ loc_database_unref(enumerator->db); ++ loc_unref(enumerator->ctx); ++ ++ if (enumerator->string) ++ free(enumerator->string); ++ ++ // Free network search ++ free(enumerator->networks_visited); ++ ++ // Free subnet stack ++ if (enumerator->stack) ++ loc_network_list_unref(enumerator->stack); ++ ++ free(enumerator); ++} ++ + LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); +@@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + + // Initialise graph search +- //e->network_stack[++e->network_stack_depth] = 0; + e->network_stack_depth = 1; + e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); + ++ // Allocate stack ++ int r = loc_network_list_new(e->ctx, &e->stack); ++ if (r) { ++ loc_database_enumerator_free(e); ++ return r; ++ } ++ + DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); + + *enumerator = e; +@@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo + return enumerator; + } + +-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { +- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); +- +- // Release all references +- loc_database_unref(enumerator->db); +- loc_unref(enumerator->ctx); +- +- if (enumerator->string) +- free(enumerator->string); +- +- // Free network search +- free(enumerator->networks_visited); +- +- free(enumerator); +-} +- + LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { + if (!enumerator) + return NULL; +@@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + +-LOC_EXPORT int loc_database_enumerator_next_network( +- struct loc_database_enumerator* enumerator, struct loc_network** network) { +- // Reset network +- *network = NULL; +- +- // Do not do anything if not in network mode +- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) ++static int __loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { ++ // Return top element from the stack ++ *network = loc_network_list_pop(enumerator->stack); ++ if (*network) + return 0; + +- int r; +- + DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", + enumerator->network_stack_depth); + +@@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( + enumerator->db->network_nodes_v1 + node->offset; + + // Add edges to stack +- r = loc_database_enumerator_stack_push_node(enumerator, ++ int r = loc_database_enumerator_stack_push_node(enumerator, + be32toh(n->one), 1, node->depth + 1); + + if (r) +@@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network( + if (r) + return r; + ++ // Return all networks when the filter is disabled ++ if (!filter) ++ return 0; ++ + // Check if we are interested in this network + + // Skip if the family does not match +@@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network( + } + + // Reached the end of the search ++ return 0; ++} + +- // Mark all nodes as non-visited +- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) +- enumerator->networks_visited[i] = 0; ++static int __loc_database_enumerator_next_network_flattened( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Fetch the next network ++ int r = __loc_database_enumerator_next_network(enumerator, network, 1); ++ if (r) ++ return r; + +- return 0; ++ // End if we could not read another network ++ if (!*network) ++ return 0; ++ ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets; ++ ++ // Create a list with all subnets ++ r = loc_network_list_new(enumerator->ctx, &subnets); ++ if (r) ++ return r; ++ ++ // Search all subnets from the database ++ while (1) { ++ // Fetch the next network in line ++ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); ++ if (r) ++ goto END; ++ ++ // End if we did not receive another subnet ++ if (!subnet) ++ break; ++ ++ // Collect all subnets in a list ++ if (loc_network_is_subnet(*network, subnet)) { ++ r = loc_network_list_push(subnets, subnet); ++ if (r) ++ goto END; ++ ++ loc_network_unref(subnet); ++ continue; ++ } ++ ++ // If this is not a subnet, we push it back onto the stack and break ++ r = loc_network_list_push(enumerator->stack, subnet); ++ if (r) ++ goto END; ++ ++ loc_network_unref(subnet); ++ break; ++ } ++ ++ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); ++ ++ // We can abort here if the network has no subnets ++ if (loc_network_list_empty(subnets)) { ++ loc_network_list_unref(subnets); ++ ++ return 0; ++ } ++ ++ // If the network has any subnets, we will break it into smaller parts ++ // without the subnets. ++ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); ++ if (!excluded || loc_network_list_empty(excluded)) { ++ r = 1; ++ goto END; ++ } ++ ++ // Sort the result ++ loc_network_list_sort(excluded); ++ ++ // Reverse the list ++ loc_network_list_reverse(excluded); ++ ++ // Replace network with the first one ++ loc_network_unref(*network); ++ ++ *network = loc_network_list_pop(excluded); ++ ++ // Push the rest onto the stack ++ loc_network_list_merge(enumerator->stack, excluded); ++ ++ loc_network_list_unref(excluded); ++ ++END: ++ if (subnet) ++ loc_network_unref(subnet); ++ ++ loc_network_list_unref(subnets); ++ ++ return r; ++} ++ ++LOC_EXPORT int loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Do not do anything if not in network mode ++ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) ++ return 0; ++ ++ // Flatten output? ++ if (enumerator->flatten) ++ return __loc_database_enumerator_next_network_flattened(enumerator, network); ++ ++ return __loc_database_enumerator_next_network(enumerator, network, 1); + } + + LOC_EXPORT int loc_database_enumerator_next_country( +-- +2.20.1 + +From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 20:09:20 +0000 +Subject: [PATCH 47/70] test: Update API + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/test-as.c | 2 +- + src/test-database.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/test-as.c b/src/test-as.c +index 839a04c..2d61675 100644 +--- a/src/test-as.c ++++ b/src/test-as.c +@@ -95,7 +95,7 @@ int main(int argc, char** argv) { + // Enumerator + + struct loc_database_enumerator* enumerator; +- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES); ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); + if (err) { + fprintf(stderr, "Could not create a database enumerator\n"); + exit(EXIT_FAILURE); +diff --git a/src/test-database.c b/src/test-database.c +index 4aef94e..da4b11c 100644 +--- a/src/test-database.c ++++ b/src/test-database.c +@@ -198,7 +198,7 @@ int main(int argc, char** argv) { + + // Enumerator + struct loc_database_enumerator* enumerator; +- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); + if (err) { + fprintf(stderr, "Could not initialise the enumerator: %d\n", err); + exit(EXIT_FAILURE); +-- +2.20.1 + +From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Thu, 12 Nov 2020 20:09:37 +0000 +Subject: [PATCH 48/70] networks: Copy all attributes when splitting networks + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 14 ++++++++++++++ + 1 file changed, 14 insertions(+) + +diff --git a/src/network.c b/src/network.c +index 751e8e5..d67f116 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw + if (r) + goto ERROR; + ++ // Copy country code ++ const char* country_code = loc_network_get_country_code(network); ++ if (country_code) { ++ loc_network_set_country_code(subnet1, country_code); ++ loc_network_set_country_code(subnet2, country_code); ++ } ++ ++ // Copy ASN ++ uint32_t asn = loc_network_get_asn(network); ++ if (asn) { ++ loc_network_set_asn(subnet1, asn); ++ loc_network_set_asn(subnet2, asn); ++ } ++ + loc_network_unref(subnet1); + loc_network_unref(subnet2); + +-- +2.20.1 + +From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 13 Nov 2020 11:23:33 +0000 +Subject: [PATCH 49/70] database: Move network filtering into a separate + function + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 56 +++++++++++++++++++++++--------------------------- + 1 file changed, 26 insertions(+), 30 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 7a3d1a7..72bc8eb 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + ++static int loc_database_enumerator_filter_network( ++ struct loc_database_enumerator* enumerator, struct loc_network* network) { ++ // Skip if the family does not match ++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) ++ return 1; ++ ++ // Skip if the country code does not match ++ if (*enumerator->country_code && ++ !loc_network_match_country_code(network, enumerator->country_code)) ++ return 1; ++ ++ // Skip if the ASN does not match ++ if (enumerator->asn && ++ !loc_network_match_asn(network, enumerator->asn)) ++ return 1; ++ ++ // Skip if flags do not match ++ if (enumerator->flags && ++ !loc_network_match_flag(network, enumerator->flags)) ++ return 1; ++ ++ // Do not filter ++ return 0; ++} ++ + static int __loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { + // Return top element from the stack +@@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network( + return 0; + + // Check if we are interested in this network +- +- // Skip if the family does not match +- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { +- loc_network_unref(*network); +- *network = NULL; +- +- continue; +- } +- +- // Skip if the country code does not match +- if (*enumerator->country_code && +- !loc_network_match_country_code(*network, enumerator->country_code)) { +- loc_network_unref(*network); +- *network = NULL; +- +- continue; +- } +- +- // Skip if the ASN does not match +- if (enumerator->asn && +- !loc_network_match_asn(*network, enumerator->asn)) { +- loc_network_unref(*network); +- *network = NULL; +- +- continue; +- } +- +- // Skip if flags do not match +- if (enumerator->flags && +- !loc_network_match_flag(*network, enumerator->flags)) { ++ if (loc_database_enumerator_filter_network(enumerator, *network)) { + loc_network_unref(*network); + *network = NULL; + +-- +2.20.1 + +From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 13 Nov 2020 11:29:02 +0000 +Subject: [PATCH 50/70] database: Filter results coming from stack + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 18 ++++++++++++++++-- + 1 file changed, 16 insertions(+), 2 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 72bc8eb..0f3cdc2 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network( + static int __loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { + // Return top element from the stack +- *network = loc_network_list_pop(enumerator->stack); +- if (*network) ++ while (1) { ++ *network = loc_network_list_pop(enumerator->stack); ++ ++ // Stack is empty ++ if (!*network) ++ break; ++ ++ // Throw away any networks by filter ++ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { ++ loc_network_unref(*network); ++ *network = NULL; ++ continue; ++ } ++ ++ // Return result + return 0; ++ } + + DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", + enumerator->network_stack_depth); +-- +2.20.1 + +From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 13 Nov 2020 11:29:15 +0000 +Subject: [PATCH 51/70] network: Sort result of excluded lists + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 3 --- + src/network.c | 3 +++ + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 0f3cdc2..6849d97 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened( + goto END; + } + +- // Sort the result +- loc_network_list_sort(excluded); +- + // Reverse the list + loc_network_list_reverse(excluded); + +diff --git a/src/network.c b/src/network.c +index d67f116..9d02bf8 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( + + loc_network_list_unref(to_check); + ++ // Sort the result ++ loc_network_list_sort(subnets); ++ + return subnets; + } + +-- +2.20.1 + +From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 13 Nov 2020 11:38:15 +0000 +Subject: [PATCH 52/70] network: Add function to pop first element from stack + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 6 ++---- + src/libloc.sym | 1 + + src/loc/network.h | 1 + + src/network.c | 19 +++++++++++++++++++ + 4 files changed, 23 insertions(+), 4 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 6849d97..b9d870f 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened( + goto END; + } + +- // Reverse the list +- loc_network_list_reverse(excluded); +- + // Replace network with the first one + loc_network_unref(*network); + +- *network = loc_network_list_pop(excluded); ++ *network = loc_network_list_pop_first(excluded); + + // Push the rest onto the stack ++ loc_network_list_reverse(excluded); + loc_network_list_merge(enumerator->stack, excluded); + + loc_network_list_unref(excluded); +diff --git a/src/libloc.sym b/src/libloc.sym +index bcd11be..6139db6 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -113,6 +113,7 @@ global: + loc_network_list_merge; + loc_network_list_new; + loc_network_list_pop; ++ loc_network_list_pop_first; + loc_network_list_push; + loc_network_list_ref; + loc_network_list_reverse; +diff --git a/src/loc/network.h b/src/loc/network.h +index 40712b9..203e61c 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list); + struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); + int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); + struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); + int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); + void loc_network_list_sort(struct loc_network_list* list); + void loc_network_list_reverse(struct loc_network_list* list); +diff --git a/src/network.c b/src/network.c +index 9d02bf8..e7dc97e 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + return network; + } + ++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } ++ ++ struct loc_network* network = list->list[0]; ++ ++ // Move all elements to the top of the stack ++ for (unsigned int i = 0; i < --list->size; i++) { ++ list->list[i] = list->list[i+1]; ++ } ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ + LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { + for (unsigned int i = 0; i < list->size; i++) { + if (loc_network_eq(list->list[i], network)) +-- +2.20.1 + +From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 13 Nov 2020 11:43:53 +0000 +Subject: [PATCH 53/70] network: Unexport all tree functions + +These should not be exported + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network.c | 28 ++++++++++++++-------------- + src/test-network.c | 9 +++++++++ + 2 files changed, 23 insertions(+), 14 deletions(-) + +diff --git a/src/network.c b/src/network.c +index e7dc97e..d015579 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -847,7 +847,7 @@ struct loc_network_tree_node { + struct loc_network* network; + }; + +-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { ++int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { + struct loc_network_tree* t = calloc(1, sizeof(*t)); + if (!t) + return -ENOMEM; +@@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree + return 0; + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { ++struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { + return loc_network_tree_node_ref(tree->root); + } + +@@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_ + return 0; + } + +-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, ++int loc_network_tree_walk(struct loc_network_tree* tree, + int(*filter_callback)(struct loc_network* network, void* data), + int(*callback)(struct loc_network* network, void* data), void* data) { + return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); +@@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) { + free(tree); + } + +-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { ++struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { + if (--tree->refcount > 0) + return tree; + +@@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) { + return 0; + } + +-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { ++int loc_network_tree_dump(struct loc_network_tree* tree) { + DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); + + return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); + } + +-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { ++int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { + DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); + + struct loc_network_tree_node* node = loc_network_tree_get_path(tree, +@@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) { + return 0; + } + +-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { + size_t counter = 0; + + int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); +@@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) + return counter; + } + +-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { + return __loc_network_tree_count_nodes(tree->root); + } + +-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { ++int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { + struct loc_network_tree_node* n = calloc(1, sizeof(*n)); + if (!n) + return -ENOMEM; +@@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network + return 0; + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { + if (node) + node->refcount++; + +@@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { + free(node); + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { + if (!node) + return NULL; + +@@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_ + return NULL; + } + +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { ++struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { + if (index == 0) + node = node->zero; + else +@@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne + return loc_network_tree_node_ref(node); + } + +-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { ++int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { + return (!!node->network); + } + +-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { ++struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } + +diff --git a/src/test-network.c b/src/test-network.c +index af1b2e6..7c90224 100644 +--- a/src/test-network.c ++++ b/src/test-network.c +@@ -37,12 +37,14 @@ int main(int argc, char** argv) { + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG); + ++#if 0 + struct loc_network_tree* tree; + err = loc_network_tree_new(ctx, &tree); + if (err) { + fprintf(stderr, "Could not create the network tree\n"); + exit(EXIT_FAILURE); + } ++#endif + + // Create a network + struct loc_network* network1; +@@ -58,12 +60,14 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network1); + if (err) { + fprintf(stderr, "Could not add network to the tree\n"); + exit(EXIT_FAILURE); + } ++#endif + + // Check if the first and last addresses are correct + char* string = loc_network_format_first_address(network1); +@@ -101,6 +105,7 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + } + ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network2); + if (err) { +@@ -117,6 +122,7 @@ int main(int argc, char** argv) { + + size_t nodes = loc_network_tree_count_nodes(tree); + printf("The tree has %zu nodes\n", nodes); ++#endif + + // Check equals function + err = loc_network_eq(network1, network1); +@@ -260,7 +266,10 @@ int main(int argc, char** argv) { + loc_network_unref(network2); + loc_network_unref(network3); + loc_network_unref(network4); ++ ++#if 0 + loc_network_tree_unref(tree); ++#endif + + // And open it again from disk + struct loc_database* db; +-- +2.20.1 + +From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Fri, 13 Nov 2020 12:09:03 +0000 +Subject: [PATCH 54/70] python: Move tree flattening into C + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/database.c | 8 ++- + src/python/export.py | 138 +++++------------------------------------- + 2 files changed, 21 insertions(+), 125 deletions(-) + +diff --git a/src/python/database.c b/src/python/database.c +index 7f8c2c2..d169547 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL }; ++ char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; + const char* country_code = NULL; + unsigned int asn = 0; + int flags = 0; + int family = 0; ++ int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, ++ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +diff --git a/src/python/export.py b/src/python/export.py +index dd44332..be4a68e 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -87,58 +87,12 @@ class OutputWriter(object): + def _write_network(self, network): + self.f.write("%s\n" % network) + +- def write(self, network, subnets): ++ def write(self, network): + if self.flatten and self._flatten(network): + log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) + return + +- # Convert network into a Python object +- _network = ipaddress.ip_network("%s" % network) +- +- # Write the network when it has no subnets +- if not subnets: +- log.debug("Writing %s to %s" % (_network, self.f)) +- return self._write_network(_network) +- +- # Convert subnets into Python objects +- _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] +- +- # Split the network into smaller bits so that +- # we can accomodate for any gaps in it later +- to_check = set() +- for _subnet in _subnets: +- to_check.update( +- _network.address_exclude(_subnet) +- ) +- +- # Clear the list of all subnets +- subnets = [] +- +- # Check if all subnets to not overlap with anything else +- while to_check: +- subnet_to_check = to_check.pop() +- +- for _subnet in _subnets: +- # Drop this subnet if it equals one of the subnets +- # or if it is subnet of one of them +- if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): +- break +- +- # Break it down if it overlaps +- if subnet_to_check.overlaps(_subnet): +- to_check.update( +- subnet_to_check.address_exclude(_subnet) +- ) +- break +- +- # Add the subnet again as it passed the check +- else: +- subnets.append(subnet_to_check) +- +- # Write all networks as compact as possible +- for network in ipaddress.collapse_addresses(subnets): +- log.debug("Writing %s to %s" % (network, self.f)) +- self._write_network(network) ++ return self._write_network(network) + + def finish(self): + """ +@@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter): + mode = "wb" + + def _write_network(self, network): +- for address in (network.network_address, network.broadcast_address): ++ for address in (network.first_address, network.last_address): + # Convert this into a string of bits + bytes = socket.inet_pton( + socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, +@@ -231,42 +185,21 @@ class Exporter(object): + writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) + + # Get all networks that match the family +- networks = self.db.search_networks(family=family) +- +- # Create a stack with all networks in order where we can put items back +- # again and retrieve them in the next iteration. +- networks = BufferedStack(networks) ++ networks = self.db.search_networks(family=family, flatten=True) + + # Walk through all networks + for network in networks: +- # Collect all networks which are a subnet of network +- subnets = [] +- for subnet in networks: +- # If the next subnet was not a subnet, we have to push +- # it back on the stack and break this loop +- if not subnet.is_subnet_of(network): +- networks.push(subnet) +- break +- +- subnets.append(subnet) +- + # Write matching countries +- if network.country_code and network.country_code in writers: +- # Mismatching subnets +- gaps = [ +- subnet for subnet in subnets if not network.country_code == subnet.country_code +- ] +- +- writers[network.country_code].write(network, gaps) ++ try: ++ writers[network.country_code].write(network) ++ except KeyError: ++ pass + + # Write matching ASNs +- if network.asn and network.asn in writers: +- # Mismatching subnets +- gaps = [ +- subnet for subnet in subnets if not network.asn == subnet.asn +- ] +- +- writers[network.asn].write(network, gaps) ++ try: ++ writers[network.asn].write(network) ++ except KeyError: ++ pass + + # Handle flags + for flag in flags: +@@ -274,19 +207,10 @@ class Exporter(object): + # Fetch the "fake" country code + country = flags[flag] + +- if not country in writers: +- continue +- +- gaps = [ +- subnet for subnet in subnets +- if not subnet.has_flag(flag) +- ] +- +- writers[country].write(network, gaps) +- +- # Push all subnets back onto the stack +- for subnet in reversed(subnets): +- networks.push(subnet) ++ try: ++ writers[country].write(network) ++ except KeyError: ++ pass + + # Write everything to the filesystem + for writer in writers.values(): +@@ -298,33 +222,3 @@ class Exporter(object): + ) + + return os.path.join(directory, filename) +- +- +-class BufferedStack(object): +- """ +- This class takes an iterator and when being iterated +- over it returns objects from that iterator for as long +- as there are any. +- +- It additionally has a function to put an item back on +- the back so that it will be returned again at the next +- iteration. +- """ +- def __init__(self, iterator): +- self.iterator = iterator +- self.stack = [] +- +- def __iter__(self): +- return self +- +- def __next__(self): +- if self.stack: +- return self.stack.pop(0) +- +- return next(self.iterator) +- +- def push(self, elem): +- """ +- Takes an element and puts it on the stack +- """ +- self.stack.insert(0, elem) +-- +2.20.1 + +From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Sun, 15 Nov 2020 15:02:28 +0000 +Subject: [PATCH 55/70] Move network lists into an own file + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + Makefile.am | 2 + + src/libloc.sym | 1 + + src/loc/network-list.h | 37 +++++++ + src/loc/network.h | 20 +--- + src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++ + src/network.c | 207 +------------------------------------ + src/python/network.c | 1 + + 7 files changed, 269 insertions(+), 223 deletions(-) + create mode 100644 src/loc/network-list.h + create mode 100644 src/network-list.c + +diff --git a/Makefile.am b/Makefile.am +index a0431a6..f0d8c4c 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -96,6 +96,7 @@ pkginclude_HEADERS = \ + src/loc/database.h \ + src/loc/format.h \ + src/loc/network.h \ ++ src/loc/network-list.h \ + src/loc/private.h \ + src/loc/stringpool.h \ + src/loc/resolv.h \ +@@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \ + src/country.c \ + src/database.c \ + src/network.c \ ++ src/network-list.c \ + src/resolv.c \ + src/stringpool.c \ + src/writer.c +diff --git a/src/libloc.sym b/src/libloc.sym +index 6139db6..453a1be 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -87,6 +87,7 @@ global: + loc_network_format_last_address; + loc_network_get_asn; + loc_network_get_country_code; ++ loc_network_gt; + loc_network_has_flag; + loc_network_is_subnet; + loc_network_is_subnet_of; +diff --git a/src/loc/network-list.h b/src/loc/network-list.h +new file mode 100644 +index 0000000..af3b28d +--- /dev/null ++++ b/src/loc/network-list.h +@@ -0,0 +1,37 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team info@ipfire.org ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#ifndef LIBLOC_NETWORK_LIST_H ++#define LIBLOC_NETWORK_LIST_H ++ ++struct loc_network_list; ++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); ++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); ++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); ++size_t loc_network_list_size(struct loc_network_list* list); ++int loc_network_list_empty(struct loc_network_list* list); ++void loc_network_list_clear(struct loc_network_list* list); ++void loc_network_list_dump(struct loc_network_list* list); ++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); ++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); ++struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); ++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); ++void loc_network_list_sort(struct loc_network_list* list); ++void loc_network_list_reverse(struct loc_network_list* list); ++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); ++ ++#endif +diff --git a/src/loc/network.h b/src/loc/network.h +index 203e61c..d86b685 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -21,6 +21,7 @@ + + #include <loc/libloc.h> + #include <loc/format.h> ++#include <loc/network-list.h> + + enum loc_network_flags { + LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 +@@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + + int loc_network_eq(struct loc_network* self, struct loc_network* other); ++int loc_network_gt(struct loc_network* self, struct loc_network* other); + int loc_network_overlaps(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); + int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); +@@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude( + struct loc_network_list* loc_network_exclude_list( + struct loc_network* network, struct loc_network_list* list); + +-// List +-struct loc_network_list; +-int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); +-struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); +-struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); +-size_t loc_network_list_size(struct loc_network_list* list); +-int loc_network_list_empty(struct loc_network_list* list); +-void loc_network_list_clear(struct loc_network_list* list); +-void loc_network_list_dump(struct loc_network_list* list); +-struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); +-int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); +-struct loc_network* loc_network_list_pop(struct loc_network_list* list); +-struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); +-int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); +-void loc_network_list_sort(struct loc_network_list* list); +-void loc_network_list_reverse(struct loc_network_list* list); +-int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); +- + #ifdef LIBLOC_PRIVATE + + int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); +diff --git a/src/network-list.c b/src/network-list.c +new file mode 100644 +index 0000000..1f6e80e +--- /dev/null ++++ b/src/network-list.c +@@ -0,0 +1,224 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team info@ipfire.org ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#include <errno.h> ++#include <stdlib.h> ++ ++#include <loc/libloc.h> ++#include <loc/network.h> ++#include <loc/private.h> ++ ++struct loc_network_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_network* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, ++ struct loc_network_list** list) { ++ struct loc_network_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "Network list allocated at %p\n", l); ++ *list = l; ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_network_list_free(struct loc_network_list* list) { ++ DEBUG(list->ctx, "Releasing network list at %p\n", list); ++ ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_network_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->list[i]); ++ ++ list->size = 0; ++} ++ ++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { ++ struct loc_network* network; ++ char* s; ++ ++ for (unsigned int i = 0; i < list->size; i++) { ++ network = list->list[i]; ++ ++ s = loc_network_str(network); ++ ++ INFO(list->ctx, "%s\n", s); ++ free(s); ++ } ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_network_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ // Do not add networks that are already on the list ++ if (loc_network_list_contains(list, network)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); ++ return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); ++ ++ list->list[list->size++] = loc_network_ref(network); ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } ++ ++ struct loc_network* network = list->list[--list->size]; ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } ++ ++ struct loc_network* network = list->list[0]; ++ ++ // Move all elements to the top of the stack ++ for (unsigned int i = 0; i < --list->size; i++) { ++ list->list[i] = list->list[i+1]; ++ } ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ ++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_network_eq(list->list[i], network)) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { ++ // Do nothing for invalid indices ++ if (i1 >= list->size || i2 >= list->size) ++ return; ++ ++ struct loc_network* network1 = list->list[i1]; ++ struct loc_network* network2 = list->list[i2]; ++ ++ list->list[i1] = network2; ++ list->list[i2] = network1; ++} ++ ++LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { ++ unsigned int i = 0; ++ unsigned int j = list->size - 1; ++ ++ while (i < j) { ++ loc_network_list_swap(list, i++, j--); ++ } ++} ++ ++LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { ++ unsigned int n = list->size; ++ int swapped; ++ ++ do { ++ swapped = 0; ++ ++ for (unsigned int i = 1; i < n; i++) { ++ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { ++ loc_network_list_swap(list, i-1, i); ++ swapped = 1; ++ } ++ } ++ ++ n--; ++ } while (swapped); ++} ++ ++LOC_EXPORT int loc_network_list_merge( ++ struct loc_network_list* self, struct loc_network_list* other) { ++ int r; ++ ++ for (unsigned int i = 0; i < other->size; i++) { ++ r = loc_network_list_push(self, other->list[i]); ++ if (r) ++ return r; ++ } ++ ++ return 0; ++} +diff --git a/src/network.c b/src/network.c +index d015579..28ca2df 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -29,6 +29,7 @@ + #include <loc/compat.h> + #include <loc/country.h> + #include <loc/network.h> ++#include <loc/network-list.h> + #include <loc/private.h> + + struct loc_network { +@@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe + return 1; + } + +-static int loc_network_gt(struct loc_network* self, struct loc_network* other) { ++LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { + // Families must match + if (self->family != other->family) + return -1; +@@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { + struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } +- +-// List +- +-struct loc_network_list { +- struct loc_ctx* ctx; +- int refcount; +- +- struct loc_network* list[1024]; +- size_t size; +- size_t max_size; +-}; +- +-LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, +- struct loc_network_list** list) { +- struct loc_network_list* l = calloc(1, sizeof(*l)); +- if (!l) +- return -ENOMEM; +- +- l->ctx = loc_ref(ctx); +- l->refcount = 1; +- +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- +- DEBUG(l->ctx, "Network list allocated at %p\n", l); +- *list = l; +- return 0; +-} +- +-LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { +- list->refcount++; +- +- return list; +-} +- +-static void loc_network_list_free(struct loc_network_list* list) { +- DEBUG(list->ctx, "Releasing network list at %p\n", list); +- +- for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); +- +- loc_unref(list->ctx); +- free(list); +-} +- +-LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { +- if (!list) +- return NULL; +- +- if (--list->refcount > 0) +- return list; +- +- loc_network_list_free(list); +- return NULL; +-} +- +-LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { +- return list->size; +-} +- +-LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { +- return list->size == 0; +-} +- +-LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { +- for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); +- +- list->size = 0; +-} +- +-LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { +- struct loc_network* network; +- char* s; +- +- for (unsigned int i = 0; i < list->size; i++) { +- network = list->list[i]; +- +- s = loc_network_str(network); +- +- INFO(list->ctx, "%s\n", s); +- free(s); +- } +-} +- +-LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { +- // Check index +- if (index >= list->size) +- return NULL; +- +- return loc_network_ref(list->list[index]); +-} +- +-LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { +- // Do not add networks that are already on the list +- if (loc_network_list_contains(list, network)) +- return 0; +- +- // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); +- return -ENOMEM; +- } +- +- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); +- +- list->list[list->size++] = loc_network_ref(network); +- +- return 0; +-} +- +-LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { +- // Return nothing when empty +- if (loc_network_list_empty(list)) { +- DEBUG(list->ctx, "%p: Popped empty stack\n", list); +- return NULL; +- } +- +- struct loc_network* network = list->list[--list->size]; +- +- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); +- +- return network; +-} +- +-LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { +- // Return nothing when empty +- if (loc_network_list_empty(list)) { +- DEBUG(list->ctx, "%p: Popped empty stack\n", list); +- return NULL; +- } +- +- struct loc_network* network = list->list[0]; +- +- // Move all elements to the top of the stack +- for (unsigned int i = 0; i < --list->size; i++) { +- list->list[i] = list->list[i+1]; +- } +- +- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); +- +- return network; +-} +- +-LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { +- for (unsigned int i = 0; i < list->size; i++) { +- if (loc_network_eq(list->list[i], network)) +- return 1; +- } +- +- return 0; +-} +- +-static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { +- // Do nothing for invalid indices +- if (i1 >= list->size || i2 >= list->size) +- return; +- +- struct loc_network* network1 = list->list[i1]; +- struct loc_network* network2 = list->list[i2]; +- +- list->list[i1] = network2; +- list->list[i2] = network1; +-} +- +-LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { +- unsigned int i = 0; +- unsigned int j = list->size - 1; +- +- while (i < j) { +- loc_network_list_swap(list, i++, j--); +- } +-} +- +-LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { +- unsigned int n = list->size; +- int swapped; +- +- do { +- swapped = 0; +- +- for (unsigned int i = 1; i < n; i++) { +- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { +- loc_network_list_swap(list, i-1, i); +- swapped = 1; +- } +- } +- +- n--; +- } while (swapped); +-} +- +-LOC_EXPORT int loc_network_list_merge( +- struct loc_network_list* self, struct loc_network_list* other) { +- int r; +- +- for (unsigned int i = 0; i < other->size; i++) { +- r = loc_network_list_push(self, other->list[i]); +- if (r) +- return r; +- } +- +- return 0; +-} +diff --git a/src/python/network.c b/src/python/network.c +index 11f672b..ed91d65 100644 +--- a/src/python/network.c ++++ b/src/python/network.c +@@ -20,6 +20,7 @@ + + #include <loc/libloc.h> + #include <loc/network.h> ++#include <loc/network-list.h> + + #include "locationmodule.h" + #include "network.h" +-- +2.20.1 + +From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Mon, 16 Nov 2020 15:13:28 +0000 +Subject: [PATCH 56/70] Implement filtering for multiple countries in the + enumerator + +This will allow us to speed up the export of the database +if only a few countries should be returned. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + Makefile.am | 2 + + src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++ + src/country.c | 3 + + src/database.c | 47 ++++++-------- + src/libloc.sym | 15 ++++- + src/loc/country-list.h | 43 +++++++++++++ + src/loc/database.h | 5 +- + src/python/database.c | 57 ++++++++++++++--- + 8 files changed, 274 insertions(+), 36 deletions(-) + create mode 100644 src/country-list.c + create mode 100644 src/loc/country-list.h + +diff --git a/Makefile.am b/Makefile.am +index f0d8c4c..f4ca3c8 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -93,6 +93,7 @@ pkginclude_HEADERS = \ + src/loc/as.h \ + src/loc/compat.h \ + src/loc/country.h \ ++ src/loc/country-list.h \ + src/loc/database.h \ + src/loc/format.h \ + src/loc/network.h \ +@@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \ + src/libloc.c \ + src/as.c \ + src/country.c \ ++ src/country-list.c \ + src/database.c \ + src/network.c \ + src/network-list.c \ +diff --git a/src/country-list.c b/src/country-list.c +new file mode 100644 +index 0000000..ae0d71a +--- /dev/null ++++ b/src/country-list.c +@@ -0,0 +1,138 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team info@ipfire.org ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#include <errno.h> ++#include <stdlib.h> ++ ++#include <loc/country.h> ++#include <loc/country-list.h> ++#include <loc/private.h> ++ ++struct loc_country_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_country* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, ++ struct loc_country_list** list) { ++ struct loc_country_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "Country list allocated at %p\n", l); ++ *list = l; ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_country_list_free(struct loc_country_list* list) { ++ DEBUG(list->ctx, "Releasing country list at %p\n", list); ++ ++ loc_country_list_clear(list); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_country_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_country_unref(list->list[i]); ++} ++ ++LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_country_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_country_list_append( ++ struct loc_country_list* list, struct loc_country* country) { ++ if (loc_country_list_contains(list, country)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); ++ return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); ++ ++ list->list[list->size++] = loc_country_ref(country); ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_country_cmp(country, list->list[i]) == 0) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code) { ++ struct loc_country* country; ++ ++ int r = loc_country_new(list->ctx, &country, code); ++ if (r) ++ return -1; ++ ++ r = loc_country_list_contains(list, country); ++ loc_country_unref(country); ++ ++ return r; ++} +diff --git a/src/country.c b/src/country.c +index 2ba93e6..7aac0db 100644 +--- a/src/country.c ++++ b/src/country.c +@@ -34,6 +34,9 @@ struct loc_country { + }; + + LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { ++ if (!loc_country_code_is_valid(country_code)) ++ return -EINVAL; ++ + struct loc_country* c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; +diff --git a/src/database.c b/src/database.c +index b9d870f..29823b2 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -40,6 +40,7 @@ + #include <loc/as.h> + #include <loc/compat.h> + #include <loc/country.h> ++#include <loc/country-list.h> + #include <loc/database.h> + #include <loc/format.h> + #include <loc/network.h> +@@ -99,7 +100,7 @@ struct loc_database_enumerator { + + // Search string + char* string; +- char country_code[3]; ++ struct loc_country_list* countries; + uint32_t asn; + enum loc_network_flags flags; + int family; +@@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator + return 0; + } + +-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { +- // Set empty country code +- if (!country_code || !*country_code) { +- *enumerator->country_code = '\0'; +- return 0; +- } ++LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->countries) ++ return NULL; + +- // Treat A1, A2, A3 as special country codes, +- // but perform search for flags instead +- if (strcmp(country_code, "A1") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); +- } else if (strcmp(country_code, "A2") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); +- } else if (strcmp(country_code, "A3") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANYCAST); +- } ++ return loc_country_list_ref(enumerator->countries); ++} + +- // Country codes must be two characters +- if (!loc_country_code_is_valid(country_code)) +- return -EINVAL; ++LOC_EXPORT int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries); + +- for (unsigned int i = 0; i < 3; i++) { +- enumerator->country_code[i] = country_code[i]; +- } ++ enumerator->countries = loc_country_list_ref(countries); + + return 0; + } +@@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + ++static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { ++ const char* country_code = loc_network_get_country_code(network); ++ ++ return loc_country_list_contains_code(countries, country_code); ++} ++ + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +@@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network( + return 1; + + // Skip if the country code does not match +- if (*enumerator->country_code && +- !loc_network_match_country_code(network, enumerator->country_code)) ++ if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) + return 1; + + // Skip if the ASN does not match +diff --git a/src/libloc.sym b/src/libloc.sym +index 453a1be..40e9f88 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -49,6 +49,18 @@ global: + loc_country_set_name; + loc_country_unref; + ++ # Country List ++ loc_country_list_append; ++ loc_country_list_clear; ++ loc_country_list_contains; ++ loc_country_list_contains_code; ++ loc_country_list_empty; ++ loc_country_list_get; ++ loc_country_list_new; ++ loc_country_list_ref; ++ loc_country_list_size; ++ loc_country_list_unref; ++ + # Database + loc_database_add_as; + loc_database_count_as; +@@ -66,13 +78,14 @@ global: + loc_database_verify; + + # Database Enumerator ++ loc_database_enumerator_get_countries; + loc_database_enumerator_new; + loc_database_enumerator_next_as; + loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; + loc_database_enumerator_set_asn; +- loc_database_enumerator_set_country_code; ++ loc_database_enumerator_set_countries; + loc_database_enumerator_set_family; + loc_database_enumerator_set_flag; + loc_database_enumerator_set_string; +diff --git a/src/loc/country-list.h b/src/loc/country-list.h +new file mode 100644 +index 0000000..a7f818a +--- /dev/null ++++ b/src/loc/country-list.h +@@ -0,0 +1,43 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2017 IPFire Development Team info@ipfire.org ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#ifndef LIBLOC_COUNTRY_LIST_H ++#define LIBLOC_COUNTRY_LIST_H ++ ++#include <stdlib.h> ++ ++#include <loc/libloc.h> ++#include <loc/country.h> ++ ++struct loc_country_list; ++ ++int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); ++struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); ++struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); ++ ++size_t loc_country_list_size(struct loc_country_list* list); ++int loc_country_list_empty(struct loc_country_list* list); ++void loc_country_list_clear(struct loc_country_list* list); ++ ++struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); ++int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); ++ ++int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country); ++int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code); ++ ++#endif +diff --git a/src/loc/database.h b/src/loc/database.h +index 14eb5ea..246e5c5 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -25,6 +25,7 @@ + #include <loc/network.h> + #include <loc/as.h> + #include <loc/country.h> ++#include <loc/country-list.h> + + struct loc_database; + int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); +@@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_ + struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); + + int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); +-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code); ++struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries); + int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); + int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); + int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); +diff --git a/src/python/database.c b/src/python/database.c +index d169547..e6f6f37 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; +- const char* country_code = NULL; ++ char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; ++ PyObject* country_codes = NULL; + unsigned int asn = 0; + int flags = 0; + int family = 0; + int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, ++ &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +@@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + } + + // Set country code we are searching for +- if (country_code) { +- r = loc_database_enumerator_set_country_code(enumerator, country_code); +- ++ if (country_codes) { ++ struct loc_country_list* countries; ++ r = loc_country_list_new(loc_ctx, &countries); + if (r) { +- PyErr_SetFromErrno(PyExc_SystemError); ++ PyErr_SetString(PyExc_SystemError, "Could not create country list"); + return NULL; + } ++ ++ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) { ++ PyObject* item = PyList_GetItem(country_codes, i); ++ ++ if (!PyUnicode_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ const char* country_code = PyUnicode_AsUTF8(item); ++ ++ struct loc_country* country; ++ r = loc_country_new(loc_ctx, &country, country_code); ++ if (r) { ++ if (r == -EINVAL) { ++ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); ++ } else { ++ PyErr_SetString(PyExc_SystemError, "Could not create country"); ++ } ++ ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ // Append it to the list ++ r = loc_country_list_append(countries, country); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); ++ ++ loc_country_list_unref(countries); ++ loc_country_unref(country); ++ return NULL; ++ } ++ ++ loc_country_unref(country); ++ } ++ ++ loc_database_enumerator_set_countries(enumerator, countries); ++ ++ Py_DECREF(country_codes); ++ loc_country_list_unref(countries); + } + + // Set the ASN we are searching for +-- +2.20.1 + +From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Mon, 16 Nov 2020 15:20:50 +0000 +Subject: [PATCH 57/70] python: Only return country codes we want + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/export.py | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/src/python/export.py b/src/python/export.py +index be4a68e..5e7fe53 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -184,8 +184,14 @@ class Exporter(object): + + writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) + ++ # Filter countries from special country codes ++ country_codes = [ ++ country_code for country_code in countries if not country_code in flags.values() ++ ] ++ + # Get all networks that match the family +- networks = self.db.search_networks(family=family, flatten=True) ++ networks = self.db.search_networks(family=family, ++ country_codes=country_codes, flatten=True) + + # Walk through all networks + for network in networks: +-- +2.20.1 + +From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Mon, 16 Nov 2020 15:25:15 +0000 +Subject: [PATCH 58/70] database: Filter flags in C + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/export.py | 16 +++++++++++----- + 1 file changed, 11 insertions(+), 5 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index 5e7fe53..739742f 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -29,7 +29,7 @@ import _location + log = logging.getLogger("location.export") + log.propagate = 1 + +-flags = { ++FLAGS = { + _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", + _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", + _location.NETWORK_FLAG_ANYCAST : "A3", +@@ -186,12 +186,18 @@ class Exporter(object): + + # Filter countries from special country codes + country_codes = [ +- country_code for country_code in countries if not country_code in flags.values() ++ country_code for country_code in countries if not country_code in FLAGS.values() + ] + ++ # Collect flags ++ flags = 0 ++ for flag in FLAGS: ++ if FLAGS[flag] in countries: ++ flags |= flag ++ + # Get all networks that match the family + networks = self.db.search_networks(family=family, +- country_codes=country_codes, flatten=True) ++ country_codes=country_codes, flags=flags, flatten=True) + + # Walk through all networks + for network in networks: +@@ -208,10 +214,10 @@ class Exporter(object): + pass + + # Handle flags +- for flag in flags: ++ for flag in FLAGS: + if network.has_flag(flag): + # Fetch the "fake" country code +- country = flags[flag] ++ country = FLAGS[flag] + + try: + writers[country].write(network) +-- +2.20.1 + +From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 16:46:48 +0000 +Subject: [PATCH 59/70] as: Add list for easier processing + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + Makefile.am | 2 + + src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++ + src/database.c | 29 +++++++-- + src/libloc.sym | 15 ++++- + src/loc/as-list.h | 41 +++++++++++++ + src/loc/database.h | 5 +- + src/python/database.c | 58 ++++++++++++++++-- + src/python/export.py | 2 +- + 8 files changed, 275 insertions(+), 15 deletions(-) + create mode 100644 src/as-list.c + create mode 100644 src/loc/as-list.h + +diff --git a/Makefile.am b/Makefile.am +index f4ca3c8..d0cc793 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -91,6 +91,7 @@ EXTRA_DIST += \ + pkginclude_HEADERS = \ + src/loc/libloc.h \ + src/loc/as.h \ ++ src/loc/as-list.h \ + src/loc/compat.h \ + src/loc/country.h \ + src/loc/country-list.h \ +@@ -109,6 +110,7 @@ lib_LTLIBRARIES = \ + src_libloc_la_SOURCES = \ + src/libloc.c \ + src/as.c \ ++ src/as-list.c \ + src/country.c \ + src/country-list.c \ + src/database.c \ +diff --git a/src/as-list.c b/src/as-list.c +new file mode 100644 +index 0000000..7c69eb0 +--- /dev/null ++++ b/src/as-list.c +@@ -0,0 +1,138 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2020 IPFire Development Team info@ipfire.org ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#include <errno.h> ++#include <stdlib.h> ++ ++#include <loc/as.h> ++#include <loc/as-list.h> ++#include <loc/private.h> ++ ++struct loc_as_list { ++ struct loc_ctx* ctx; ++ int refcount; ++ ++ struct loc_as* list[1024]; ++ size_t size; ++ size_t max_size; ++}; ++ ++LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, ++ struct loc_as_list** list) { ++ struct loc_as_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; ++ ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; ++ ++ // Do not allow this list to grow larger than this ++ l->max_size = 1024; ++ ++ DEBUG(l->ctx, "AS list allocated at %p\n", l); ++ *list = l; ++ ++ return 0; ++} ++ ++LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { ++ list->refcount++; ++ ++ return list; ++} ++ ++static void loc_as_list_free(struct loc_as_list* list) { ++ DEBUG(list->ctx, "Releasing AS list at %p\n", list); ++ ++ loc_as_list_clear(list); ++ ++ loc_unref(list->ctx); ++ free(list); ++} ++ ++LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_as_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_as_unref(list->list[i]); ++} ++ ++LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_as_ref(list->list[index]); ++} ++ ++LOC_EXPORT int loc_as_list_append( ++ struct loc_as_list* list, struct loc_as* as) { ++ if (loc_as_list_contains(list, as)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size == list->max_size) { ++ ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); ++ return -ENOMEM; ++ } ++ ++ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); ++ ++ list->list[list->size++] = loc_as_ref(as); ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_as_cmp(as, list->list[i]) == 0) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number) { ++ struct loc_as* as; ++ ++ int r = loc_as_new(list->ctx, &as, number); ++ if (r) ++ return -1; ++ ++ r = loc_as_list_contains(list, as); ++ loc_as_unref(as); ++ ++ return r; ++} +diff --git a/src/database.c b/src/database.c +index 29823b2..51cb5cd 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -38,6 +38,7 @@ + + #include <loc/libloc.h> + #include <loc/as.h> ++#include <loc/as-list.h> + #include <loc/compat.h> + #include <loc/country.h> + #include <loc/country-list.h> +@@ -101,7 +102,7 @@ struct loc_database_enumerator { + // Search string + char* string; + struct loc_country_list* countries; +- uint32_t asn; ++ struct loc_as_list* asns; + enum loc_network_flags flags; + int family; + +@@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries( + return 0; + } + +-LOC_EXPORT int loc_database_enumerator_set_asn( +- struct loc_database_enumerator* enumerator, unsigned int asn) { +- enumerator->asn = asn; ++LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->asns) ++ return NULL; ++ ++ return loc_as_list_ref(enumerator->asns); ++} ++ ++LOC_EXPORT int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ ++ enumerator->asns = loc_as_list_ref(asns); + + return 0; + } +@@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c + return loc_country_list_contains_code(countries, country_code); + } + ++static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { ++ uint32_t asn = loc_network_get_asn(network); ++ ++ return loc_as_list_contains_number(asns, asn); ++} ++ + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +@@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network( + return 1; + + // Skip if the ASN does not match +- if (enumerator->asn && +- !loc_network_match_asn(network, enumerator->asn)) ++ if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) + return 1; + + // Skip if flags do not match +diff --git a/src/libloc.sym b/src/libloc.sym +index 40e9f88..53273cd 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -37,6 +37,18 @@ global: + loc_as_set_name; + loc_as_unref; + ++ # AS List ++ loc_as_list_append; ++ loc_as_list_clear; ++ loc_as_list_contains; ++ loc_as_list_contains_number; ++ loc_as_list_empty; ++ loc_as_list_get; ++ loc_as_list_new; ++ loc_as_list_ref; ++ loc_as_list_size; ++ loc_as_list_unref; ++ + # Country + loc_country_cmp; + loc_country_code_is_valid; +@@ -78,13 +90,14 @@ global: + loc_database_verify; + + # Database Enumerator ++ loc_database_enumerator_get_asns; + loc_database_enumerator_get_countries; + loc_database_enumerator_new; + loc_database_enumerator_next_as; + loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; +- loc_database_enumerator_set_asn; ++ loc_database_enumerator_set_asns; + loc_database_enumerator_set_countries; + loc_database_enumerator_set_family; + loc_database_enumerator_set_flag; +diff --git a/src/loc/as-list.h b/src/loc/as-list.h +new file mode 100644 +index 0000000..7b5c4e8 +--- /dev/null ++++ b/src/loc/as-list.h +@@ -0,0 +1,41 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2017 IPFire Development Team info@ipfire.org ++ ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ ++ ++#ifndef LIBLOC_AS_LIST_H ++#define LIBLOC_AS_LIST_H ++ ++#include <loc/as.h> ++#include <loc/libloc.h> ++ ++struct loc_as_list; ++ ++int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); ++struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); ++struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); ++ ++size_t loc_as_list_size(struct loc_as_list* list); ++int loc_as_list_empty(struct loc_as_list* list); ++void loc_as_list_clear(struct loc_as_list* list); ++ ++struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); ++int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); ++ ++int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as); ++int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number); ++ ++#endif +diff --git a/src/loc/database.h b/src/loc/database.h +index 246e5c5..70801f0 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato + struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); + int loc_database_enumerator_set_countries( + struct loc_database_enumerator* enumerator, struct loc_country_list* countries); +-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); ++struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); + int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); + int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); + int loc_database_enumerator_next_as( +diff --git a/src/python/database.c b/src/python/database.c +index e6f6f37..38a804c 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -17,6 +17,8 @@ + #include <Python.h> + + #include <loc/libloc.h> ++#include <loc/as.h> ++#include <loc/as-list.h> + #include <loc/database.h> + + #include "locationmodule.h" +@@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; ++ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; + PyObject* country_codes = NULL; +- unsigned int asn = 0; ++ PyObject* asn_list = NULL; + int flags = 0; + int family = 0; + int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, +- &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, ++ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +@@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + } + + // Set the ASN we are searching for +- if (asn) { +- r = loc_database_enumerator_set_asn(enumerator, asn); ++ if (asn_list) { ++ struct loc_as_list* asns; ++ r = loc_as_list_new(loc_ctx, &asns); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); ++ return NULL; ++ } ++ ++ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) { ++ PyObject* item = PyList_GetItem(asn_list, i); ++ ++ if (!PyLong_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); ++ ++ loc_as_list_unref(asns); ++ return NULL; ++ } ++ ++ unsigned long number = PyLong_AsLong(item); + ++ struct loc_as* as; ++ r = loc_as_new(loc_ctx, &as, number); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS"); ++ ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } ++ ++ r = loc_as_list_append(asns, as); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); ++ ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } ++ ++ loc_as_unref(as); ++ } ++ ++ r = loc_database_enumerator_set_asns(enumerator, asns); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); ++ ++ loc_as_list_unref(asns); + return NULL; + } ++ ++ loc_as_list_unref(asns); + } + + // Set the flags we are searching for +diff --git a/src/python/export.py b/src/python/export.py +index 739742f..f675eb3 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -197,7 +197,7 @@ class Exporter(object): + + # Get all networks that match the family + networks = self.db.search_networks(family=family, +- country_codes=country_codes, flags=flags, flatten=True) ++ country_codes=country_codes, asns=asns, flags=flags, flatten=True) + + # Walk through all networks + for network in networks: +-- +2.20.1 + +From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 16:56:43 +0000 +Subject: [PATCH 60/70] database: Simplify network matching code + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 51cb5cd..1a354f6 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + +-static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { +- const char* country_code = loc_network_get_country_code(network); +- +- return loc_country_list_contains_code(countries, country_code); +-} +- + static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { + uint32_t asn = loc_network_get_asn(network); + +@@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network( + return 1; + + // Skip if the country code does not match +- if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) +- return 1; ++ if (enumerator->countries) { ++ if (!loc_country_list_empty(enumerator->countries)) { ++ const char* country_code = loc_network_get_country_code(network); ++ ++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) ++ return 1; ++ } ++ } + + // Skip if the ASN does not match + if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) +-- +2.20.1 + +From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 16:58:55 +0000 +Subject: [PATCH 61/70] database: Simplify AS matching code + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/database.c b/src/database.c +index 1a354f6..be93e00 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( + return 0; + } + +-static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { +- uint32_t asn = loc_network_get_asn(network); +- +- return loc_as_list_contains_number(asns, asn); +-} +- + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +@@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network( + } + + // Skip if the ASN does not match +- if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) +- return 1; ++ if (enumerator->asns) { ++ if (!loc_as_list_empty(enumerator->asns)) { ++ uint32_t asn = loc_network_get_asn(network); ++ ++ if (!loc_as_list_contains_number(enumerator->asns, asn)) ++ return 1; ++ } ++ } + + // Skip if flags do not match + if (enumerator->flags && +-- +2.20.1 + +From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 17:50:17 +0000 +Subject: [PATCH 62/70] countries: Make list grow dynamically + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/country-list.c | 38 ++++++++++++++++++++++++++------------ + 1 file changed, 26 insertions(+), 12 deletions(-) + +diff --git a/src/country-list.c b/src/country-list.c +index ae0d71a..1ce2d06 100644 +--- a/src/country-list.c ++++ b/src/country-list.c +@@ -25,11 +25,27 @@ struct loc_country_list { + struct loc_ctx* ctx; + int refcount; + +- struct loc_country* list[1024]; ++ struct loc_country** elements; ++ size_t elements_size; ++ + size_t size; +- size_t max_size; + }; + ++static int loc_country_list_grow(struct loc_country_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_country** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, + struct loc_country_list** list) { + struct loc_country_list* l = calloc(1, sizeof(*l)); +@@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, + l->ctx = loc_ref(ctx); + l->refcount = 1; + +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- + DEBUG(l->ctx, "Country list allocated at %p\n", l); + *list = l; + +@@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { + + LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { + for (unsigned int i = 0; i < list->size; i++) +- loc_country_unref(list->list[i]); ++ loc_country_unref(list->elements[i]); + } + + LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { +@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis + if (index >= list->size) + return NULL; + +- return loc_country_ref(list->list[index]); ++ return loc_country_ref(list->elements[index]); + } + + LOC_EXPORT int loc_country_list_append( +@@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append( + return 0; + + // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); +- return -ENOMEM; ++ if (list->size >= list->elements_size) { ++ int r = loc_country_list_grow(list, 64); ++ if (r) ++ return r; + } + + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); + +- list->list[list->size++] = loc_country_ref(country); ++ list->elements[list->size++] = loc_country_ref(country); + + return 0; + } +@@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append( + LOC_EXPORT int loc_country_list_contains( + struct loc_country_list* list, struct loc_country* country) { + for (unsigned int i = 0; i < list->size; i++) { +- if (loc_country_cmp(country, list->list[i]) == 0) ++ if (loc_country_cmp(country, list->elements[i]) == 0) + return 1; + } + +-- +2.20.1 + +From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 17:55:51 +0000 +Subject: [PATCH 63/70] networks: Make list grow dynamically + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/network-list.c | 60 ++++++++++++++++++++++++++++------------------ + 1 file changed, 37 insertions(+), 23 deletions(-) + +diff --git a/src/network-list.c b/src/network-list.c +index 1f6e80e..4912c02 100644 +--- a/src/network-list.c ++++ b/src/network-list.c +@@ -25,11 +25,27 @@ struct loc_network_list { + struct loc_ctx* ctx; + int refcount; + +- struct loc_network* list[1024]; ++ struct loc_network** elements; ++ size_t elements_size; ++ + size_t size; +- size_t max_size; + }; + ++static int loc_network_list_grow(struct loc_network_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_network** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, + struct loc_network_list** list) { + struct loc_network_list* l = calloc(1, sizeof(*l)); +@@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, + l->ctx = loc_ref(ctx); + l->refcount = 1; + +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- + DEBUG(l->ctx, "Network list allocated at %p\n", l); + *list = l; + return 0; +@@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) { + DEBUG(list->ctx, "Releasing network list at %p\n", list); + + for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); ++ loc_network_unref(list->elements[i]); + + loc_unref(list->ctx); + free(list); +@@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { + + LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { + for (unsigned int i = 0; i < list->size; i++) +- loc_network_unref(list->list[i]); ++ loc_network_unref(list->elements[i]); + + list->size = 0; + } +@@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { + char* s; + + for (unsigned int i = 0; i < list->size; i++) { +- network = list->list[i]; ++ network = list->elements[i]; + + s = loc_network_str(network); + +@@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis + if (index >= list->size) + return NULL; + +- return loc_network_ref(list->list[index]); ++ return loc_network_ref(list->elements[index]); + } + + LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { +@@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n + return 0; + + // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); +- return -ENOMEM; ++ if (list->size >= list->elements_size) { ++ int r = loc_network_list_grow(list, 64); ++ if (r) ++ return r; + } + + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); + +- list->list[list->size++] = loc_network_ref(network); ++ list->elements[list->size++] = loc_network_ref(network); + + return 0; + } +@@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis + return NULL; + } + +- struct loc_network* network = list->list[--list->size]; ++ struct loc_network* network = list->elements[--list->size]; + + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); + +@@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis + return NULL; + } + +- struct loc_network* network = list->list[0]; ++ struct loc_network* network = list->elements[0]; + + // Move all elements to the top of the stack + for (unsigned int i = 0; i < --list->size; i++) { +- list->list[i] = list->list[i+1]; ++ list->elements[i] = list->elements[i+1]; + } + + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); +@@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis + + LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { + for (unsigned int i = 0; i < list->size; i++) { +- if (loc_network_eq(list->list[i], network)) ++ if (loc_network_eq(list->elements[i], network)) + return 1; + } + +@@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 + if (i1 >= list->size || i2 >= list->size) + return; + +- struct loc_network* network1 = list->list[i1]; +- struct loc_network* network2 = list->list[i2]; ++ struct loc_network* network1 = list->elements[i1]; ++ struct loc_network* network2 = list->elements[i2]; + +- list->list[i1] = network2; +- list->list[i2] = network1; ++ list->elements[i1] = network2; ++ list->elements[i2] = network1; + } + + LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { +@@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { + swapped = 0; + + for (unsigned int i = 1; i < n; i++) { +- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { ++ if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) { + loc_network_list_swap(list, i-1, i); + swapped = 1; + } +@@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge( + int r; + + for (unsigned int i = 0; i < other->size; i++) { +- r = loc_network_list_push(self, other->list[i]); ++ r = loc_network_list_push(self, other->elements[i]); + if (r) + return r; + } +-- +2.20.1 + +From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 17:57:55 +0000 +Subject: [PATCH 64/70] as: Make lists grow dynamically + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/as-list.c | 38 ++++++++++++++++++++++++++------------ + 1 file changed, 26 insertions(+), 12 deletions(-) + +diff --git a/src/as-list.c b/src/as-list.c +index 7c69eb0..17de23e 100644 +--- a/src/as-list.c ++++ b/src/as-list.c +@@ -25,11 +25,27 @@ struct loc_as_list { + struct loc_ctx* ctx; + int refcount; + +- struct loc_as* list[1024]; ++ struct loc_as** elements; ++ size_t elements_size; ++ + size_t size; +- size_t max_size; + }; + ++static int loc_as_list_grow(struct loc_as_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_as** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ + LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, + struct loc_as_list** list) { + struct loc_as_list* l = calloc(1, sizeof(*l)); +@@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, + l->ctx = loc_ref(ctx); + l->refcount = 1; + +- // Do not allow this list to grow larger than this +- l->max_size = 1024; +- + DEBUG(l->ctx, "AS list allocated at %p\n", l); + *list = l; + +@@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { + + LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { + for (unsigned int i = 0; i < list->size; i++) +- loc_as_unref(list->list[i]); ++ loc_as_unref(list->elements[i]); + } + + LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { +@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index + if (index >= list->size) + return NULL; + +- return loc_as_ref(list->list[index]); ++ return loc_as_ref(list->elements[index]); + } + + LOC_EXPORT int loc_as_list_append( +@@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append( + return 0; + + // Check if we have space left +- if (list->size == list->max_size) { +- ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); +- return -ENOMEM; ++ if (list->size >= list->elements_size) { ++ int r = loc_as_list_grow(list, 64); ++ if (r) ++ return r; + } + + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); + +- list->list[list->size++] = loc_as_ref(as); ++ list->elements[list->size++] = loc_as_ref(as); + + return 0; + } +@@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append( + LOC_EXPORT int loc_as_list_contains( + struct loc_as_list* list, struct loc_as* as) { + for (unsigned int i = 0; i < list->size; i++) { +- if (loc_as_cmp(as, list->list[i]) == 0) ++ if (loc_as_cmp(as, list->elements[i]) == 0) + return 1; + } + +-- +2.20.1 + +From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 18:13:49 +0000 +Subject: [PATCH 65/70] export: Change back to use Network objects + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/export.py | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index f675eb3..67e437f 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter): + def _write_network(self, network): + for address in (network.first_address, network.last_address): + # Convert this into a string of bits +- bytes = socket.inet_pton( +- socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, +- ) ++ bytes = socket.inet_pton(network.family, address) + + self.f.write(bytes) + +-- +2.20.1 + +From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 18:14:15 +0000 +Subject: [PATCH 66/70] Actually clear all lists + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/as-list.c | 8 ++++++++ + src/country-list.c | 8 ++++++++ + src/network-list.c | 6 ++++++ + 3 files changed, 22 insertions(+) + +diff --git a/src/as-list.c b/src/as-list.c +index 17de23e..76620c7 100644 +--- a/src/as-list.c ++++ b/src/as-list.c +@@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { + } + + LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { ++ if (!list->elements) ++ return; ++ + for (unsigned int i = 0; i < list->size; i++) + loc_as_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements_size = 0; ++ ++ list->size = 0; + } + + LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { +diff --git a/src/country-list.c b/src/country-list.c +index 1ce2d06..1c49c47 100644 +--- a/src/country-list.c ++++ b/src/country-list.c +@@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { + } + + LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { ++ if (!list->elements) ++ return; ++ + for (unsigned int i = 0; i < list->size; i++) + loc_country_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements_size = 0; ++ ++ list->size = 0; + } + + LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { +diff --git a/src/network-list.c b/src/network-list.c +index 4912c02..9cb4547 100644 +--- a/src/network-list.c ++++ b/src/network-list.c +@@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { + } + + LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ if (!list->elements) ++ return; ++ + for (unsigned int i = 0; i < list->size; i++) + loc_network_unref(list->elements[i]); + ++ free(list->elements); ++ list->elements_size = 0; ++ + list->size = 0; + } + +-- +2.20.1 + +From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 18:34:51 +0000 +Subject: [PATCH 67/70] database: Do not clean up python list + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/database.c | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/src/python/database.c b/src/python/database.c +index 38a804c..ed22275 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + loc_country_unref(country); + } + +- loc_database_enumerator_set_countries(enumerator, countries); ++ r = loc_database_enumerator_set_countries(enumerator, countries); ++ if (r) { ++ PyErr_SetFromErrno(PyExc_SystemError); ++ ++ loc_as_list_unref(countries); ++ return NULL; ++ } + +- Py_DECREF(country_codes); + loc_country_list_unref(countries); + } + +-- +2.20.1 + +From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Tue, 17 Nov 2020 19:01:04 +0000 +Subject: [PATCH 68/70] database: Free filter lists in enumerator + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/src/database.c b/src/database.c +index be93e00..ca35fe1 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera + if (enumerator->string) + free(enumerator->string); + ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries); ++ ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ + // Free network search + free(enumerator->networks_visited); + +-- +2.20.1 + +From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 18 Nov 2020 13:18:52 +0000 +Subject: [PATCH 69/70] database: Add debug output to filtering + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/database.c | 29 ++++++++++++++++------------- + 1 file changed, 16 insertions(+), 13 deletions(-) + +diff --git a/src/database.c b/src/database.c +index ca35fe1..83dd752 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node( + static int loc_database_enumerator_filter_network( + struct loc_database_enumerator* enumerator, struct loc_network* network) { + // Skip if the family does not match +- if (enumerator->family && loc_network_address_family(network) != enumerator->family) ++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); + return 1; ++ } + + // Skip if the country code does not match +- if (enumerator->countries) { +- if (!loc_country_list_empty(enumerator->countries)) { +- const char* country_code = loc_network_get_country_code(network); ++ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { ++ const char* country_code = loc_network_get_country_code(network); + +- if (!loc_country_list_contains_code(enumerator->countries, country_code)) +- return 1; ++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); ++ return 1; + } + } + + // Skip if the ASN does not match +- if (enumerator->asns) { +- if (!loc_as_list_empty(enumerator->asns)) { +- uint32_t asn = loc_network_get_asn(network); ++ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { ++ uint32_t asn = loc_network_get_asn(network); + +- if (!loc_as_list_contains_number(enumerator->asns, asn)) +- return 1; ++ if (!loc_as_list_contains_number(enumerator->asns, asn)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); ++ return 1; + } + } + + // Skip if flags do not match +- if (enumerator->flags && +- !loc_network_match_flag(network, enumerator->flags)) ++ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); + return 1; ++ } + + // Do not filter + return 0; +-- +2.20.1 + +From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Wed, 18 Nov 2020 13:19:04 +0000 +Subject: [PATCH 70/70] export: Remove filtering for flags + +The filter is an AND filter and if we set the flags from +the special country codes, we won't get back much. + +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/export.py | 8 +------- + 1 file changed, 1 insertion(+), 7 deletions(-) + +diff --git a/src/python/export.py b/src/python/export.py +index 67e437f..4219957 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -187,15 +187,9 @@ class Exporter(object): + country_code for country_code in countries if not country_code in FLAGS.values() + ] + +- # Collect flags +- flags = 0 +- for flag in FLAGS: +- if FLAGS[flag] in countries: +- flags |= flag +- + # Get all networks that match the family + networks = self.db.search_networks(family=family, +- country_codes=country_codes, asns=asns, flags=flags, flatten=True) ++ country_codes=country_codes, asns=asns, flatten=True) + + # Walk through all networks + for network in networks: +-- +2.20.1 +

hooks/post-receive -- IPFire 2.x development tree