[git.ipfire.org] IPFire 2.x development tree branch, next, updated. 7adacda04c2ca766bd25ac62aa17cdbeadaa1abc
This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "IPFire 2.x development tree".
The branch, next has been updated via 7adacda04c2ca766bd25ac62aa17cdbeadaa1abc (commit) via b952a52b706da0d0cc208dbc6870785d7460c32b (commit) from d6989b4b0b42937485f5008cda6cdd730275f1ec (commit)
Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.
- Log ----------------------------------------------------------------- commit 7adacda04c2ca766bd25ac62aa17cdbeadaa1abc Author: Arne Fitzenreiter arne_f@ipfire.org Date: Tue Nov 24 20:51:25 2020 +0100
transmission: update to 3.00
Signed-off-by: Arne Fitzenreiter arne_f@ipfire.org Signed-off-by: Michael Tremer michael.tremer@ipfire.org
commit b952a52b706da0d0cc208dbc6870785d7460c32b Author: Michael Tremer michael.tremer@ipfire.org Date: Thu Nov 26 16:15:07 2020 +0000
libloc: Import latest changes from upstream
This is now a unified patch instead of being split into individual commits from upstream.
Signed-off-by: Michael Tremer michael.tremer@ipfire.org
-----------------------------------------------------------------------
Summary of changes: config/rootfiles/packages/transmission | 2 + lfs/transmission | 8 +- src/patches/libloc-0.9.4-upstream.patch | 13050 +++++++----------------------- 3 files changed, 3047 insertions(+), 10013 deletions(-)
Difference in files: diff --git a/config/rootfiles/packages/transmission b/config/rootfiles/packages/transmission index fbf1ad191..d831ecde9 100644 --- a/config/rootfiles/packages/transmission +++ b/config/rootfiles/packages/transmission @@ -31,7 +31,9 @@ usr/share/transmission #usr/share/transmission/web/javascript/jquery/jquery.transmenu.min.js #usr/share/transmission/web/javascript/jquery/jquery.ui-contextmenu.min.js #usr/share/transmission/web/javascript/jquery/json2.min.js +#usr/share/transmission/web/javascript/main.js #usr/share/transmission/web/javascript/notifications.js +#usr/share/transmission/web/javascript/polyfill.js #usr/share/transmission/web/javascript/prefs-dialog.js #usr/share/transmission/web/javascript/remote.js #usr/share/transmission/web/javascript/torrent-row.js diff --git a/lfs/transmission b/lfs/transmission index 6f52d6e5a..d9c82aaae 100644 --- a/lfs/transmission +++ b/lfs/transmission @@ -1,7 +1,7 @@ ############################################################################### # # # IPFire.org - A linux based firewall # -# Copyright (C) 2007-2019 IPFire Team info@ipfire.org # +# Copyright (C) 2007-2020 IPFire Team info@ipfire.org # # # # This program is free software: you can redistribute it and/or modify # # it under the terms of the GNU General Public License as published by # @@ -24,7 +24,7 @@
include Config
-VER = 2.94 +VER = 3.00
THISAPP = transmission-$(VER) DL_FILE = $(THISAPP).tar.xz @@ -32,7 +32,7 @@ DL_FROM = $(URL_IPFIRE) DIR_APP = $(DIR_SRC)/$(THISAPP) TARGET = $(DIR_INFO)/$(THISAPP) PROG = transmission -PAK_VER = 16 +PAK_VER = 17
DEPS =
@@ -46,7 +46,7 @@ objects = $(DL_FILE)
$(DL_FILE) = $(DL_FROM)/$(DL_FILE)
-$(DL_FILE)_MD5 = c92829294edfa391c046407eeb16358a +$(DL_FILE)_MD5 = a23a32672b83c89b9b61e90408f53d98
install : $(TARGET)
diff --git a/src/patches/libloc-0.9.4-upstream.patch b/src/patches/libloc-0.9.4-upstream.patch index 416420532..79db50903 100644 --- a/src/patches/libloc-0.9.4-upstream.patch +++ b/src/patches/libloc-0.9.4-upstream.patch @@ -1,834 +1,251 @@ -From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 24 Sep 2020 10:17:58 +0000 -Subject: [PATCH 001/111] Revert "importer: Purge any redundant entries" - -This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6. - -The query stalls the database and therefore the automatic -scripts are no longer able to generate a new version of the -database. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 22 +--------------------- - 1 file changed, 1 insertion(+), 21 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 1467923..e3a07a0 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -374,27 +374,7 @@ class CLI(object): - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums - JOIN _organizations ON _autnums.organization = _organizations.handle -- ON CONFLICT (number) DO UPDATE SET name = excluded.name -- """) -- -- self.db.execute(""" -- --- Purge any redundant entries -- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP -- AS -- SELECT network FROM networks candidates -- WHERE EXISTS ( -- SELECT FROM networks -- WHERE -- networks.network << candidates.network -- AND -- networks.country = candidates.country -- ); -- -- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); -- -- DELETE FROM networks WHERE EXISTS ( -- SELECT FROM _garbage WHERE networks.network = _garbage.network -- ); -+ ON CONFLICT (number) DO UPDATE SET name = excluded.name; - """) - - # Download all extended sources --- -2.20.1 - -From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 24 Sep 2020 10:18:58 +0000 -Subject: [PATCH 002/111] Revert "importer: Import raw sources for inetnum's - again" - -This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 --------------------------------- - 2 files changed, 7 insertions(+), 70 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..de20f37 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) - - class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e3a07a0..77952f2 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -393,10 +393,6 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -- # inetnum -- if line.startswith("inet6num:") or line.startswith("inetnum:"): -- return self._parse_inetnum_block(block) -- - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -426,65 +422,6 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) - -- def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -- -- inetnum = {} -- for line in block: -- logging.debug(line) -- -- # Split line -- key, val = split_line(line) -- -- if key == "inetnum": -- start_address, delim, end_address = val.partition("-") -- -- # Strip any excess space -- start_address, end_address = start_address.rstrip(), end_address.strip() -- -- # Convert to IP address -- try: -- start_address = ipaddress.ip_address(start_address) -- end_address = ipaddress.ip_address(end_address) -- except ValueError: -- logging.warning("Could not parse line: %s" % line) -- return -- -- # Set prefix to default -- prefix = 32 -- -- # Count number of addresses in this subnet -- num_addresses = int(end_address) - int(start_address) -- if num_addresses: -- prefix -= math.log(num_addresses, 2) -- -- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) -- -- elif key == "inet6num": -- inetnum[key] = val -- -- elif key == "country": -- if val == "UNITED STATES": -- val = "US" -- -- inetnum[key] = val.upper() -- -- # Skip empty objects -- if not inetnum: -- return -- -- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) -- -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -- return -- -- self.db.execute("INSERT INTO networks(network, country) \ -- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -- "%s" % network, inetnum.get("country"), -- ) -- - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Mon, 12 Oct 2020 20:53:31 +0000 -Subject: [PATCH 003/111] Revert "Revert "importer: Import raw sources for - inetnum's again"" - -This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ - 2 files changed, 70 insertions(+), 7 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index de20f37..f19db4b 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) +diff --git a/Makefile.am b/Makefile.am +index a0431a6..ebd7e17 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -91,11 +91,14 @@ EXTRA_DIST += \ + pkginclude_HEADERS = \ + src/loc/libloc.h \ + src/loc/as.h \ ++ src/loc/as-list.h \ + src/loc/compat.h \ + src/loc/country.h \ ++ src/loc/country-list.h \ + src/loc/database.h \ + src/loc/format.h \ + src/loc/network.h \ ++ src/loc/network-list.h \ + src/loc/private.h \ + src/loc/stringpool.h \ + src/loc/resolv.h \ +@@ -107,9 +110,12 @@ lib_LTLIBRARIES = \ + src_libloc_la_SOURCES = \ + src/libloc.c \ + src/as.c \ ++ src/as-list.c \ + src/country.c \ ++ src/country-list.c \ + src/database.c \ + src/network.c \ ++ src/network-list.c \ + src/resolv.c \ + src/stringpool.c \ + src/writer.c +@@ -312,6 +318,7 @@ check_PROGRAMS = \ + src/test-database \ + src/test-as \ + src/test-network \ ++ src/test-network-list \ + src/test-country \ + src/test-signature
- class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 77952f2..e3a07a0 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -393,6 +393,10 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -+ # inetnum -+ if line.startswith("inet6num:") or line.startswith("inetnum:"): -+ return self._parse_inetnum_block(block) -+ - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -422,6 +426,65 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) +@@ -351,6 +358,15 @@ src_test_network_CFLAGS = \ + src_test_network_LDADD = \ + src/libloc.la
-+ def _parse_inetnum_block(self, block): -+ logging.debug("Parsing inetnum block:") ++src_test_network_list_SOURCES = \ ++ src/test-network-list.c + -+ inetnum = {} -+ for line in block: -+ logging.debug(line) ++src_test_network_list_CFLAGS = \ ++ $(TESTS_CFLAGS) + -+ # Split line -+ key, val = split_line(line) ++src_test_network_list_LDADD = \ ++ src/libloc.la + -+ if key == "inetnum": -+ start_address, delim, end_address = val.partition("-") + src_test_stringpool_SOURCES = \ + src/test-stringpool.c + +diff --git a/configure.ac b/configure.ac +index 2364dfd..012d8ca 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1,6 +1,6 @@ + AC_PREREQ(2.60) + AC_INIT([libloc], +- [0.9.4], ++ [0.9.5], + [location@lists.ipfire.org], + [libloc], + [https://location.ipfire.org/]) +diff --git a/src/.gitignore b/src/.gitignore +index caf80b5..3ccbdb8 100644 +--- a/src/.gitignore ++++ b/src/.gitignore +@@ -10,5 +10,6 @@ test-libloc + test-database + test-country + test-network ++test-network-list + test-signature + test-stringpool +diff --git a/src/as-list.c b/src/as-list.c +new file mode 100644 +index 0000000..5acbb8a +--- /dev/null ++++ b/src/as-list.c +@@ -0,0 +1,161 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet + -+ # Strip any excess space -+ start_address, end_address = start_address.rstrip(), end_address.strip() ++ Copyright (C) 2020 IPFire Development Team info@ipfire.org + -+ # Convert to IP address -+ try: -+ start_address = ipaddress.ip_address(start_address) -+ end_address = ipaddress.ip_address(end_address) -+ except ValueError: -+ logging.warning("Could not parse line: %s" % line) -+ return ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. + -+ # Set prefix to default -+ prefix = 32 ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ # Count number of addresses in this subnet -+ num_addresses = int(end_address) - int(start_address) -+ if num_addresses: -+ prefix -= math.log(num_addresses, 2) ++#include <errno.h> ++#include <stdlib.h> + -+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) ++#include <loc/as.h> ++#include <loc/as-list.h> ++#include <loc/private.h> + -+ elif key == "inet6num": -+ inetnum[key] = val ++struct loc_as_list { ++ struct loc_ctx* ctx; ++ int refcount; + -+ elif key == "country": -+ if val == "UNITED STATES": -+ val = "US" ++ struct loc_as** elements; ++ size_t elements_size; + -+ inetnum[key] = val.upper() ++ size_t size; ++}; + -+ # Skip empty objects -+ if not inetnum: -+ return ++static int loc_as_list_grow(struct loc_as_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); + -+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) ++ struct loc_as** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; + -+ # Bail out in case we have processed a non-public IP network -+ if network.is_private: -+ logging.warning("Skipping non-globally routable network: %s" % network) -+ return ++ list->elements = elements; ++ list->elements_size += size; + -+ self.db.execute("INSERT INTO networks(network, country) \ -+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -+ "%s" % network, inetnum.get("country"), -+ ) ++ return 0; ++} + - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Mon, 12 Oct 2020 20:53:32 +0000 -Subject: [PATCH 004/111] location-importer.in: only import relevant data from - AFRINIC, APNIC and RIPE -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -In contrast to ARIN and LACNIC, we are able to process more detailled -feeds from those RIRs, avoiding storage of obviously unnecessary data. - -Thanks to various SQL optimisations, doing so now takes less time than -the first version of this did. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- - 1 file changed, 87 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e3a07a0..093f325 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -165,6 +165,7 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - - -- overrides -@@ -363,6 +364,16 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); ++LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, ++ struct loc_as_list** list) { ++ struct loc_as_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; + -+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) -+ ON COMMIT DROP; -+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); -+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -+ """) ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; + -+ # Remove all previously imported content -+ self.db.execute(""" -+ TRUNCATE TABLE networks; - """) - - for source in location.importer.WHOIS_SOURCES: -@@ -370,6 +381,67 @@ class CLI(object): - for block in f: - self._parse_block(block) - -+ # Process all parsed networks from every RIR we happen to have access to, -+ # insert the largest network chunks into the networks table immediately... -+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") ++ DEBUG(l->ctx, "AS list allocated at %p\n", l); ++ *list = l; + -+ for family in (row.family for row in families): -+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) ++ return 0; ++} + -+ self.db.execute("INSERT INTO networks(network, country) \ -+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) ++LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { ++ list->refcount++; + -+ # ... determine any other prefixes for this network family, ... -+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ -+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) ++ return list; ++} + -+ # ... and insert networks with this prefix in case they provide additional -+ # information (i. e. subnet of a larger chunk with a different country) -+ for prefix in (row.prefix for row in prefixes): -+ self.db.execute(""" -+ WITH candidates AS ( -+ SELECT -+ _rirdata.network, -+ _rirdata.country -+ FROM -+ _rirdata -+ WHERE -+ family(_rirdata.network) = %s -+ AND -+ masklen(_rirdata.network) = %s -+ ), -+ filtered AS ( -+ SELECT -+ DISTINCT ON (c.network) -+ c.network, -+ c.country, -+ masklen(networks.network), -+ networks.country AS parent_country -+ FROM -+ candidates c -+ LEFT JOIN -+ networks -+ ON -+ c.network << networks.network -+ ORDER BY -+ c.network, -+ masklen(networks.network) DESC NULLS LAST -+ ) -+ INSERT INTO -+ networks(network, country) -+ SELECT -+ network, -+ country -+ FROM -+ filtered -+ WHERE -+ parent_country IS NULL -+ OR -+ country <> parent_country -+ ON CONFLICT DO NOTHING""", -+ family, prefix, -+ ) ++static void loc_as_list_free(struct loc_as_list* list) { ++ DEBUG(list->ctx, "Releasing AS list at %p\n", list); + - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums -@@ -470,17 +542,30 @@ class CLI(object): - inetnum[key] = val.upper() - - # Skip empty objects -- if not inetnum: -+ if not inetnum or not "country" in inetnum: - return - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -+ # Bail out in case we have processed a network covering the entire IP range, which -+ # is necessary to work around faulty (?) IPv6 network processing -+ if network.prefixlen == 0: -+ logging.warning("Skipping network covering the entire IP adress range: %s" % network) -+ return ++ loc_as_list_clear(list); + -+ # Bail out in case we have processed a network whose prefix length indicates it is -+ # not globally routable (we have decided not to process them at the moment, as they -+ # significantly enlarge our database without providing very helpful additional information) -+ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return ++ loc_unref(list->ctx); ++ free(list); ++} + - # Bail out in case we have processed a non-public IP network - if network.is_private: - logging.warning("Skipping non-globally routable network: %s" % network) - return - -- self.db.execute("INSERT INTO networks(network, country) \ -+ self.db.execute("INSERT INTO _rirdata(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % network, inetnum.get("country"), - ) --- -2.20.1 - -From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 16 Oct 2020 12:24:58 +0000 -Subject: [PATCH 005/111] location-importer: Include all overridden networks - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 093f325..d249a35 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -240,6 +240,8 @@ class CLI(object): - SELECT network FROM announcements - UNION - SELECT network FROM networks -+ UNION -+ SELECT network FROM network_overrides - ORDER BY network - ) - --- -2.20.1 - -From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 16 Oct 2020 12:26:38 +0000 -Subject: [PATCH 006/111] Revert "location-importer.in: only import relevant - data from AFRINIC, APNIC and RIPE" - -This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 89 +-------------------------------- - 1 file changed, 2 insertions(+), 87 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index d249a35..b220eaf 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -165,7 +165,6 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -- CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - - -- overrides -@@ -366,16 +365,6 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); -- -- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) -- ON COMMIT DROP; -- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); -- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -- """) -- -- # Remove all previously imported content -- self.db.execute(""" -- TRUNCATE TABLE networks; - """) - - for source in location.importer.WHOIS_SOURCES: -@@ -383,67 +372,6 @@ class CLI(object): - for block in f: - self._parse_block(block) - -- # Process all parsed networks from every RIR we happen to have access to, -- # insert the largest network chunks into the networks table immediately... -- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") -- -- for family in (row.family for row in families): -- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) -- -- self.db.execute("INSERT INTO networks(network, country) \ -- SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) -- -- # ... determine any other prefixes for this network family, ... -- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ -- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) -- -- # ... and insert networks with this prefix in case they provide additional -- # information (i. e. subnet of a larger chunk with a different country) -- for prefix in (row.prefix for row in prefixes): -- self.db.execute(""" -- WITH candidates AS ( -- SELECT -- _rirdata.network, -- _rirdata.country -- FROM -- _rirdata -- WHERE -- family(_rirdata.network) = %s -- AND -- masklen(_rirdata.network) = %s -- ), -- filtered AS ( -- SELECT -- DISTINCT ON (c.network) -- c.network, -- c.country, -- masklen(networks.network), -- networks.country AS parent_country -- FROM -- candidates c -- LEFT JOIN -- networks -- ON -- c.network << networks.network -- ORDER BY -- c.network, -- masklen(networks.network) DESC NULLS LAST -- ) -- INSERT INTO -- networks(network, country) -- SELECT -- network, -- country -- FROM -- filtered -- WHERE -- parent_country IS NULL -- OR -- country <> parent_country -- ON CONFLICT DO NOTHING""", -- family, prefix, -- ) -- - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums -@@ -544,30 +472,17 @@ class CLI(object): - inetnum[key] = val.upper() - - # Skip empty objects -- if not inetnum or not "country" in inetnum: -+ if not inetnum: - return - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -- # Bail out in case we have processed a network covering the entire IP range, which -- # is necessary to work around faulty (?) IPv6 network processing -- if network.prefixlen == 0: -- logging.warning("Skipping network covering the entire IP adress range: %s" % network) -- return -- -- # Bail out in case we have processed a network whose prefix length indicates it is -- # not globally routable (we have decided not to process them at the moment, as they -- # significantly enlarge our database without providing very helpful additional information) -- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -- logging.info("Skipping network too small to be publicly announced: %s" % network) -- return -- - # Bail out in case we have processed a non-public IP network - if network.is_private: - logging.warning("Skipping non-globally routable network: %s" % network) - return - -- self.db.execute("INSERT INTO _rirdata(network, country) \ -+ self.db.execute("INSERT INTO networks(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % network, inetnum.get("country"), - ) --- -2.20.1 - -From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 16 Oct 2020 12:26:43 +0000 -Subject: [PATCH 007/111] Revert "Revert "Revert "importer: Import raw sources - for inetnum's again""" - -This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 --------------------------------- - 2 files changed, 7 insertions(+), 70 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..de20f37 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) - - class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index b220eaf..e87d378 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -395,10 +395,6 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -- # inetnum -- if line.startswith("inet6num:") or line.startswith("inetnum:"): -- return self._parse_inetnum_block(block) -- - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -428,65 +424,6 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) - -- def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -- -- inetnum = {} -- for line in block: -- logging.debug(line) -- -- # Split line -- key, val = split_line(line) -- -- if key == "inetnum": -- start_address, delim, end_address = val.partition("-") -- -- # Strip any excess space -- start_address, end_address = start_address.rstrip(), end_address.strip() -- -- # Convert to IP address -- try: -- start_address = ipaddress.ip_address(start_address) -- end_address = ipaddress.ip_address(end_address) -- except ValueError: -- logging.warning("Could not parse line: %s" % line) -- return -- -- # Set prefix to default -- prefix = 32 -- -- # Count number of addresses in this subnet -- num_addresses = int(end_address) - int(start_address) -- if num_addresses: -- prefix -= math.log(num_addresses, 2) -- -- inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) -- -- elif key == "inet6num": -- inetnum[key] = val -- -- elif key == "country": -- if val == "UNITED STATES": -- val = "US" -- -- inetnum[key] = val.upper() -- -- # Skip empty objects -- if not inetnum: -- return -- -- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) -- -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -- return -- -- self.db.execute("INSERT INTO networks(network, country) \ -- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -- "%s" % network, inetnum.get("country"), -- ) -- - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 20 Oct 2020 20:44:43 +0000 -Subject: [PATCH 008/111] as: Fix dereferencing NULL pointer when setting AS - name - -Reported-by: Gisle Vanem gisle.vanem@gmail.com -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/as.c | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - ++LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { ++ if (!list) ++ return NULL; ++ ++ if (--list->refcount > 0) ++ return list; ++ ++ loc_as_list_free(list); ++ return NULL; ++} ++ ++LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { ++ return list->size; ++} ++ ++LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { ++ return list->size == 0; ++} ++ ++LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { ++ if (!list->elements) ++ return; ++ ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_as_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements = NULL; ++ list->elements_size = 0; ++ ++ list->size = 0; ++} ++ ++LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; ++ ++ return loc_as_ref(list->elements[index]); ++} ++ ++LOC_EXPORT int loc_as_list_append( ++ struct loc_as_list* list, struct loc_as* as) { ++ if (loc_as_list_contains(list, as)) ++ return 0; ++ ++ // Check if we have space left ++ if (list->size >= list->elements_size) { ++ int r = loc_as_list_grow(list, 64); ++ if (r) ++ return r; ++ } ++ ++ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); ++ ++ list->elements[list->size++] = loc_as_ref(as); ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_as_cmp(as, list->elements[i]) == 0) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number) { ++ struct loc_as* as; ++ ++ int r = loc_as_new(list->ctx, &as, number); ++ if (r) ++ return -1; ++ ++ r = loc_as_list_contains(list, as); ++ loc_as_unref(as); ++ ++ return r; ++} diff --git a/src/as.c b/src/as.c -index e1fbb01..8421ac8 100644 +index e1fbb01..757bf3d 100644 --- a/src/as.c +++ b/src/as.c @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) { @@ -846,24 +263,7 @@ index e1fbb01..8421ac8 100644
return 0; } --- -2.20.1 - -From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 09:18:08 +0000 -Subject: [PATCH 009/111] as: Do not attempt to match name when it wasn't set - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/as.c | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/src/as.c b/src/as.c -index 8421ac8..757bf3d 100644 ---- a/src/as.c -+++ b/src/as.c -@@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) { +@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) { if (!string) return 1;
@@ -874,4414 +274,857 @@ index 8421ac8..757bf3d 100644 // Search if string is in name if (strcasestr(as->name, string) != NULL) return 1; --- -2.20.1 - -From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 09:28:39 +0000 -Subject: [PATCH 010/111] writer: Free array with pointer to ASes, too - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/writer.c | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/src/writer.c b/src/writer.c -index 5939cff..160650f 100644 ---- a/src/writer.c -+++ b/src/writer.c -@@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) { - EVP_PKEY_free(writer->private_key2); - - // Unref all AS -- for (unsigned int i = 0; i < writer->as_count; i++) { -- loc_as_unref(writer->as[i]); -+ if (writer->as) { -+ for (unsigned int i = 0; i < writer->as_count; i++) { -+ loc_as_unref(writer->as[i]); -+ } -+ free(writer->as); - } - - // Release network tree --- -2.20.1 - -From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 09:31:29 +0000 -Subject: [PATCH 011/111] writer: Free countries when the writer is being - destroyed - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/writer.c | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/writer.c b/src/writer.c -index 160650f..2f09b56 100644 ---- a/src/writer.c -+++ b/src/writer.c -@@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) { - free(writer->as); - } - -+ // Unref all countries -+ if (writer->countries) { -+ for (unsigned int i = 0; i < writer->countries_count; i++) { -+ loc_country_unref(writer->countries[i]); -+ } -+ free(writer->countries); -+ } -+ - // Release network tree - if (writer->networks) - loc_network_tree_unref(writer->networks); --- -2.20.1 - -From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 13:19:44 +0000 -Subject: [PATCH 012/111] tests: Try adding an invalid network - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/test-network.c | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/test-network.c b/src/test-network.c -index d38f13d..e908b57 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -160,6 +160,14 @@ int main(int argc, char** argv) { - // Set ASN - loc_network_set_asn(network4, 1024); - -+ // Try adding an invalid network -+ struct loc_network* network; -+ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); --- -2.20.1 - -From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 13:36:35 +0000 -Subject: [PATCH 013/111] networks: Improve parsing IP addresses - -loc_network_new_from_string() seem to have had some unexpected -behaviour for invalid inputs. - -The function has been tidied up slightly and returns as soon as -some invalid input was detected. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 52 +++++++++++++++++++++++++++++++--------------- - src/test-network.c | 7 +++++++ - 2 files changed, 42 insertions(+), 17 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 366caa2..c112a41 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network - LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, - const char* address_string) { - struct in6_addr first_address; -- unsigned int prefix = 0; - char* prefix_string; -- int r = 1; -+ int r = -EINVAL; -+ -+ DEBUG(ctx, "Attempting to parse network %s\n", address_string); - - // Make a copy of the string to work on it - char* buffer = strdup(address_string); -@@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - // Split address and prefix - address_string = strsep(&prefix_string, "/"); - -- // Did we find a prefix? -- if (prefix_string) { -- // Convert prefix to integer -- prefix = strtol(prefix_string, NULL, 10); -+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); - -- if (prefix) { -- // Parse the address -- r = loc_parse_address(ctx, address_string, &first_address); -+ // We need to have a prefix -+ if (!prefix_string) { -+ DEBUG(ctx, "No prefix set\n"); -+ goto FAIL; -+ } - -- // Map the prefix to IPv6 if needed -- if (IN6_IS_ADDR_V4MAPPED(&first_address)) -- prefix += 96; -- } -+ // Convert prefix to integer -+ unsigned int prefix = strtol(prefix_string, NULL, 10); -+ -+ // End if the prefix was invalid -+ if (!prefix) { -+ DEBUG(ctx, "The prefix is zero or not a number\n"); -+ goto FAIL; -+ } -+ -+ // Parse the address -+ r = loc_parse_address(ctx, address_string, &first_address); -+ if (r) { -+ DEBUG(ctx, "The address could not be parsed\n"); -+ goto FAIL; - } - -+ // Map the prefix to IPv6 if needed -+ if (IN6_IS_ADDR_V4MAPPED(&first_address)) -+ prefix += 96; -+ -+FAIL: - // Free temporary buffer - free(buffer); - -- if (r == 0) { -- r = loc_network_new(ctx, network, &first_address, prefix); -- } -+ // Exit if the parsing was unsuccessful -+ if (r) -+ return r; -+ -+ DEBUG(ctx, "GOT HERE\n"); - -- return r; -+ // Create a new network -+ return loc_network_new(ctx, network, &first_address, prefix); - } - - LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { -diff --git a/src/test-network.c b/src/test-network.c -index e908b57..85eca00 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -168,6 +168,13 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Try adding a single address -+ err = loc_writer_add_network(writer, &network, "2001:db8::"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); --- -2.20.1 - -From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 13:43:21 +0000 -Subject: [PATCH 014/111] networks: Test if we can add localhost (IPv6) - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/test-network.c | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/src/test-network.c b/src/test-network.c -index 85eca00..8c7e898 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -175,6 +175,13 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Try adding localhost -+ err = loc_writer_add_network(writer, &network, "::1/128"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); --- -2.20.1 - -From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 13:44:50 +0000 -Subject: [PATCH 015/111] networks: Remove accidentially committed debug line - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index c112a41..be88d75 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -208,8 +208,6 @@ FAIL: - if (r) - return r; - -- DEBUG(ctx, "GOT HERE\n"); -- - // Create a new network - return loc_network_new(ctx, network, &first_address, prefix); - } --- -2.20.1 - -From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 13:53:36 +0000 -Subject: [PATCH 016/111] importer: Add search index to announcements table - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e87d378..d0fe5a6 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -152,6 +152,7 @@ class CLI(object): - last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); - CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); - CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); -+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); - - -- autnums - CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); --- -2.20.1 - -From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 13:54:45 +0000 -Subject: [PATCH 017/111] importer: Add search index to network_overrides table - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index d0fe5a6..fe21d73 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -189,6 +189,8 @@ class CLI(object): - ); - CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network - ON network_overrides(network); -+ CREATE INDEX IF NOT EXISTS network_overrides_search -+ ON network_overrides USING GIST(network inet_ops); - """) - - return db --- -2.20.1 - -From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 21 Oct 2020 16:01:57 +0000 -Subject: [PATCH 018/111] importer: Restructure SQL query to be executed in - parallel - -There are no functional changes, this just runs quicker now. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 67 ++++++++++++++++++--------------- - 1 file changed, 37 insertions(+), 30 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index fe21d73..3c1e5e2 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -237,34 +237,24 @@ class CLI(object): - - # Select all known networks - rows = self.db.query(""" -- -- Get a (sorted) list of all known networks -- WITH known_networks AS ( -- SELECT network FROM announcements -- UNION -- SELECT network FROM networks -- UNION -- SELECT network FROM network_overrides -- ORDER BY network -- ) -- - -- Return a list of those networks enriched with all - -- other information that we store in the database - SELECT -- DISTINCT ON (known_networks.network) -- known_networks.network AS network, -- announcements.autnum AS autnum, -+ DISTINCT ON (network) -+ network, -+ autnum, - - -- Country - COALESCE( - ( - SELECT country FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT country FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - networks.country - ) AS country, -@@ -273,50 +263,67 @@ class CLI(object): - COALESCE( - ( - SELECT is_anonymous_proxy FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anonymous_proxy FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE - ) AS is_anonymous_proxy, - COALESCE( - ( - SELECT is_satellite_provider FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_satellite_provider FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE - ) AS is_satellite_provider, - COALESCE( - ( - SELECT is_anycast FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anycast FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE -- ) AS is_anycast, -- -- -- Must be part of returned values for ORDER BY clause -- masklen(announcements.network) AS sort_a, -- masklen(networks.network) AS sort_b -- FROM known_networks -- LEFT JOIN announcements ON known_networks.network <<= announcements.network -- LEFT JOIN networks ON known_networks.network <<= networks.network -- ORDER BY known_networks.network, sort_a DESC, sort_b DESC -+ ) AS is_anycast -+ FROM ( -+ SELECT -+ known_networks.network AS network, -+ announcements.autnum AS autnum, -+ networks.country AS country, -+ -+ -- Must be part of returned values for ORDER BY clause -+ masklen(announcements.network) AS sort_a, -+ masklen(networks.network) AS sort_b -+ FROM ( -+ SELECT network FROM announcements -+ UNION ALL -+ SELECT network FROM networks -+ UNION ALL -+ SELECT network FROM network_overrides -+ ) known_networks -+ LEFT JOIN -+ announcements ON known_networks.network <<= announcements.network -+ LEFT JOIN -+ networks ON known_networks.network <<= networks.network -+ ORDER BY -+ known_networks.network, -+ sort_a DESC, -+ sort_b DESC -+ ) networks - """) - - for row in rows: --- -2.20.1 - -From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 22 Oct 2020 12:24:34 +0000 -Subject: [PATCH 019/111] network: Allow adding single IP addresses and - automatically add the prefix - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 33 +++++++++++++++------------------ - src/test-network.c | 4 ++-- - 2 files changed, 17 insertions(+), 20 deletions(-) - -diff --git a/src/network.c b/src/network.c -index be88d75..d7b1645 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - const char* address_string) { - struct in6_addr first_address; - char* prefix_string; -+ unsigned int prefix = 128; - int r = -EINVAL; - - DEBUG(ctx, "Attempting to parse network %s\n", address_string); -@@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - - DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); - -- // We need to have a prefix -- if (!prefix_string) { -- DEBUG(ctx, "No prefix set\n"); -- goto FAIL; -- } -- -- // Convert prefix to integer -- unsigned int prefix = strtol(prefix_string, NULL, 10); -- -- // End if the prefix was invalid -- if (!prefix) { -- DEBUG(ctx, "The prefix is zero or not a number\n"); -- goto FAIL; -- } -- - // Parse the address - r = loc_parse_address(ctx, address_string, &first_address); - if (r) { -@@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - goto FAIL; - } - -- // Map the prefix to IPv6 if needed -- if (IN6_IS_ADDR_V4MAPPED(&first_address)) -- prefix += 96; -+ // If a prefix was given, we will try to parse it -+ if (prefix_string) { -+ // Convert prefix to integer -+ prefix = strtol(prefix_string, NULL, 10); -+ -+ if (!prefix) { -+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); -+ goto FAIL; -+ } -+ -+ // Map the prefix to IPv6 if needed -+ if (IN6_IS_ADDR_V4MAPPED(&first_address)) -+ prefix += 96; -+ } - - FAIL: - // Free temporary buffer -diff --git a/src/test-network.c b/src/test-network.c -index 8c7e898..b6776b4 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -170,8 +170,8 @@ int main(int argc, char** argv) { - - // Try adding a single address - err = loc_writer_add_network(writer, &network, "2001:db8::"); -- if (err != -EINVAL) { -- fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); -+ if (err) { -+ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); - exit(EXIT_FAILURE); - } - --- -2.20.1 - -From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Wed, 21 Oct 2020 14:47:36 +0000 -Subject: [PATCH 020/111] Revert "Revert "Revert "Revert "importer: Import raw - sources for inetnum's again"""" -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173. - -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/importer.py | 14 ++++---- - src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++ - 2 files changed, 70 insertions(+), 7 deletions(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index de20f37..f19db4b 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -30,8 +30,8 @@ WHOIS_SOURCES = ( - "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz", - - # Asia Pacific Network Information Centre -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", -+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", - #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz", - "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", -@@ -45,8 +45,8 @@ WHOIS_SOURCES = ( - # XXX ??? - - # Réseaux IP Européens -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz", -+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz", - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz", - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz", -@@ -55,10 +55,10 @@ WHOIS_SOURCES = ( - - EXTENDED_SOURCES = ( - # African Network Information Centre -- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", -+ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest", - - # Asia Pacific Network Information Centre -- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", -+ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest", - - # American Registry for Internet Numbers - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -@@ -67,7 +67,7 @@ EXTENDED_SOURCES = ( - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens -- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", - ) - - class Downloader(object): -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 3c1e5e2..e8a4fc5 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -405,6 +405,10 @@ class CLI(object): - if line.startswith("aut-num:"): - return self._parse_autnum_block(block) - -+ # inetnum -+ if line.startswith("inet6num:") or line.startswith("inetnum:"): -+ return self._parse_inetnum_block(block) -+ - # organisation - elif line.startswith("organisation:"): - return self._parse_org_block(block) -@@ -434,6 +438,65 @@ class CLI(object): - autnum.get("asn"), autnum.get("org"), - ) - -+ def _parse_inetnum_block(self, block): -+ logging.debug("Parsing inetnum block:") -+ -+ inetnum = {} -+ for line in block: -+ logging.debug(line) -+ -+ # Split line -+ key, val = split_line(line) -+ -+ if key == "inetnum": -+ start_address, delim, end_address = val.partition("-") -+ -+ # Strip any excess space -+ start_address, end_address = start_address.rstrip(), end_address.strip() -+ -+ # Convert to IP address -+ try: -+ start_address = ipaddress.ip_address(start_address) -+ end_address = ipaddress.ip_address(end_address) -+ except ValueError: -+ logging.warning("Could not parse line: %s" % line) -+ return -+ -+ # Set prefix to default -+ prefix = 32 -+ -+ # Count number of addresses in this subnet -+ num_addresses = int(end_address) - int(start_address) -+ if num_addresses: -+ prefix -= math.log(num_addresses, 2) -+ -+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) -+ -+ elif key == "inet6num": -+ inetnum[key] = val -+ -+ elif key == "country": -+ if val == "UNITED STATES": -+ val = "US" -+ -+ inetnum[key] = val.upper() -+ -+ # Skip empty objects -+ if not inetnum: -+ return -+ -+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) -+ -+ # Bail out in case we have processed a non-public IP network -+ if network.is_private: -+ logging.warning("Skipping non-globally routable network: %s" % network) -+ return -+ -+ self.db.execute("INSERT INTO networks(network, country) \ -+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", -+ "%s" % network, inetnum.get("country"), -+ ) -+ - def _parse_org_block(self, block): - org = {} - for line in block: --- -2.20.1 - -From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Wed, 21 Oct 2020 14:47:37 +0000 -Subject: [PATCH 021/111] Revert "Revert "location-importer.in: only import - relevant data from AFRINIC, APNIC and RIPE"" -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc. - -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++- - 1 file changed, 87 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index e8a4fc5..5656c41 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -166,6 +166,7 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - - -- overrides -@@ -375,6 +376,16 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); -+ -+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) -+ ON COMMIT DROP; -+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); -+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -+ """) -+ -+ # Remove all previously imported content -+ self.db.execute(""" -+ TRUNCATE TABLE networks; - """) - - for source in location.importer.WHOIS_SOURCES: -@@ -382,6 +393,67 @@ class CLI(object): - for block in f: - self._parse_block(block) - -+ # Process all parsed networks from every RIR we happen to have access to, -+ # insert the largest network chunks into the networks table immediately... -+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") -+ -+ for family in (row.family for row in families): -+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) -+ -+ self.db.execute("INSERT INTO networks(network, country) \ -+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) -+ -+ # ... determine any other prefixes for this network family, ... -+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ -+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) -+ -+ # ... and insert networks with this prefix in case they provide additional -+ # information (i. e. subnet of a larger chunk with a different country) -+ for prefix in (row.prefix for row in prefixes): -+ self.db.execute(""" -+ WITH candidates AS ( -+ SELECT -+ _rirdata.network, -+ _rirdata.country -+ FROM -+ _rirdata -+ WHERE -+ family(_rirdata.network) = %s -+ AND -+ masklen(_rirdata.network) = %s -+ ), -+ filtered AS ( -+ SELECT -+ DISTINCT ON (c.network) -+ c.network, -+ c.country, -+ masklen(networks.network), -+ networks.country AS parent_country -+ FROM -+ candidates c -+ LEFT JOIN -+ networks -+ ON -+ c.network << networks.network -+ ORDER BY -+ c.network, -+ masklen(networks.network) DESC NULLS LAST -+ ) -+ INSERT INTO -+ networks(network, country) -+ SELECT -+ network, -+ country -+ FROM -+ filtered -+ WHERE -+ parent_country IS NULL -+ OR -+ country <> parent_country -+ ON CONFLICT DO NOTHING""", -+ family, prefix, -+ ) -+ - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums -@@ -482,17 +554,30 @@ class CLI(object): - inetnum[key] = val.upper() - - # Skip empty objects -- if not inetnum: -+ if not inetnum or not "country" in inetnum: - return - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -+ # Bail out in case we have processed a network covering the entire IP range, which -+ # is necessary to work around faulty (?) IPv6 network processing -+ if network.prefixlen == 0: -+ logging.warning("Skipping network covering the entire IP adress range: %s" % network) -+ return -+ -+ # Bail out in case we have processed a network whose prefix length indicates it is -+ # not globally routable (we have decided not to process them at the moment, as they -+ # significantly enlarge our database without providing very helpful additional information) -+ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return -+ - # Bail out in case we have processed a non-public IP network - if network.is_private: - logging.warning("Skipping non-globally routable network: %s" % network) - return - -- self.db.execute("INSERT INTO networks(network, country) \ -+ self.db.execute("INSERT INTO _rirdata(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % network, inetnum.get("country"), - ) --- -2.20.1 - -From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Wed, 21 Oct 2020 14:47:38 +0000 -Subject: [PATCH 022/111] export.py: fix exporting IP networks for crappy - xt_geoip module -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -In contrast to the location database itself, the xt_geoip module -consumes a list of IP networks for each country, and returns after the -first match. - -We therefore need to... - -(a) sort IP networks by their size, allow as precise matches as possible -(b) export _any_ IP networks - including inverted subnets - to prevent - undefined overlaps -(c) do the entire thing as fast as possible, consuming as less disk - space as possible, which is why we can't just iterate over all /24 - chunks - -Partially fixes: #12499 - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 69 ++++++++++++++++++++++++++++++++++---------- - 1 file changed, 54 insertions(+), 15 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index d15c6f0..5eaf43f 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -39,8 +39,8 @@ class OutputWriter(object): - suffix = "networks" - mode = "w" - -- def __init__(self, f, prefix=None, flatten=True): -- self.f, self.prefix, self.flatten = f, prefix, flatten -+ def __init__(self, db, f, prefix=None, flatten=True): -+ self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten - - # The previously written network - self._last_network = None -@@ -49,13 +49,13 @@ class OutputWriter(object): - self._write_header() - - @classmethod -- def open(cls, filename, **kwargs): -+ def open(cls, db, filename, **kwargs): - """ - Convenience function to open a file - """ - f = open(filename, cls.mode) - -- return cls(f, **kwargs) -+ return cls(db, f, **kwargs) - - def __repr__(self): - return "<%s f=%s>" % (self.__class__.__name__, self.f) -@@ -87,13 +87,31 @@ class OutputWriter(object): - def _write_network(self, network): - self.f.write("%s\n" % network) - -- def write(self, network): -+ def write(self, network, subnets): - if self.flatten and self._flatten(network): - log.debug("Skipping writing network %s" % network) - return - -- # Write the network to file -- self._write_network(network) -+ # Write the network when it has no subnets -+ if not subnets: -+ network = ipaddress.ip_network("%s" % network) -+ return self._write_network(network) -+ -+ # Collect all matching subnets -+ matching_subnets = [] -+ -+ for subnet in sorted(subnets): -+ # Try to find the subnet in the database -+ n = self.db.lookup("%s" % subnet.network_address) -+ -+ # No entry or matching country means those IP addresses belong here -+ if not n or n.country_code == network.country_code: -+ matching_subnets.append(subnet) -+ -+ # Write all networks as compact as possible -+ for network in ipaddress.collapse_addresses(matching_subnets): -+ log.debug("Writing %s to database" % network) -+ self._write_network(network) - - def finish(self): - """ -@@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" - - def _write_network(self, network): -- for address in (network.first_address, network.last_address): -+ for address in (network.network_address, network.broadcast_address): - # Convert this into a string of bits - bytes = socket.inet_pton( -- network.family, address, -+ socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, - ) - - self.f.write(bytes) -@@ -175,7 +193,7 @@ class Exporter(object): - directory, prefix=country_code, suffix=self.writer.suffix, family=family, - ) - -- writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) -+ writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code) - - # Create writers for ASNs - for asn in asns: -@@ -183,22 +201,43 @@ class Exporter(object): - directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, - ) - -- writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) -+ writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) - - # Get all networks that match the family - networks = self.db.search_networks(family=family) - -+ # Materialise the generator into a list (uses quite some memory) -+ networks = list(networks) -+ - # Walk through all networks -- for network in networks: -+ for i, network in enumerate(networks): -+ _network = ipaddress.ip_network("%s" % network) -+ -+ # Search for all subnets -+ subnets = set() -+ -+ while i < len(networks): -+ subnet = networks[i+1] -+ -+ if subnet.is_subnet_of(network): -+ _subnet = ipaddress.ip_network("%s" % subnet) -+ -+ subnets.add(_subnet) -+ subnets.update(_network.address_exclude(_subnet)) -+ -+ i += 1 -+ else: -+ break -+ - # Write matching countries - try: -- writers[network.country_code].write(network) -+ writers[network.country_code].write(network, subnets) - except KeyError: - pass - - # Write matching ASNs - try: -- writers[network.asn].write(network) -+ writers[network.asn].write(network, subnets) - except KeyError: - pass - -@@ -209,7 +248,7 @@ class Exporter(object): - country = flags[flag] - - try: -- writers[country].write(network) -+ writers[country].write(network, subnets) - except KeyError: - pass - --- -2.20.1 - -From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Wed, 21 Oct 2020 14:47:39 +0000 -Subject: [PATCH 023/111] location-importer.in: filter bogus IP networks for - both Whois and extended sources -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Sanity checks for parsed networks have been put into a separate function -to avoid boilerplate code for extended sources. This makes the location -database less vulnerable to garbage written into RIR databases on -purpose or by chance. - -Fixes: #12500 - -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 83 ++++++++++++++++++++++++++------- - 1 file changed, 67 insertions(+), 16 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 5656c41..f24d357 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -469,6 +469,69 @@ class CLI(object): - for line in f: - self._parse_line(line) - -+ def _check_parsed_network(self, network): -+ """ -+ Assistive function to detect and subsequently sort out parsed -+ networks from RIR data (both Whois and so-called "extended sources"), -+ which are or have... -+ -+ (a) not globally routable (RFC 1918 space, et al.) -+ (b) covering a too large chunk of the IP address space (prefix length -+ is < 7 for IPv4 networks, and < 10 for IPv6) -+ (c) "0.0.0.0" or "::" as a network address -+ (d) are too small for being publicly announced (we have decided not to -+ process them at the moment, as they significantly enlarge our -+ database without providing very helpful additional information) -+ -+ This unfortunately is necessary due to brain-dead clutter across -+ various RIR databases, causing mismatches and eventually disruptions. -+ -+ We will return False in case a network is not suitable for adding -+ it to our database, and True otherwise. -+ """ -+ -+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): -+ return False -+ -+ if not network.is_global: -+ logging.warning("Skipping non-globally routable network: %s" % network) -+ return False -+ -+ if network.version == 4: -+ if network.prefixlen < 7: -+ logging.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 24: -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return False -+ -+ if str(network.network_address) == "0.0.0.0": -+ logging.warning("Skipping network based on 0.0.0.0: %s" % network) -+ return False -+ -+ elif network.version == 6: -+ if network.prefixlen < 10: -+ logging.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 48: -+ logging.info("Skipping network too small to be publicly announced: %s" % network) -+ return False -+ -+ if str(network.network_address) == "::": -+ logging.warning("Skipping network based on '::': %s" % network) -+ return False -+ -+ else: -+ # This should not happen... -+ logging.warning("Skipping network of unknown family, this should not happen: %s" % network) -+ return False -+ -+ # In case we have made it here, the network is considered to -+ # be suitable for libloc consumption... -+ return True -+ - def _parse_block(self, block): - # Get first line to find out what type of block this is - line = block[0] -@@ -559,22 +622,7 @@ class CLI(object): - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -- # Bail out in case we have processed a network covering the entire IP range, which -- # is necessary to work around faulty (?) IPv6 network processing -- if network.prefixlen == 0: -- logging.warning("Skipping network covering the entire IP adress range: %s" % network) -- return -- -- # Bail out in case we have processed a network whose prefix length indicates it is -- # not globally routable (we have decided not to process them at the moment, as they -- # significantly enlarge our database without providing very helpful additional information) -- if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6): -- logging.info("Skipping network too small to be publicly announced: %s" % network) -- return -- -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -+ if not self._check_parsed_network(network): - return - - self.db.execute("INSERT INTO _rirdata(network, country) \ -@@ -658,6 +706,9 @@ class CLI(object): - log.warning("Invalid IP address: %s" % address) - return - -+ if not self._check_parsed_network(network): -+ return -+ - self.db.execute("INSERT INTO networks(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO \ - UPDATE SET country = excluded.country", --- -2.20.1 - -From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Wed, 21 Oct 2020 14:47:40 +0000 -Subject: [PATCH 024/111] importer.py: fetch LACNIC data via HTTPS -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/importer.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..5f46bc3 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -64,7 +64,7 @@ EXTENDED_SOURCES = ( - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", - - # Latin America and Caribbean Network Information Centre -- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", -+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", --- -2.20.1 - -From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Wed, 21 Oct 2020 14:47:41 +0000 -Subject: [PATCH 025/111] location-importer.in: omit historic/orphaned RIR data -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Some RIRs include detailled information regarding networks not managed -by or allocated to themselves, particually APNIC. We need to filter -those networks (they usually have a characteristic network name) in -order to prevent operational quirks or returning wrong country codes. - -Fixes: #12501 -Partially fixes: #12499 - -Cc: Michael Tremer michael.tremer@ipfire.org -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 38 +++++++++++++++++++++------------ - 1 file changed, 24 insertions(+), 14 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index f24d357..a869256 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -494,38 +494,38 @@ class CLI(object): - return False - - if not network.is_global: -- logging.warning("Skipping non-globally routable network: %s" % network) -+ log.warning("Skipping non-globally routable network: %s" % network) - return False - - if network.version == 4: - if network.prefixlen < 7: -- logging.warning("Skipping too big IP chunk: %s" % network) -+ log.warning("Skipping too big IP chunk: %s" % network) - return False - - if network.prefixlen > 24: -- logging.info("Skipping network too small to be publicly announced: %s" % network) -+ log.info("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "0.0.0.0": -- logging.warning("Skipping network based on 0.0.0.0: %s" % network) -+ log.warning("Skipping network based on 0.0.0.0: %s" % network) - return False - - elif network.version == 6: - if network.prefixlen < 10: -- logging.warning("Skipping too big IP chunk: %s" % network) -+ log.warning("Skipping too big IP chunk: %s" % network) - return False - - if network.prefixlen > 48: -- logging.info("Skipping network too small to be publicly announced: %s" % network) -+ log.info("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "::": -- logging.warning("Skipping network based on '::': %s" % network) -+ log.warning("Skipping network based on '::': %s" % network) - return False - - else: - # This should not happen... -- logging.warning("Skipping network of unknown family, this should not happen: %s" % network) -+ log.warning("Skipping network of unknown family, this should not happen: %s" % network) - return False - - # In case we have made it here, the network is considered to -@@ -574,15 +574,22 @@ class CLI(object): - ) - - def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -+ log.debug("Parsing inetnum block:") - - inetnum = {} - for line in block: -- logging.debug(line) -+ log.debug(line) - - # Split line - key, val = split_line(line) - -+ # Filter any inetnum records which are only referring to IP space -+ # not managed by that specific RIR... -+ if key == "netname": -+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): -+ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) -+ return -+ - if key == "inetnum": - start_address, delim, end_address = val.partition("-") - -@@ -594,7 +601,7 @@ class CLI(object): - start_address = ipaddress.ip_address(start_address) - end_address = ipaddress.ip_address(end_address) - except ValueError: -- logging.warning("Could not parse line: %s" % line) -+ log.warning("Could not parse line: %s" % line) - return - - # Set prefix to default -@@ -611,15 +618,18 @@ class CLI(object): - inetnum[key] = val - - elif key == "country": -- if val == "UNITED STATES": -- val = "US" -- - inetnum[key] = val.upper() - - # Skip empty objects - if not inetnum or not "country" in inetnum: - return - -+ # Skip objects with bogus country code 'ZZ' -+ if inetnum.get("country") == "ZZ": -+ log.warning("Skipping network with bogus country 'ZZ': %s" % \ -+ (inetnum.get("inet6num") or inetnum.get("inetnum"))) -+ return -+ - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - - if not self._check_parsed_network(network): --- -2.20.1 - -From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Wed, 21 Oct 2020 14:47:43 +0000 -Subject: [PATCH 026/111] location-importer.in: avoid log spam for too small - networks -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index a869256..864eab1 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -503,7 +503,7 @@ class CLI(object): - return False - - if network.prefixlen > 24: -- log.info("Skipping network too small to be publicly announced: %s" % network) -+ log.debug("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "0.0.0.0": -@@ -516,7 +516,7 @@ class CLI(object): - return False - - if network.prefixlen > 48: -- log.info("Skipping network too small to be publicly announced: %s" % network) -+ log.debug("Skipping network too small to be publicly announced: %s" % network) - return False - - if str(network.network_address) == "::": --- -2.20.1 - -From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 27 Oct 2020 17:14:30 +0000 -Subject: [PATCH 027/111] export: Flatten the tree before exporting it - -This patch removes the possibility that any IP address ranges -might show up in multiple exported files. - -If this was content from the database: - - * 10.0.0.0/16 - DE - * 10.0.1.0/24 - FR - -Then the IP address 10.0.1.1 would match for both countries. - -The algorithm will now break the larger /16 subnet apart into -smaller subnets so that 10.0.1.0/24 is no longer overlapped. - -There was some time spent on this to make this as efficient -as possible. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 154 ++++++++++++++++++++++++++++++------------- - 1 file changed, 110 insertions(+), 44 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 5eaf43f..dd44332 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -89,28 +89,55 @@ class OutputWriter(object): - - def write(self, network, subnets): - if self.flatten and self._flatten(network): -- log.debug("Skipping writing network %s" % network) -+ log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) - return - -+ # Convert network into a Python object -+ _network = ipaddress.ip_network("%s" % network) -+ - # Write the network when it has no subnets - if not subnets: -- network = ipaddress.ip_network("%s" % network) -- return self._write_network(network) -+ log.debug("Writing %s to %s" % (_network, self.f)) -+ return self._write_network(_network) -+ -+ # Convert subnets into Python objects -+ _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] -+ -+ # Split the network into smaller bits so that -+ # we can accomodate for any gaps in it later -+ to_check = set() -+ for _subnet in _subnets: -+ to_check.update( -+ _network.address_exclude(_subnet) -+ ) -+ -+ # Clear the list of all subnets -+ subnets = [] -+ -+ # Check if all subnets to not overlap with anything else -+ while to_check: -+ subnet_to_check = to_check.pop() - -- # Collect all matching subnets -- matching_subnets = [] -+ for _subnet in _subnets: -+ # Drop this subnet if it equals one of the subnets -+ # or if it is subnet of one of them -+ if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): -+ break - -- for subnet in sorted(subnets): -- # Try to find the subnet in the database -- n = self.db.lookup("%s" % subnet.network_address) -+ # Break it down if it overlaps -+ if subnet_to_check.overlaps(_subnet): -+ to_check.update( -+ subnet_to_check.address_exclude(_subnet) -+ ) -+ break - -- # No entry or matching country means those IP addresses belong here -- if not n or n.country_code == network.country_code: -- matching_subnets.append(subnet) -+ # Add the subnet again as it passed the check -+ else: -+ subnets.append(subnet_to_check) - - # Write all networks as compact as possible -- for network in ipaddress.collapse_addresses(matching_subnets): -- log.debug("Writing %s to database" % network) -+ for network in ipaddress.collapse_addresses(subnets): -+ log.debug("Writing %s to %s" % (network, self.f)) - self._write_network(network) - - def finish(self): -@@ -206,40 +233,40 @@ class Exporter(object): - # Get all networks that match the family - networks = self.db.search_networks(family=family) - -- # Materialise the generator into a list (uses quite some memory) -- networks = list(networks) -+ # Create a stack with all networks in order where we can put items back -+ # again and retrieve them in the next iteration. -+ networks = BufferedStack(networks) - - # Walk through all networks -- for i, network in enumerate(networks): -- _network = ipaddress.ip_network("%s" % network) -- -- # Search for all subnets -- subnets = set() -- -- while i < len(networks): -- subnet = networks[i+1] -- -- if subnet.is_subnet_of(network): -- _subnet = ipaddress.ip_network("%s" % subnet) -- -- subnets.add(_subnet) -- subnets.update(_network.address_exclude(_subnet)) -- -- i += 1 -- else: -+ for network in networks: -+ # Collect all networks which are a subnet of network -+ subnets = [] -+ for subnet in networks: -+ # If the next subnet was not a subnet, we have to push -+ # it back on the stack and break this loop -+ if not subnet.is_subnet_of(network): -+ networks.push(subnet) - break - -+ subnets.append(subnet) -+ - # Write matching countries -- try: -- writers[network.country_code].write(network, subnets) -- except KeyError: -- pass -+ if network.country_code and network.country_code in writers: -+ # Mismatching subnets -+ gaps = [ -+ subnet for subnet in subnets if not network.country_code == subnet.country_code -+ ] -+ -+ writers[network.country_code].write(network, gaps) - - # Write matching ASNs -- try: -- writers[network.asn].write(network, subnets) -- except KeyError: -- pass -+ if network.asn and network.asn in writers: -+ # Mismatching subnets -+ gaps = [ -+ subnet for subnet in subnets if not network.asn == subnet.asn -+ ] -+ -+ writers[network.asn].write(network, gaps) - - # Handle flags - for flag in flags: -@@ -247,10 +274,19 @@ class Exporter(object): - # Fetch the "fake" country code - country = flags[flag] - -- try: -- writers[country].write(network, subnets) -- except KeyError: -- pass -+ if not country in writers: -+ continue -+ -+ gaps = [ -+ subnet for subnet in subnets -+ if not subnet.has_flag(flag) -+ ] -+ -+ writers[country].write(network, gaps) -+ -+ # Push all subnets back onto the stack -+ for subnet in reversed(subnets): -+ networks.push(subnet) - - # Write everything to the filesystem - for writer in writers.values(): -@@ -262,3 +298,33 @@ class Exporter(object): - ) - - return os.path.join(directory, filename) -+ -+ -+class BufferedStack(object): -+ """ -+ This class takes an iterator and when being iterated -+ over it returns objects from that iterator for as long -+ as there are any. -+ -+ It additionally has a function to put an item back on -+ the back so that it will be returned again at the next -+ iteration. -+ """ -+ def __init__(self, iterator): -+ self.iterator = iterator -+ self.stack = [] -+ -+ def __iter__(self): -+ return self -+ -+ def __next__(self): -+ if self.stack: -+ return self.stack.pop(0) -+ -+ return next(self.iterator) -+ -+ def push(self, elem): -+ """ -+ Takes an element and puts it on the stack -+ """ -+ self.stack.insert(0, elem) --- -2.20.1 - -From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 28 Oct 2020 09:52:36 +0000 -Subject: [PATCH 028/111] location: Fix Python syntax error in verify() - -The database is now being opened before the requested -method is called and handle_verify() wasn't updated. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location.in | 8 +------- - 1 file changed, 1 insertion(+), 7 deletions(-) - -diff --git a/src/python/location.in b/src/python/location.in -index 44ad726..b5e5758 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -453,13 +453,7 @@ class CLI(object): - - return 0 - -- def handle_verify(self, ns): -- try: -- db = location.Database(ns.database) -- except FileNotFoundError as e: -- log.error("%s: %s" % (ns.database, e)) -- return 127 -- -+ def handle_verify(self, db, ns): - # Verify the database - with open(ns.public_key, "r") as f: - if not db.verify(f): --- -2.20.1 - -From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Thu, 29 Oct 2020 07:25:53 -0700 -Subject: [PATCH 029/111] location update: Remove double conversion of - timestamps -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This caused that the timestamp in DNS was misinterpreted -as local time and often, databases could not be downloaded. - -Signed-off-by: Peter Müller peter.mueller@ipfire.org ---- - src/python/location.in | 5 +---- - 1 file changed, 1 insertion(+), 4 deletions(-) - -diff --git a/src/python/location.in b/src/python/location.in -index b5e5758..070640c 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -421,11 +421,8 @@ class CLI(object): - # Fetch the timestamp we need from DNS - t = location.discover_latest_version() - -- # Parse timestamp into datetime format -- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None -- - # Check the version of the local database -- if db and timestamp and db.created_at >= timestamp.timestamp(): -+ if db and t and db.created_at >= t: - log.info("Already on the latest version") - return - --- -2.20.1 - -From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Thu, 29 Oct 2020 07:36:46 -0700 -Subject: [PATCH 030/111] location.in: do not confuse UTC with local time zones -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Signed-off-by: Peter Müller peter.mueller@ipfire.org ---- - src/python/location.in | 12 +++++------- - 1 file changed, 5 insertions(+), 7 deletions(-) - -diff --git a/src/python/location.in b/src/python/location.in -index 070640c..0d09210 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -398,10 +398,7 @@ class CLI(object): - - def handle_update(self, db, ns): - if ns.cron and db: -- now = datetime.datetime.utcnow() -- -- # Parse the database timestamp -- t = datetime.datetime.utcfromtimestamp(db.created_at) -+ now = time.time() - - if ns.cron == "daily": - delta = datetime.timedelta(days=1) -@@ -410,11 +407,12 @@ class CLI(object): - elif ns.cron == "monthly": - delta = datetime.timedelta(days=30) - -+ delta = delta.total_seconds() -+ - # Check if the database has recently been updated -- if t >= (now - delta): -+ if db.created_at >= (now - delta): - log.info( -- _("The database has been updated recently (%s)") % \ -- format_timedelta(now - t), -+ _("The database has been updated recently"), - ) - return 3 - --- -2.20.1 - -From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Peter=20M=C3=BCller?= peter.mueller@ipfire.org -Date: Tue, 3 Nov 2020 15:31:08 +0000 -Subject: [PATCH 031/111] location-importer.in: always convert organisation - handles into upper cases -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Fixes: #12523 - -Signed-off-by: Peter Müller peter.mueller@ipfire.org -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location-importer.in | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 864eab1..2dec89e 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -560,7 +560,7 @@ class CLI(object): - autnum["asn"] = m.group(2) - - elif key == "org": -- autnum[key] = val -+ autnum[key] = val.upper() - - # Skip empty objects - if not autnum: -@@ -646,7 +646,9 @@ class CLI(object): - # Split line - key, val = split_line(line) - -- if key in ("organisation", "org-name"): -+ if key == "organisation": -+ org[key] = val.upper() -+ elif key == "org-name": - org[key] = val - - # Skip empty objects --- -2.20.1 - -From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 11 Nov 2020 21:16:45 +0000 -Subject: [PATCH 032/111] test: Add tests for database enumerator - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 53 insertions(+) - -diff --git a/src/test-database.c b/src/test-database.c -index b4a75c4..4aef94e 100644 ---- a/src/test-database.c -+++ b/src/test-database.c -@@ -38,6 +38,14 @@ const char* DESCRIPTION = - "Maecenas ut venenatis nunc."; - const char* LICENSE = "CC"; - -+const char* networks[] = { -+ "2001:db8::/32", -+ "2001:db8:1000::/48", -+ "2001:db8:2000::/48", -+ "2001:db8:2020::/48", -+ NULL, -+}; -+ - static int attempt_to_open(struct loc_ctx* ctx, char* path) { - FILE* f = fopen(path, "r"); - if (!f) -@@ -139,6 +147,24 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ struct loc_network* network = NULL; -+ -+ // Add some networks -+ const char** n = networks; -+ while (*n) { -+ err = loc_writer_add_network(writer, &network, *n); -+ if (err) { -+ fprintf(stderr, "Could not add network %s\n", *n); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Set a country -+ loc_network_set_country_code(network, "XX"); -+ -+ // Next one -+ n++; -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); -@@ -170,6 +196,33 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Enumerator -+ struct loc_database_enumerator* enumerator; -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); -+ if (err) { -+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Walk through all networks -+ while (1) { -+ err = loc_database_enumerator_next_network(enumerator, &network); -+ if (err) { -+ fprintf(stderr, "Error fetching the next network: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!network) -+ break; -+ -+ char* s = loc_network_str(network); -+ printf("Got network: %s\n", s); -+ free(s); -+ } -+ -+ // Free the enumerator -+ loc_database_enumerator_unref(enumerator); -+ - // Close the database - loc_database_unref(db); - loc_unref(ctx); --- -2.20.1 - -From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 11 Nov 2020 23:01:19 +0000 -Subject: [PATCH 033/111] networks: Add list to manage groups of networks - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 11 ++++++ - src/loc/network.h | 12 ++++++ - src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 119 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index b8296eb..6ef2d27 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -98,6 +98,17 @@ global: - loc_network_str; - loc_network_unref; - -+ # Network List -+ loc_network_list_clear; -+ loc_network_list_empty; -+ loc_network_list_get; -+ loc_network_list_new; -+ loc_network_list_pop; -+ loc_network_list_push; -+ loc_network_list_ref; -+ loc_network_list_size; -+ loc_network_list_unref; -+ - # Writer - loc_writer_add_as; - loc_writer_add_country; -diff --git a/src/loc/network.h b/src/loc/network.h -index 70c3803..fd20812 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - -+// List -+struct loc_network_list; -+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); -+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); -+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); -+size_t loc_network_list_size(struct loc_network_list* list); -+int loc_network_list_empty(struct loc_network_list* list); -+void loc_network_list_clear(struct loc_network_list* list); -+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); -+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); -+struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+ - #ifdef LIBLOC_PRIVATE - - int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); -diff --git a/src/network.c b/src/network.c -index d7b1645..c9e7979 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) - LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { - return loc_network_ref(node->network); - } -+ -+// List -+ -+struct loc_network_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_network* list[1024]; -+ size_t size; -+ size_t max_size; -+}; -+ -+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -+ struct loc_network_list** list) { -+ struct loc_network_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; -+ -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; -+ -+ DEBUG(l->ctx, "Network list allocated at %p\n", l); -+ *list = l; -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_network_list_free(struct loc_network_list* list) { -+ DEBUG(list->ctx, "Releasing network list at %p\n", list); -+ -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_network_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -+ return list->size == 0; -+} -+ -+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ list->size = 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_network_ref(list->list[index]); -+} -+ -+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -+ // Check if we have space left -+ if (list->size == list->max_size) -+ return -ENOMEM; -+ -+ list->list[list->size++] = loc_network_ref(network); -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) -+ return NULL; -+ -+ return list->list[list->size--]; -+} --- -2.20.1 - -From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 13:57:35 +0000 -Subject: [PATCH 034/111] networks: Add function to dump lists - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 14 ++++++++++++++ - 3 files changed, 16 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 6ef2d27..a5641c6 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -100,6 +100,7 @@ global: - - # Network List - loc_network_list_clear; -+ loc_network_list_dump; - loc_network_list_empty; - loc_network_list_get; - loc_network_list_new; -diff --git a/src/loc/network.h b/src/loc/network.h -index fd20812..44c50a4 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); - size_t loc_network_list_size(struct loc_network_list* list); - int loc_network_list_empty(struct loc_network_list* list); - void loc_network_list_clear(struct loc_network_list* list); -+void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -diff --git a/src/network.c b/src/network.c -index c9e7979..0977406 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { - list->size = 0; - } - -+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -+ struct loc_network* network; -+ char* s; -+ -+ for (unsigned int i = 0; i < list->size; i++) { -+ network = list->list[i]; -+ -+ s = loc_network_str(network); -+ -+ INFO(list->ctx, "%s\n", s); -+ free(s); -+ } -+} -+ - LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { - // Check index - if (index >= list->size) --- -2.20.1 - -From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 14:18:40 +0000 -Subject: [PATCH 035/111] network: Add functions to break network into subnets - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 5 + - src/loc/network.h | 6 + - src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++- - src/test-network.c | 61 ++++++++++ - 4 files changed, 361 insertions(+), 1 deletion(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index a5641c6..fcb9ea5 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -80,6 +80,8 @@ global: - - # Network - loc_network_address_family; -+ loc_network_eq; -+ loc_network_exclude; - loc_network_format_first_address; - loc_network_format_last_address; - loc_network_get_asn; -@@ -96,6 +98,7 @@ global: - loc_network_set_country_code; - loc_network_set_flag; - loc_network_str; -+ loc_network_subnets; - loc_network_unref; - - # Network List -@@ -107,7 +110,9 @@ global: - loc_network_list_pop; - loc_network_list_push; - loc_network_list_ref; -+ loc_network_list_reverse; - loc_network_list_size; -+ loc_network_list_sort; - loc_network_list_unref; - - # Writer -diff --git a/src/loc/network.h b/src/loc/network.h -index 44c50a4..4e51a62 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); - int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - -+int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); -+struct loc_network_list* loc_network_subnets(struct loc_network* network); -+struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other); - - // List - struct loc_network_list; -@@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+void loc_network_list_sort(struct loc_network_list* list); -+void loc_network_list_reverse(struct loc_network_list* list); - - #ifdef LIBLOC_PRIVATE - -diff --git a/src/network.c b/src/network.c -index 0977406..6c08070 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s - return a; - } - -+static struct in6_addr address_increment(const struct in6_addr* address) { -+ struct in6_addr a = *address; -+ -+ for (int octet = 15; octet >= 0; octet--) { -+ if (a.s6_addr[octet] < 255) { -+ a.s6_addr[octet]++; -+ break; -+ } else { -+ a.s6_addr[octet] = 0; -+ } -+ } -+ -+ return a; -+} -+ - LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, - struct in6_addr* address, unsigned int prefix) { - // Address cannot be unspecified -@@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - return loc_network_has_flag(network, flag); - } - -+LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { -+#ifdef ENABLE_DEBUG -+ char* n1 = loc_network_str(self); -+ char* n2 = loc_network_str(other); -+ -+ DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); -+ -+ free(n1); -+ free(n2); -+#endif -+ -+ // Family must be the same -+ if (self->family != other->family) { -+ DEBUG(self->ctx, " Family mismatch\n"); -+ -+ return 0; -+ } -+ -+ // The start address must be the same -+ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { -+ DEBUG(self->ctx, " Address mismatch\n"); -+ -+ return 0; -+ } -+ -+ // The prefix length must be the same -+ if (self->prefix != other->prefix) { -+ DEBUG(self->ctx, " Prefix mismatch\n"); -+ return 0; -+ } -+ -+ DEBUG(self->ctx, " Yes!\n"); -+ -+ return 1; -+} -+ -+static int loc_network_gt(struct loc_network* self, struct loc_network* other) { -+ // Families must match -+ if (self->family != other->family) -+ return -1; -+ -+ int r = in6_addr_cmp(&self->first_address, &other->first_address); -+ -+ switch (r) { -+ // Smaller -+ case -1: -+ return 0; -+ -+ // Larger -+ case 1: -+ return 1; -+ -+ default: -+ break; -+ } -+ -+ if (self->prefix > other->prefix) -+ return 1; -+ -+ // Dunno -+ return 0; -+} -+ - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. -@@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net - return 1; - } - -+LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { -+ struct loc_network_list* list; -+ -+ // New prefix length -+ unsigned int prefix = network->prefix + 1; -+ -+ // Check if the new prefix is valid -+ if (valid_prefix(&network->first_address, prefix)) -+ return NULL; -+ -+ // Create a new list with the result -+ int r = loc_network_list_new(network->ctx, &list); -+ if (r) { -+ ERROR(network->ctx, "Could not create network list: %d\n", r); -+ return NULL; -+ } -+ -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; -+ -+ // Create the first half of the network -+ r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix); -+ if (r) -+ goto ERROR; -+ -+ // The next subnet starts after the first one -+ struct in6_addr first_address = address_increment(&subnet1->last_address); -+ -+ // Create the second half of the network -+ r = loc_network_new(network->ctx, &subnet2, &first_address, prefix); -+ if (r) -+ goto ERROR; -+ -+ // Push the both onto the stack (in reverse order) -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ -+ return list; -+ -+ERROR: -+ if (subnet1) -+ loc_network_unref(subnet1); -+ -+ if (subnet2) -+ loc_network_unref(subnet2); -+ -+ if (list) -+ loc_network_list_unref(list); -+ -+ return NULL; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other) { -+ struct loc_network_list* list; -+ -+#ifdef ENABLE_DEBUG -+ char* n1 = loc_network_str(self); -+ char* n2 = loc_network_str(other); -+ -+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); -+ -+ free(n1); -+ free(n2); -+#endif -+ -+ // Family must match -+ if (self->family != other->family) { -+ DEBUG(self->ctx, "Family mismatch\n"); -+ -+ return NULL; -+ } -+ -+ // Other must be a subnet of self -+ if (!loc_network_is_subnet_of(other, self)) { -+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); -+ -+ return NULL; -+ } -+ -+ // We cannot perform this operation if both networks equal -+ if (loc_network_eq(self, other)) { -+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); -+ -+ return NULL; -+ } -+ -+ // Create a new list with the result -+ int r = loc_network_list_new(self->ctx, &list); -+ if (r) { -+ ERROR(self->ctx, "Could not create network list: %d\n", r); -+ return NULL; -+ } -+ -+ struct loc_network_list* subnets = loc_network_subnets(self); -+ -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; -+ -+ while (subnets) { -+ // Fetch both subnets -+ subnet1 = loc_network_list_get(subnets, 0); -+ subnet2 = loc_network_list_get(subnets, 1); -+ -+ // Free list -+ loc_network_list_unref(subnets); -+ subnets = NULL; -+ -+ if (loc_network_eq(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_eq(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_is_subnet_of(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ subnets = loc_network_subnets(subnet1); -+ -+ } else if (loc_network_is_subnet_of(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ subnets = loc_network_subnets(subnet2); -+ -+ } else { -+ ERROR(self->ctx, "We should never get here\n"); -+ goto ERROR; -+ } -+ -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ } -+ -+#ifdef ENABLE_DEBUG -+ loc_network_list_dump(list); -+#endif -+ -+ // Return the result -+ return list; -+ -+ERROR: -+ if (subnet1) -+ loc_network_unref(subnet1); -+ -+ if (subnet2) -+ loc_network_unref(subnet2); -+ -+ if (list) -+ loc_network_list_unref(list); -+ -+ return NULL; -+} -+ - LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { - // Add country code - loc_country_code_copy(dbobj->country_code, network->country_code); -@@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - if (loc_network_list_empty(list)) - return NULL; - -- return list->list[list->size--]; -+ return list->list[--list->size]; -+} -+ -+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -+ // Do nothing for invalid indices -+ if (i1 >= list->size || i2 >= list->size) -+ return; -+ -+ DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); -+ -+ struct loc_network* network1 = list->list[i1]; -+ struct loc_network* network2 = list->list[i2]; -+ -+ list->list[i1] = network2; -+ list->list[i2] = network1; -+} -+ -+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -+ unsigned int i = 0; -+ unsigned int j = list->size - 1; -+ -+ while (i < j) { -+ loc_network_list_swap(list, i++, j--); -+ } -+} -+ -+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -+ unsigned int n = list->size; -+ int swapped; -+ -+ do { -+ swapped = 0; -+ -+ for (unsigned int i = 1; i < n; i++) { -+ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -+ loc_network_list_swap(list, i-1, i); -+ swapped = 1; -+ } -+ } -+ -+ n--; -+ } while (swapped); - } -diff --git a/src/test-network.c b/src/test-network.c -index b6776b4..af1b2e6 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -118,6 +118,19 @@ int main(int argc, char** argv) { - size_t nodes = loc_network_tree_count_nodes(tree); - printf("The tree has %zu nodes\n", nodes); - -+ // Check equals function -+ err = loc_network_eq(network1, network1); -+ if (!err) { -+ fprintf(stderr, "Network is not equal with itself\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_eq(network1, network2); -+ if (err) { -+ fprintf(stderr, "Networks equal unexpectedly\n"); -+ exit(EXIT_FAILURE); -+ } -+ - // Check subnet function - err = loc_network_is_subnet_of(network1, network2); - if (err != 0) { -@@ -131,6 +144,54 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Make list of subnets -+ struct loc_network_list* subnets = loc_network_subnets(network1); -+ if (!subnets) { -+ fprintf(stderr, "Could not find subnets of network\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(subnets); -+ -+ while (!loc_network_list_empty(subnets)) { -+ struct loc_network* subnet = loc_network_list_pop(subnets); -+ if (!subnet) { -+ fprintf(stderr, "Received an empty subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ char* s = loc_network_str(subnet); -+ printf("Received subnet %s\n", s); -+ free(s); -+ -+ if (!loc_network_is_subnet_of(subnet, network1)) { -+ fprintf(stderr, "Not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_unref(subnet); -+ } -+ -+ loc_network_list_unref(subnets); -+ -+ struct loc_network_list* excluded = loc_network_exclude(network1, network2); -+ if (!excluded) { -+ fprintf(stderr, "Could not create excluded list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ // Reverse it -+ loc_network_list_reverse(excluded); -+ loc_network_list_dump(excluded); -+ -+ // Sort them and dump them again -+ loc_network_list_sort(excluded); -+ loc_network_list_dump(excluded); -+ -+ loc_network_list_unref(excluded); -+ - // Create a database - struct loc_writer* writer; - err = loc_writer_new(ctx, &writer, NULL, NULL); --- -2.20.1 - -From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 14:24:58 +0000 -Subject: [PATCH 036/111] networks: Add function to check if two networks - overlap - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 16 ++++++++++++++++ - 3 files changed, 18 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index fcb9ea5..0c2835b 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -93,6 +93,7 @@ global: - loc_network_match_flag; - loc_network_new; - loc_network_new_from_string; -+ loc_network_overlaps; - loc_network_ref; - loc_network_set_asn; - loc_network_set_country_code; -diff --git a/src/loc/network.h b/src/loc/network.h -index 4e51a62..ef13756 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_eq(struct loc_network* self, struct loc_network* other); -+int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - struct loc_network_list* loc_network_subnets(struct loc_network* network); - struct loc_network_list* loc_network_exclude( -diff --git a/src/network.c b/src/network.c -index 6c08070..d826511 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) { - return 0; - } - -+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { -+ if (loc_network_match_address(self, &other->first_address) == 0) -+ return 1; -+ -+ if (loc_network_match_address(self, &other->last_address) == 0) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->first_address) == 0) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->last_address) == 0) -+ return 1; -+ -+ return 0; -+} -+ - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. --- -2.20.1 - -From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 14:28:15 +0000 -Subject: [PATCH 037/111] networks: Add function to check if network is part of - a list - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 9 +++++++++ - 3 files changed, 11 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 0c2835b..f1b63a2 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -104,6 +104,7 @@ global: - - # Network List - loc_network_list_clear; -+ loc_network_list_contains; - loc_network_list_dump; - loc_network_list_empty; - loc_network_list_get; -diff --git a/src/loc/network.h b/src/loc/network.h -index ef13756..7804512 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); - -diff --git a/src/network.c b/src/network.c -index d826511..fcbdc59 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - return list->list[--list->size]; - } - -+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_network_eq(list->list[i], network)) -+ return 1; -+ } -+ -+ return 0; -+} -+ - static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { - // Do nothing for invalid indices - if (i1 >= list->size || i2 >= list->size) --- -2.20.1 - -From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 14:33:22 +0000 -Subject: [PATCH 038/111] networks: Add function to merge two lists - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 13 +++++++++++++ - 3 files changed, 15 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index f1b63a2..c0b6b1f 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -108,6 +108,7 @@ global: - loc_network_list_dump; - loc_network_list_empty; - loc_network_list_get; -+ loc_network_list_merge; - loc_network_list_new; - loc_network_list_pop; - loc_network_list_push; -diff --git a/src/loc/network.h b/src/loc/network.h -index 7804512..e30d91c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list); - int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); -+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); - - #ifdef LIBLOC_PRIVATE - -diff --git a/src/network.c b/src/network.c -index fcbdc59..541286d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { - n--; - } while (swapped); - } -+ -+LOC_EXPORT int loc_network_list_merge( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; -+ -+ for (unsigned int i = 0; i < other->size; i++) { -+ r = loc_network_list_push(self, other->list[i]); -+ if (r) -+ return r; -+ } -+ -+ return 0; -+} --- -2.20.1 - -From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 14:35:43 +0000 -Subject: [PATCH 039/111] network: Make lists unique - -Networks that are in the list won't be added again - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/src/network.c b/src/network.c -index 541286d..44571b3 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - } - - LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -+ // Do not add networks that are already on the list -+ if (loc_network_list_contains(list, network)) -+ return 0; -+ - // Check if we have space left - if (list->size == list->max_size) - return -ENOMEM; --- -2.20.1 - -From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 19:21:13 +0000 -Subject: [PATCH 040/111] database: Pass flag to enumerator to flatten output - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 8 +++++++- - src/loc/database.h | 6 +++++- - src/perl/Location.xs | 2 +- - src/python/database.c | 25 ++++++++++++++++++------- - 4 files changed, 31 insertions(+), 10 deletions(-) - -diff --git a/src/database.c b/src/database.c -index fa1dad0..9baab33 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -104,6 +104,9 @@ struct loc_database_enumerator { - enum loc_network_flags flags; - int family; - -+ // Flatten output? -+ int flatten; -+ - // Index of the AS we are looking at - unsigned int as_index; - -@@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - // Enumerator - - LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode) { -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { - struct loc_database_enumerator* e = calloc(1, sizeof(*e)); - if (!e) - return -ENOMEM; -@@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum - e->mode = mode; - e->refcount = 1; - -+ // Flatten output? -+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); -+ - // Initialise graph search - //e->network_stack[++e->network_stack_depth] = 0; - e->network_stack_depth = 1; -diff --git a/src/loc/database.h b/src/loc/database.h -index 43173dd..14eb5ea 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -55,9 +55,13 @@ enum loc_database_enumerator_mode { - LOC_DB_ENUMERATE_COUNTRIES = 3, - }; - -+enum loc_database_enumerator_flags { -+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), -+}; -+ - struct loc_database_enumerator; - int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode); -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); - struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); - struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); - -diff --git a/src/perl/Location.xs b/src/perl/Location.xs -index dcf3f0d..b7676d2 100644 ---- a/src/perl/Location.xs -+++ b/src/perl/Location.xs -@@ -125,7 +125,7 @@ database_countries(db) - PPCODE: - // Create Database enumerator - struct loc_database_enumerator* enumerator; -- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES); -+ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0); - - if (err) { - croak("Could not create a database enumerator\n"); -diff --git a/src/python/database.c b/src/python/database.c -index 1013a58..7f8c2c2 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database - return (PyObject*)self; - } - --static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) { -+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) { - struct loc_database_enumerator* enumerator; - -- int r = loc_database_enumerator_new(&enumerator, self->db, what); -+ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en - } - - static PyObject* Database_ases(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); - } - - static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { -@@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { - - struct loc_database_enumerator* enumerator; - -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { - } - - static PyObject* Database_networks(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); -+} -+ -+static PyObject* Database_networks_flattened(DatabaseObject *self) { -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN); - } - - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -@@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - return NULL; - - struct loc_database_enumerator* enumerator; -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - } - - static PyObject* Database_countries(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); - } - - static struct PyMethodDef Database_methods[] = { -@@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "networks_flattened", -+ (getter)Database_networks_flattened, -+ NULL, -+ NULL, -+ NULL, -+ }, - { - "vendor", - (getter)Database_get_vendor, --- -2.20.1 - -From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 19:21:58 +0000 -Subject: [PATCH 041/111] network: Reduce debugging output - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 28 +++------------------------- - 1 file changed, 3 insertions(+), 25 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 44571b3..f7071a6 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - } - - LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { --#ifdef ENABLE_DEBUG -- char* n1 = loc_network_str(self); -- char* n2 = loc_network_str(other); -- -- DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2); -- -- free(n1); -- free(n2); --#endif -- - // Family must be the same -- if (self->family != other->family) { -- DEBUG(self->ctx, " Family mismatch\n"); -- -+ if (self->family != other->family) - return 0; -- } - - // The start address must be the same -- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) { -- DEBUG(self->ctx, " Address mismatch\n"); -- -+ if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) - return 0; -- } - - // The prefix length must be the same -- if (self->prefix != other->prefix) { -- DEBUG(self->ctx, " Prefix mismatch\n"); -+ if (self->prefix != other->prefix) - return 0; -- } -- -- DEBUG(self->ctx, " Yes!\n"); - - return 1; - } -@@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 - if (i1 >= list->size || i2 >= list->size) - return; - -- DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2); -- - struct loc_network* network1 = list->list[i1]; - struct loc_network* network2 = list->list[i2]; - --- -2.20.1 - -From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 19:36:38 +0000 -Subject: [PATCH 042/111] python: Export networks exclude function - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++ - 1 file changed, 39 insertions(+) - -diff --git a/src/python/network.c b/src/python/network.c -index 5496d1e..11f672b 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -24,6 +24,24 @@ - #include "locationmodule.h" - #include "network.h" - -+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { -+ PyObject* list = PyList_New(0); -+ if (!networks) -+ return list; -+ -+ while (!loc_network_list_empty(networks)) { -+ struct loc_network* network = loc_network_list_pop(networks); -+ -+ PyObject* n = new_network(&NetworkType, network); -+ PyList_Append(list, n); -+ -+ loc_network_unref(network); -+ Py_DECREF(n); -+ } -+ -+ return list; -+} -+ - PyObject* new_network(PyTypeObject* type, struct loc_network* network) { - NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); - if (self) { -@@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { - Py_RETURN_NONE; - } - -+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { -+ NetworkObject* other = NULL; -+ -+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) -+ return NULL; -+ -+ struct loc_network_list* list = loc_network_exclude(self->network, other->network); -+ -+ // Convert to Python objects -+ PyObject* obj = PyList_FromNetworkList(list); -+ loc_network_list_unref(list); -+ -+ return obj; -+} -+ - static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { - NetworkObject* other = NULL; - -@@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) { - } - - static struct PyMethodDef Network_methods[] = { -+ { -+ "exclude", -+ (PyCFunction)Network_exclude, -+ METH_VARARGS, -+ NULL, -+ }, - { - "has_flag", - (PyCFunction)Network_has_flag, --- -2.20.1 - -From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 19:59:22 +0000 -Subject: [PATCH 043/111] network: Add more debugging output to stacks - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 16 +++++++++++++--- - 1 file changed, 13 insertions(+), 3 deletions(-) - -diff --git a/src/network.c b/src/network.c -index f7071a6..d41e873 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - return 0; - - // Check if we have space left -- if (list->size == list->max_size) -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); - return -ENOMEM; -+ } -+ -+ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); - - list->list[list->size++] = loc_network_ref(network); - -@@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - - LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { - // Return nothing when empty -- if (loc_network_list_empty(list)) -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); - return NULL; -+ } - -- return list->list[--list->size]; -+ struct loc_network* network = list->list[--list->size]; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; - } - - LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { --- -2.20.1 - -From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 20:00:09 +0000 -Subject: [PATCH 044/111] network: Add new subnet function - -The old one is too difficult to use in terms of order -of input parameters and return value. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 15 +++++++++++++++ - 3 files changed, 17 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index c0b6b1f..5392437 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -87,6 +87,7 @@ global: - loc_network_get_asn; - loc_network_get_country_code; - loc_network_has_flag; -+ loc_network_is_subnet; - loc_network_is_subnet_of; - loc_network_match_asn; - loc_network_match_country_code; -diff --git a/src/loc/network.h b/src/loc/network.h -index e30d91c..2154cdc 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); -+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - struct loc_network_list* loc_network_subnets(struct loc_network* network); - struct loc_network_list* loc_network_exclude( -diff --git a/src/network.c b/src/network.c -index d41e873..5719111 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - return 0; - } - -+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { -+ // If the start address of the other network is smaller than this network, -+ // it cannot be a subnet. -+ if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -+ return 0; -+ -+ // If the end address of the other network is greater than this network, -+ // it cannot be a subnet. -+ if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -+ return 0; -+ -+ return 1; -+} -+ -+// XXX DEPRECATED - I find this too difficult to use - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. --- -2.20.1 - -From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 20:01:17 +0000 -Subject: [PATCH 045/111] network: Add function to exclude multiple networks at - once - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 2 ++ - src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 88 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 5392437..bcd11be 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -82,6 +82,7 @@ global: - loc_network_address_family; - loc_network_eq; - loc_network_exclude; -+ loc_network_exclude_list; - loc_network_format_first_address; - loc_network_format_last_address; - loc_network_get_asn; -diff --git a/src/loc/network.h b/src/loc/network.h -index 2154cdc..40712b9 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other - struct loc_network_list* loc_network_subnets(struct loc_network* network); - struct loc_network_list* loc_network_exclude( - struct loc_network* self, struct loc_network* other); -+struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list); - - // List - struct loc_network_list; -diff --git a/src/network.c b/src/network.c -index 5719111..751e8e5 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -677,6 +677,91 @@ ERROR: - return NULL; - } - -+LOC_EXPORT struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list) { -+ struct loc_network_list* to_check; -+ -+ // Create a new list with all networks to look at -+ int r = loc_network_list_new(network->ctx, &to_check); -+ if (r) -+ return NULL; -+ -+ struct loc_network* subnet = NULL; -+ struct loc_network_list* subnets = NULL; -+ -+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { -+ subnet = loc_network_list_get(list, i); -+ -+ // Find all excluded networks -+ struct loc_network_list* excluded = loc_network_exclude(network, subnet); -+ if (excluded) { -+ // Add them all to the "to check" list -+ loc_network_list_merge(to_check, excluded); -+ loc_network_list_unref(excluded); -+ } -+ -+ // Cleanup -+ loc_network_unref(subnet); -+ } -+ -+ r = loc_network_list_new(network->ctx, &subnets); -+ if (r) { -+ loc_network_list_unref(to_check); -+ return NULL; -+ } -+ -+ while (!loc_network_list_empty(to_check)) { -+ struct loc_network* subnet_to_check = loc_network_list_pop(to_check); -+ -+ // Marks whether this subnet passed all checks -+ int passed = 1; -+ -+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { -+ subnet = loc_network_list_get(list, i); -+ -+ // Drop this subnet if is is already in list -+ if (loc_network_eq(subnet_to_check, subnet)) { -+ passed = 0; -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // Drop this subnet if is a subnet of another subnet -+ if (loc_network_is_subnet_of(subnet, subnet_to_check)) { -+ passed = 0; -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // Break it down if it overlaps -+ if (loc_network_overlaps(subnet_to_check, subnet)) { -+ passed = 0; -+ -+ struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet); -+ if (excluded) { -+ loc_network_list_merge(to_check, excluded); -+ loc_network_list_unref(excluded); -+ } -+ -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ loc_network_unref(subnet); -+ } -+ -+ if (passed) { -+ r = loc_network_list_push(subnets, subnet_to_check); -+ } -+ -+ loc_network_unref(subnet_to_check); -+ } -+ -+ loc_network_list_unref(to_check); -+ -+ return subnets; -+} -+ - LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { - // Add country code - loc_country_code_copy(dbobj->country_code, network->country_code); --- -2.20.1 - -From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 20:02:03 +0000 -Subject: [PATCH 046/111] database: Add option to return networks flattened - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++--------- - 1 file changed, 143 insertions(+), 31 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 9baab33..7a3d1a7 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -118,6 +118,9 @@ struct loc_database_enumerator { - struct loc_node_stack network_stack[MAX_STACK_DEPTH]; - int network_stack_depth; - unsigned int* networks_visited; -+ -+ // For subnet search -+ struct loc_network_list* stack; - }; - - static int loc_database_read_magic(struct loc_database* db) { -@@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - - // Enumerator - -+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { -+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -+ -+ // Release all references -+ loc_database_unref(enumerator->db); -+ loc_unref(enumerator->ctx); +diff --git a/src/country-list.c b/src/country-list.c +new file mode 100644 +index 0000000..cc36740 +--- /dev/null ++++ b/src/country-list.c +@@ -0,0 +1,161 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet + -+ if (enumerator->string) -+ free(enumerator->string); ++ Copyright (C) 2020 IPFire Development Team info@ipfire.org + -+ // Free network search -+ free(enumerator->networks_visited); ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. + -+ // Free subnet stack -+ if (enumerator->stack) -+ loc_network_list_unref(enumerator->stack); ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ free(enumerator); -+} ++#include <errno.h> ++#include <stdlib.h> + - LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, - struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { - struct loc_database_enumerator* e = calloc(1, sizeof(*e)); -@@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum - e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); - - // Initialise graph search -- //e->network_stack[++e->network_stack_depth] = 0; - e->network_stack_depth = 1; - e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); - -+ // Allocate stack -+ int r = loc_network_list_new(e->ctx, &e->stack); -+ if (r) { -+ loc_database_enumerator_free(e); -+ return r; -+ } ++#include <loc/country.h> ++#include <loc/country-list.h> ++#include <loc/private.h> + - DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); - - *enumerator = e; -@@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo - return enumerator; - } - --static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { -- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -- -- // Release all references -- loc_database_unref(enumerator->db); -- loc_unref(enumerator->ctx); -- -- if (enumerator->string) -- free(enumerator->string); -- -- // Free network search -- free(enumerator->networks_visited); -- -- free(enumerator); --} -- - LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { - if (!enumerator) - return NULL; -@@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - --LOC_EXPORT int loc_database_enumerator_next_network( -- struct loc_database_enumerator* enumerator, struct loc_network** network) { -- // Reset network -- *network = NULL; -- -- // Do not do anything if not in network mode -- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) -+static int __loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { -+ // Return top element from the stack -+ *network = loc_network_list_pop(enumerator->stack); -+ if (*network) - return 0; - -- int r; -- - DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", - enumerator->network_stack_depth); - -@@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( - enumerator->db->network_nodes_v1 + node->offset; - - // Add edges to stack -- r = loc_database_enumerator_stack_push_node(enumerator, -+ int r = loc_database_enumerator_stack_push_node(enumerator, - be32toh(n->one), 1, node->depth + 1); - - if (r) -@@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network( - if (r) - return r; - -+ // Return all networks when the filter is disabled -+ if (!filter) -+ return 0; ++struct loc_country_list { ++ struct loc_ctx* ctx; ++ int refcount; + - // Check if we are interested in this network - - // Skip if the family does not match -@@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network( - } - - // Reached the end of the search -+ return 0; -+} - -- // Mark all nodes as non-visited -- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) -- enumerator->networks_visited[i] = 0; -+static int __loc_database_enumerator_next_network_flattened( -+ struct loc_database_enumerator* enumerator, struct loc_network** network) { -+ // Fetch the next network -+ int r = __loc_database_enumerator_next_network(enumerator, network, 1); -+ if (r) -+ return r; - -- return 0; -+ // End if we could not read another network -+ if (!*network) -+ return 0; ++ struct loc_country** elements; ++ size_t elements_size; + -+ struct loc_network* subnet = NULL; -+ struct loc_network_list* subnets; ++ size_t size; ++}; + -+ // Create a list with all subnets -+ r = loc_network_list_new(enumerator->ctx, &subnets); -+ if (r) -+ return r; ++static int loc_country_list_grow(struct loc_country_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); + -+ // Search all subnets from the database -+ while (1) { -+ // Fetch the next network in line -+ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); -+ if (r) -+ goto END; ++ struct loc_country** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; + -+ // End if we did not receive another subnet -+ if (!subnet) -+ break; ++ list->elements = elements; ++ list->elements_size += size; + -+ // Collect all subnets in a list -+ if (loc_network_is_subnet(*network, subnet)) { -+ r = loc_network_list_push(subnets, subnet); -+ if (r) -+ goto END; ++ return 0; ++} + -+ loc_network_unref(subnet); -+ continue; -+ } ++LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, ++ struct loc_country_list** list) { ++ struct loc_country_list* l = calloc(1, sizeof(*l)); ++ if (!l) ++ return -ENOMEM; + -+ // If this is not a subnet, we push it back onto the stack and break -+ r = loc_network_list_push(enumerator->stack, subnet); -+ if (r) -+ goto END; ++ l->ctx = loc_ref(ctx); ++ l->refcount = 1; + -+ loc_network_unref(subnet); -+ break; -+ } ++ DEBUG(l->ctx, "Country list allocated at %p\n", l); ++ *list = l; + -+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); ++ return 0; ++} + -+ // We can abort here if the network has no subnets -+ if (loc_network_list_empty(subnets)) { -+ loc_network_list_unref(subnets); ++LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { ++ list->refcount++; + -+ return 0; -+ } ++ return list; ++} + -+ // If the network has any subnets, we will break it into smaller parts -+ // without the subnets. -+ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); -+ if (!excluded || loc_network_list_empty(excluded)) { -+ r = 1; -+ goto END; -+ } ++static void loc_country_list_free(struct loc_country_list* list) { ++ DEBUG(list->ctx, "Releasing country list at %p\n", list); + -+ // Sort the result -+ loc_network_list_sort(excluded); ++ loc_country_list_clear(list); + -+ // Reverse the list -+ loc_network_list_reverse(excluded); ++ loc_unref(list->ctx); ++ free(list); ++} + -+ // Replace network with the first one -+ loc_network_unref(*network); ++LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { ++ if (!list) ++ return NULL; + -+ *network = loc_network_list_pop(excluded); ++ if (--list->refcount > 0) ++ return list; + -+ // Push the rest onto the stack -+ loc_network_list_merge(enumerator->stack, excluded); ++ loc_country_list_free(list); ++ return NULL; ++} + -+ loc_network_list_unref(excluded); ++LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { ++ return list->size; ++} + -+END: -+ if (subnet) -+ loc_network_unref(subnet); ++LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { ++ return list->size == 0; ++} + -+ loc_network_list_unref(subnets); ++LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { ++ if (!list->elements) ++ return; + -+ return r; ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_country_unref(list->elements[i]); ++ ++ free(list->elements); ++ list->elements = NULL; ++ list->elements_size = 0; ++ ++ list->size = 0; +} + -+LOC_EXPORT int loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network) { -+ // Do not do anything if not in network mode -+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) -+ return 0; ++LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; + -+ // Flatten output? -+ if (enumerator->flatten) -+ return __loc_database_enumerator_next_network_flattened(enumerator, network); ++ return loc_country_ref(list->elements[index]); ++} + -+ return __loc_database_enumerator_next_network(enumerator, network, 1); - } - - LOC_EXPORT int loc_database_enumerator_next_country( --- -2.20.1 - -From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 20:09:20 +0000 -Subject: [PATCH 047/111] test: Update API - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/test-as.c | 2 +- - src/test-database.c | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/test-as.c b/src/test-as.c -index 839a04c..2d61675 100644 ---- a/src/test-as.c -+++ b/src/test-as.c -@@ -95,7 +95,7 @@ int main(int argc, char** argv) { - // Enumerator - - struct loc_database_enumerator* enumerator; -- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES); -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); - if (err) { - fprintf(stderr, "Could not create a database enumerator\n"); - exit(EXIT_FAILURE); -diff --git a/src/test-database.c b/src/test-database.c -index 4aef94e..da4b11c 100644 ---- a/src/test-database.c -+++ b/src/test-database.c -@@ -198,7 +198,7 @@ int main(int argc, char** argv) { - - // Enumerator - struct loc_database_enumerator* enumerator; -- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS); -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); - if (err) { - fprintf(stderr, "Could not initialise the enumerator: %d\n", err); - exit(EXIT_FAILURE); --- -2.20.1 - -From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 12 Nov 2020 20:09:37 +0000 -Subject: [PATCH 048/111] networks: Copy all attributes when splitting networks - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 14 ++++++++++++++ - 1 file changed, 14 insertions(+) - -diff --git a/src/network.c b/src/network.c -index 751e8e5..d67f116 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw - if (r) - goto ERROR; - -+ // Copy country code -+ const char* country_code = loc_network_get_country_code(network); -+ if (country_code) { -+ loc_network_set_country_code(subnet1, country_code); -+ loc_network_set_country_code(subnet2, country_code); -+ } ++LOC_EXPORT int loc_country_list_append( ++ struct loc_country_list* list, struct loc_country* country) { ++ if (loc_country_list_contains(list, country)) ++ return 0; + -+ // Copy ASN -+ uint32_t asn = loc_network_get_asn(network); -+ if (asn) { -+ loc_network_set_asn(subnet1, asn); -+ loc_network_set_asn(subnet2, asn); ++ // Check if we have space left ++ if (list->size >= list->elements_size) { ++ int r = loc_country_list_grow(list, 64); ++ if (r) ++ return r; + } + - loc_network_unref(subnet1); - loc_network_unref(subnet2); - --- -2.20.1 - -From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 13 Nov 2020 11:23:33 +0000 -Subject: [PATCH 049/111] database: Move network filtering into a separate - function - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 56 +++++++++++++++++++++++--------------------------- - 1 file changed, 26 insertions(+), 30 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 7a3d1a7..72bc8eb 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - -+static int loc_database_enumerator_filter_network( -+ struct loc_database_enumerator* enumerator, struct loc_network* network) { -+ // Skip if the family does not match -+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) -+ return 1; ++ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); + -+ // Skip if the country code does not match -+ if (*enumerator->country_code && -+ !loc_network_match_country_code(network, enumerator->country_code)) -+ return 1; ++ list->elements[list->size++] = loc_country_ref(country); + -+ // Skip if the ASN does not match -+ if (enumerator->asn && -+ !loc_network_match_asn(network, enumerator->asn)) -+ return 1; ++ return 0; ++} + -+ // Skip if flags do not match -+ if (enumerator->flags && -+ !loc_network_match_flag(network, enumerator->flags)) -+ return 1; ++LOC_EXPORT int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country) { ++ for (unsigned int i = 0; i < list->size; i++) { ++ if (loc_country_cmp(country, list->elements[i]) == 0) ++ return 1; ++ } + -+ // Do not filter + return 0; +} + - static int __loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { - // Return top element from the stack -@@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network( - return 0; - - // Check if we are interested in this network -- -- // Skip if the family does not match -- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { -- loc_network_unref(*network); -- *network = NULL; -- -- continue; -- } -- -- // Skip if the country code does not match -- if (*enumerator->country_code && -- !loc_network_match_country_code(*network, enumerator->country_code)) { -- loc_network_unref(*network); -- *network = NULL; -- -- continue; -- } -- -- // Skip if the ASN does not match -- if (enumerator->asn && -- !loc_network_match_asn(*network, enumerator->asn)) { -- loc_network_unref(*network); -- *network = NULL; -- -- continue; -- } -- -- // Skip if flags do not match -- if (enumerator->flags && -- !loc_network_match_flag(*network, enumerator->flags)) { -+ if (loc_database_enumerator_filter_network(enumerator, *network)) { - loc_network_unref(*network); - *network = NULL; - --- -2.20.1 - -From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 13 Nov 2020 11:29:02 +0000 -Subject: [PATCH 050/111] database: Filter results coming from stack - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 18 ++++++++++++++++-- - 1 file changed, 16 insertions(+), 2 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 72bc8eb..0f3cdc2 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network( - static int __loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { - // Return top element from the stack -- *network = loc_network_list_pop(enumerator->stack); -- if (*network) -+ while (1) { -+ *network = loc_network_list_pop(enumerator->stack); ++LOC_EXPORT int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code) { ++ struct loc_country* country; + -+ // Stack is empty -+ if (!*network) -+ break; ++ int r = loc_country_new(list->ctx, &country, code); ++ if (r) ++ return -1; + -+ // Throw away any networks by filter -+ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { -+ loc_network_unref(*network); -+ *network = NULL; -+ continue; -+ } ++ r = loc_country_list_contains(list, country); ++ loc_country_unref(country); + -+ // Return result - return 0; -+ } ++ return r; ++} +diff --git a/src/country.c b/src/country.c +index 2ba93e6..7aac0db 100644 +--- a/src/country.c ++++ b/src/country.c +@@ -34,6 +34,9 @@ struct loc_country { + };
- DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", - enumerator->network_stack_depth); --- -2.20.1 - -From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 13 Nov 2020 11:29:15 +0000 -Subject: [PATCH 051/111] network: Sort result of excluded lists - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 3 --- - src/network.c | 3 +++ - 2 files changed, 3 insertions(+), 3 deletions(-) - + LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { ++ if (!loc_country_code_is_valid(country_code)) ++ return -EINVAL; ++ + struct loc_country* c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; diff --git a/src/database.c b/src/database.c -index 0f3cdc2..6849d97 100644 +index fa1dad0..1871b74 100644 --- a/src/database.c +++ b/src/database.c -@@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened( - goto END; - } +@@ -38,8 +38,10 @@
-- // Sort the result -- loc_network_list_sort(excluded); -- - // Reverse the list - loc_network_list_reverse(excluded); + #include <loc/libloc.h> + #include <loc/as.h> ++#include <loc/as-list.h> + #include <loc/compat.h> + #include <loc/country.h> ++#include <loc/country-list.h> + #include <loc/database.h> + #include <loc/format.h> + #include <loc/network.h> +@@ -99,11 +101,14 @@ struct loc_database_enumerator {
-diff --git a/src/network.c b/src/network.c -index d67f116..9d02bf8 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( + // Search string + char* string; +- char country_code[3]; +- uint32_t asn; ++ struct loc_country_list* countries; ++ struct loc_as_list* asns; + enum loc_network_flags flags; + int family;
- loc_network_list_unref(to_check); ++ // Flatten output? ++ int flatten; ++ + // Index of the AS we are looking at + unsigned int as_index;
-+ // Sort the result -+ loc_network_list_sort(subnets); +@@ -115,6 +120,9 @@ struct loc_database_enumerator { + struct loc_node_stack network_stack[MAX_STACK_DEPTH]; + int network_stack_depth; + unsigned int* networks_visited; + - return subnets; - } ++ // For subnet search ++ struct loc_network_list* stack; + };
--- -2.20.1 - -From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 13 Nov 2020 11:38:15 +0000 -Subject: [PATCH 052/111] network: Add function to pop first element from stack - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 6 ++---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 19 +++++++++++++++++++ - 4 files changed, 23 insertions(+), 4 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 6849d97..b9d870f 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened( - goto END; + static int loc_database_read_magic(struct loc_database* db) { +@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { }
-- // Reverse the list -- loc_network_list_reverse(excluded); -- - // Replace network with the first one - loc_network_unref(*network); + clock_t end = clock(); +- DEBUG(db->ctx, "Signature checked in %.4fms\n", ++ INFO(db->ctx, "Signature checked in %.4fms\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000);
-- *network = loc_network_list_pop(excluded); -+ *network = loc_network_list_pop_first(excluded); + CLEANUP: +@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, + off_t lo = 0; + off_t hi = db->as_count - 1;
- // Push the rest onto the stack -+ loc_network_list_reverse(excluded); - loc_network_list_merge(enumerator->stack, excluded); ++#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); ++#endif
- loc_network_list_unref(excluded); -diff --git a/src/libloc.sym b/src/libloc.sym -index bcd11be..6139db6 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -113,6 +113,7 @@ global: - loc_network_list_merge; - loc_network_list_new; - loc_network_list_pop; -+ loc_network_list_pop_first; - loc_network_list_push; - loc_network_list_ref; - loc_network_list_reverse; -diff --git a/src/loc/network.h b/src/loc/network.h -index 40712b9..203e61c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list); - struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); - int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); - struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); - int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); -diff --git a/src/network.c b/src/network.c -index 9d02bf8..e7dc97e 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - return network; - } + while (lo <= hi) { + off_t i = (lo + hi) / 2; +@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, + // Check if this is a match + uint32_t as_number = loc_as_get_number(*as); + if (as_number == number) { ++#ifdef ENABLE_DEBUG + clock_t end = clock();
-+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->list[0]; -+ -+ // Move all elements to the top of the stack -+ for (unsigned int i = 0; i < --list->size; i++) { -+ list->list[i] = list->list[i+1]; -+ } -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ - LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { - for (unsigned int i = 0; i < list->size; i++) { - if (loc_network_eq(list->list[i], network)) --- -2.20.1 - -From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 13 Nov 2020 11:43:53 +0000 -Subject: [PATCH 053/111] network: Unexport all tree functions - -These should not be exported - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 28 ++++++++++++++-------------- - src/test-network.c | 9 +++++++++ - 2 files changed, 23 insertions(+), 14 deletions(-) - -diff --git a/src/network.c b/src/network.c -index e7dc97e..d015579 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -847,7 +847,7 @@ struct loc_network_tree_node { - struct loc_network* network; - }; + // Log how fast this has been + DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, + (double)(end - start) / CLOCKS_PER_SEC * 1000); ++#endif
--LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { -+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { - struct loc_network_tree* t = calloc(1, sizeof(*t)); - if (!t) - return -ENOMEM; -@@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree - return 0; - } + return 0; + } +@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ + return -1; + }
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { -+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { - return loc_network_tree_node_ref(tree->root); - } ++#ifdef ENABLE_DEBUG + if (r == 0) { + char* string = loc_network_str(*network); + DEBUG(db->ctx, "Got network %s\n", string); + free(string); + } ++#endif
-@@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_ - return 0; + return r; } +@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru + }
--LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, -+int loc_network_tree_walk(struct loc_network_tree* tree, - int(*filter_callback)(struct loc_network* network, void* data), - int(*callback)(struct loc_network* network, void* data), void* data) { - return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); -@@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) { - free(tree); - } + // Check if the given IP address is inside the network +- r = loc_network_match_address(*network, address); +- if (r) { ++ if (!loc_network_match_address(*network, address)) { + DEBUG(db->ctx, "Searched address is not part of the network\n");
--LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { -+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { - if (--tree->refcount > 0) - return tree; + loc_network_unref(*network); +@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
-@@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) { - return 0; - } + *network = NULL;
--LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { -+int loc_network_tree_dump(struct loc_network_tree* tree) { - DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); ++#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); ++#endif
- return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); - } + int r = __loc_database_lookup(db, address, network, &network_address, + db->network_nodes_v1, 0);
--LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { -+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { - DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); ++#ifdef ENABLE_DEBUG + clock_t end = clock();
- struct loc_network_tree_node* node = loc_network_tree_get_path(tree, -@@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) { - return 0; + // Log how fast this has been + DEBUG(db->ctx, "Executed network search in %.4fms\n", + (double)(end - start) / CLOCKS_PER_SEC * 1000); ++#endif + + return r; } +@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + off_t lo = 0; + off_t hi = db->countries_count - 1;
--LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { -+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { - size_t counter = 0; ++#ifdef ENABLE_DEBUG + // Save start time + clock_t start = clock(); ++#endif + + while (lo <= hi) { + off_t i = (lo + hi) / 2; +@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + int result = strcmp(code, cc); + + if (result == 0) { ++#ifdef ENABLE_DEBUG + clock_t end = clock(); + + // Log how fast this has been + DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, + (double)(end - start) / CLOCKS_PER_SEC * 1000); ++#endif + + return 0; + } +@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, + + // Enumerator + ++static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { ++ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); ++ ++ // Release all references ++ loc_database_unref(enumerator->db); ++ loc_unref(enumerator->ctx); ++ ++ if (enumerator->string) ++ free(enumerator->string); ++ ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries); ++ ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ ++ // Free network search ++ free(enumerator->networks_visited); ++ ++ // Free subnet stack ++ if (enumerator->stack) ++ loc_network_list_unref(enumerator->stack); ++ ++ free(enumerator); ++} ++ + LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode) { ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); + if (!e) + return -ENOMEM; +@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum + e->mode = mode; + e->refcount = 1; + ++ // Flatten output? ++ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); ++ + // Initialise graph search +- //e->network_stack[++e->network_stack_depth] = 0; + e->network_stack_depth = 1; + e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
- int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); -@@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) - return counter; - } ++ // Allocate stack ++ int r = loc_network_list_new(e->ctx, &e->stack); ++ if (r) { ++ loc_database_enumerator_free(e); ++ return r; ++ } ++ + DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
--LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { -+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { - return __loc_network_tree_count_nodes(tree->root); + *enumerator = e; +@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo + return enumerator; }
--LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { -+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { - struct loc_network_tree_node* n = calloc(1, sizeof(*n)); - if (!n) - return -ENOMEM; -@@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network +-static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { +- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); +- +- // Release all references +- loc_database_unref(enumerator->db); +- loc_unref(enumerator->ctx); +- +- if (enumerator->string) +- free(enumerator->string); +- +- // Free network search +- free(enumerator->networks_visited); +- +- free(enumerator); +-} +- + LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { + if (!enumerator) + return NULL; +@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator return 0; }
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { - if (node) - node->refcount++; +-LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { +- // Set empty country code +- if (!country_code || !*country_code) { +- *enumerator->country_code = '\0'; +- return 0; +- } ++LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->countries) ++ return NULL;
-@@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { - free(node); - } +- // Treat A1, A2, A3 as special country codes, +- // but perform search for flags instead +- if (strcmp(country_code, "A1") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); +- } else if (strcmp(country_code, "A2") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); +- } else if (strcmp(country_code, "A3") == 0) { +- return loc_database_enumerator_set_flag(enumerator, +- LOC_NETWORK_FLAG_ANYCAST); +- } ++ return loc_country_list_ref(enumerator->countries); ++}
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { - if (!node) - return NULL; +- // Country codes must be two characters +- if (!loc_country_code_is_valid(country_code)) +- return -EINVAL; ++LOC_EXPORT int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { ++ if (enumerator->countries) ++ loc_country_list_unref(enumerator->countries);
-@@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_ - return NULL; - } +- for (unsigned int i = 0; i < 3; i++) { +- enumerator->country_code[i] = country_code[i]; +- } ++ enumerator->countries = loc_country_list_ref(countries);
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { -+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { - if (index == 0) - node = node->zero; - else -@@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne - return loc_network_tree_node_ref(node); + return 0; }
--LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { -+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { - return (!!node->network); - } +-LOC_EXPORT int loc_database_enumerator_set_asn( +- struct loc_database_enumerator* enumerator, unsigned int asn) { +- enumerator->asn = asn; ++LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator) { ++ if (!enumerator->asns) ++ return NULL; ++ ++ return loc_as_list_ref(enumerator->asns); ++} ++ ++LOC_EXPORT int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { ++ if (enumerator->asns) ++ loc_as_list_unref(enumerator->asns); ++ ++ enumerator->asns = loc_as_list_ref(asns);
--LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { -+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { - return loc_network_ref(node->network); + return 0; + } +@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node( + return 0; }
-diff --git a/src/test-network.c b/src/test-network.c -index af1b2e6..7c90224 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -37,12 +37,14 @@ int main(int argc, char** argv) { - // Enable debug logging - loc_set_log_priority(ctx, LOG_DEBUG); - -+#if 0 - struct loc_network_tree* tree; - err = loc_network_tree_new(ctx, &tree); - if (err) { - fprintf(stderr, "Could not create the network tree\n"); - exit(EXIT_FAILURE); - } -+#endif +-LOC_EXPORT int loc_database_enumerator_next_network( +- struct loc_database_enumerator* enumerator, struct loc_network** network) { +- // Reset network +- *network = NULL; ++static int loc_database_enumerator_filter_network( ++ struct loc_database_enumerator* enumerator, struct loc_network* network) { ++ // Skip if the family does not match ++ if (enumerator->family && loc_network_address_family(network) != enumerator->family) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); ++ return 1; ++ }
- // Create a network - struct loc_network* network1; -@@ -58,12 +60,14 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } +- // Do not do anything if not in network mode +- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) +- return 0; ++ // Skip if the country code does not match ++ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { ++ const char* country_code = loc_network_get_country_code(network);
-+#if 0 - // Adding network to the tree - err = loc_network_tree_add_network(tree, network1); - if (err) { - fprintf(stderr, "Could not add network to the tree\n"); - exit(EXIT_FAILURE); - } -+#endif +- int r; ++ if (!loc_country_list_contains_code(enumerator->countries, country_code)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); ++ return 1; ++ } ++ } ++ ++ // Skip if the ASN does not match ++ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { ++ uint32_t asn = loc_network_get_asn(network); ++ ++ if (!loc_as_list_contains_number(enumerator->asns, asn)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); ++ return 1; ++ } ++ } ++ ++ // Skip if flags do not match ++ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { ++ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); ++ return 1; ++ } ++ ++ // Do not filter ++ return 0; ++} ++ ++static int __loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { ++ // Return top element from the stack ++ while (1) { ++ *network = loc_network_list_pop_first(enumerator->stack); ++ ++ // Stack is empty ++ if (!*network) ++ break; ++ ++ // Throw away any networks by filter ++ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { ++ loc_network_unref(*network); ++ *network = NULL; ++ continue; ++ } ++ ++ // Return result ++ return 0; ++ }
- // Check if the first and last addresses are correct - char* string = loc_network_format_first_address(network1); -@@ -101,6 +105,7 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } + DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", + enumerator->network_stack_depth); +@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( + enumerator->db->network_nodes_v1 + node->offset;
-+#if 0 - // Adding network to the tree - err = loc_network_tree_add_network(tree, network2); - if (err) { -@@ -117,6 +122,7 @@ int main(int argc, char** argv) { + // Add edges to stack +- r = loc_database_enumerator_stack_push_node(enumerator, ++ int r = loc_database_enumerator_stack_push_node(enumerator, + be32toh(n->one), 1, node->depth + 1);
- size_t nodes = loc_network_tree_count_nodes(tree); - printf("The tree has %zu nodes\n", nodes); -+#endif + if (r) +@@ -1175,56 +1261,145 @@ LOC_EXPORT int loc_database_enumerator_next_network( + if (r) + return r;
- // Check equals function - err = loc_network_eq(network1, network1); -@@ -260,7 +266,10 @@ int main(int argc, char** argv) { - loc_network_unref(network2); - loc_network_unref(network3); - loc_network_unref(network4); -+ -+#if 0 - loc_network_tree_unref(tree); -+#endif +- // Check if we are interested in this network ++ // Return all networks when the filter is disabled ++ if (!filter) ++ return 0;
- // And open it again from disk - struct loc_database* db; --- -2.20.1 - -From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 13 Nov 2020 12:09:03 +0000 -Subject: [PATCH 054/111] python: Move tree flattening into C - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/database.c | 8 ++- - src/python/export.py | 138 +++++------------------------------------- - 2 files changed, 21 insertions(+), 125 deletions(-) - -diff --git a/src/python/database.c b/src/python/database.c -index 7f8c2c2..d169547 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { - } +- // Skip if the family does not match +- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { ++ // Check if we are interested in this network ++ if (loc_database_enumerator_filter_network(enumerator, *network)) { + loc_network_unref(*network); + *network = NULL;
- static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL }; -+ char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; - const char* country_code = NULL; - unsigned int asn = 0; - int flags = 0; - int family = 0; -+ int flatten = 0; + continue; + }
-- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) - return NULL; +- // Skip if the country code does not match +- if (*enumerator->country_code && +- !loc_network_match_country_code(*network, enumerator->country_code)) { +- loc_network_unref(*network); +- *network = NULL; ++ return 0; ++ } ++ }
- struct loc_database_enumerator* enumerator; -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, -+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -diff --git a/src/python/export.py b/src/python/export.py -index dd44332..be4a68e 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -87,58 +87,12 @@ class OutputWriter(object): - def _write_network(self, network): - self.f.write("%s\n" % network) +- continue; +- } ++ // Reached the end of the search ++ return 0; ++}
-- def write(self, network, subnets): -+ def write(self, network): - if self.flatten and self._flatten(network): - log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) - return +- // Skip if the ASN does not match +- if (enumerator->asn && +- !loc_network_match_asn(*network, enumerator->asn)) { +- loc_network_unref(*network); +- *network = NULL; ++static int __loc_database_enumerator_next_network_flattened( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Fetch the next network ++ int r = __loc_database_enumerator_next_network(enumerator, network, 1); ++ if (r) ++ return r;
-- # Convert network into a Python object -- _network = ipaddress.ip_network("%s" % network) -- -- # Write the network when it has no subnets -- if not subnets: -- log.debug("Writing %s to %s" % (_network, self.f)) -- return self._write_network(_network) -- -- # Convert subnets into Python objects -- _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets] -- -- # Split the network into smaller bits so that -- # we can accomodate for any gaps in it later -- to_check = set() -- for _subnet in _subnets: -- to_check.update( -- _network.address_exclude(_subnet) -- ) -- -- # Clear the list of all subnets -- subnets = [] -- -- # Check if all subnets to not overlap with anything else -- while to_check: -- subnet_to_check = to_check.pop() -- -- for _subnet in _subnets: -- # Drop this subnet if it equals one of the subnets -- # or if it is subnet of one of them -- if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet): -- break -- -- # Break it down if it overlaps -- if subnet_to_check.overlaps(_subnet): -- to_check.update( -- subnet_to_check.address_exclude(_subnet) -- ) -- break -- -- # Add the subnet again as it passed the check -- else: -- subnets.append(subnet_to_check) -- -- # Write all networks as compact as possible -- for network in ipaddress.collapse_addresses(subnets): -- log.debug("Writing %s to %s" % (network, self.f)) -- self._write_network(network) -+ return self._write_network(network) +- continue; +- } ++ // End if we could not read another network ++ if (!*network) ++ return 0;
- def finish(self): - """ -@@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" +- // Skip if flags do not match +- if (enumerator->flags && +- !loc_network_match_flag(*network, enumerator->flags)) { +- loc_network_unref(*network); +- *network = NULL; ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets;
- def _write_network(self, network): -- for address in (network.network_address, network.broadcast_address): -+ for address in (network.first_address, network.last_address): - # Convert this into a string of bits - bytes = socket.inet_pton( - socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, -@@ -231,42 +185,21 @@ class Exporter(object): - writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) +- continue; ++ // Create a list with all subnets ++ r = loc_network_list_new(enumerator->ctx, &subnets); ++ if (r) ++ return r; ++ ++ // Search all subnets from the database ++ while (1) { ++ // Fetch the next network in line ++ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); ++ if (r) { ++ loc_network_unref(subnet); ++ loc_network_list_unref(subnets); ++ ++ return r; ++ } ++ ++ // End if we did not receive another subnet ++ if (!subnet) ++ break; ++ ++ // Collect all subnets in a list ++ if (loc_network_is_subnet(*network, subnet)) { ++ r = loc_network_list_push(subnets, subnet); ++ if (r) { ++ loc_network_unref(subnet); ++ loc_network_list_unref(subnets); ++ ++ return r; + }
- # Get all networks that match the family -- networks = self.db.search_networks(family=family) -- -- # Create a stack with all networks in order where we can put items back -- # again and retrieve them in the next iteration. -- networks = BufferedStack(networks) -+ networks = self.db.search_networks(family=family, flatten=True) +- return 0; ++ loc_network_unref(subnet); ++ continue; + } ++ ++ // If this is not a subnet, we push it back onto the stack and break ++ r = loc_network_list_push(enumerator->stack, subnet); ++ if (r) { ++ loc_network_unref(subnet); ++ loc_network_list_unref(subnets); ++ ++ return r; ++ } ++ ++ loc_network_unref(subnet); ++ break; + }
- # Walk through all networks - for network in networks: -- # Collect all networks which are a subnet of network -- subnets = [] -- for subnet in networks: -- # If the next subnet was not a subnet, we have to push -- # it back on the stack and break this loop -- if not subnet.is_subnet_of(network): -- networks.push(subnet) -- break -- -- subnets.append(subnet) -- - # Write matching countries -- if network.country_code and network.country_code in writers: -- # Mismatching subnets -- gaps = [ -- subnet for subnet in subnets if not network.country_code == subnet.country_code -- ] -- -- writers[network.country_code].write(network, gaps) -+ try: -+ writers[network.country_code].write(network) -+ except KeyError: -+ pass - - # Write matching ASNs -- if network.asn and network.asn in writers: -- # Mismatching subnets -- gaps = [ -- subnet for subnet in subnets if not network.asn == subnet.asn -- ] -- -- writers[network.asn].write(network, gaps) -+ try: -+ writers[network.asn].write(network) -+ except KeyError: -+ pass +- // Reached the end of the search ++ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); ++ ++ // We can abort here if the network has no subnets ++ if (loc_network_list_empty(subnets)) { ++ loc_network_list_unref(subnets); ++ ++ return 0; ++ }
- # Handle flags - for flag in flags: -@@ -274,19 +207,10 @@ class Exporter(object): - # Fetch the "fake" country code - country = flags[flag] +- // Mark all nodes as non-visited +- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) +- enumerator->networks_visited[i] = 0; ++ // If the network has any subnets, we will break it into smaller parts ++ // without the subnets. ++ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); ++ if (!excluded) { ++ loc_network_list_unref(subnets); ++ return -1; ++ } ++ ++ // Merge subnets onto the stack ++ r = loc_network_list_merge(enumerator->stack, subnets); ++ if (r) { ++ loc_network_list_unref(subnets); ++ loc_network_list_unref(excluded); ++ ++ return r; ++ } ++ ++ // Push excluded list onto the stack ++ r = loc_network_list_merge(enumerator->stack, excluded); ++ if (r) { ++ loc_network_list_unref(subnets); ++ loc_network_list_unref(excluded); ++ ++ return r; ++ } ++ ++ loc_network_list_unref(subnets); ++ loc_network_list_unref(excluded); ++ ++ // Replace network with the first one from the stack ++ loc_network_unref(*network); ++ *network = loc_network_list_pop_first(enumerator->stack);
-- if not country in writers: -- continue -- -- gaps = [ -- subnet for subnet in subnets -- if not subnet.has_flag(flag) -- ] -- -- writers[country].write(network, gaps) -- -- # Push all subnets back onto the stack -- for subnet in reversed(subnets): -- networks.push(subnet) -+ try: -+ writers[country].write(network) -+ except KeyError: -+ pass - - # Write everything to the filesystem - for writer in writers.values(): -@@ -298,33 +222,3 @@ class Exporter(object): - ) + return 0; + }
- return os.path.join(directory, filename) -- -- --class BufferedStack(object): -- """ -- This class takes an iterator and when being iterated -- over it returns objects from that iterator for as long -- as there are any. -- -- It additionally has a function to put an item back on -- the back so that it will be returned again at the next -- iteration. -- """ -- def __init__(self, iterator): -- self.iterator = iterator -- self.stack = [] -- -- def __iter__(self): -- return self -- -- def __next__(self): -- if self.stack: -- return self.stack.pop(0) -- -- return next(self.iterator) -- -- def push(self, elem): -- """ -- Takes an element and puts it on the stack -- """ -- self.stack.insert(0, elem) --- -2.20.1 - -From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Sun, 15 Nov 2020 15:02:28 +0000 -Subject: [PATCH 055/111] Move network lists into an own file - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - Makefile.am | 2 + - src/libloc.sym | 1 + - src/loc/network-list.h | 37 +++++++ - src/loc/network.h | 20 +--- - src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++ - src/network.c | 207 +------------------------------------ - src/python/network.c | 1 + - 7 files changed, 269 insertions(+), 223 deletions(-) - create mode 100644 src/loc/network-list.h - create mode 100644 src/network-list.c - -diff --git a/Makefile.am b/Makefile.am -index a0431a6..f0d8c4c 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -96,6 +96,7 @@ pkginclude_HEADERS = \ - src/loc/database.h \ - src/loc/format.h \ - src/loc/network.h \ -+ src/loc/network-list.h \ - src/loc/private.h \ - src/loc/stringpool.h \ - src/loc/resolv.h \ -@@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \ - src/country.c \ - src/database.c \ - src/network.c \ -+ src/network-list.c \ - src/resolv.c \ - src/stringpool.c \ - src/writer.c ++LOC_EXPORT int loc_database_enumerator_next_network( ++ struct loc_database_enumerator* enumerator, struct loc_network** network) { ++ // Do not do anything if not in network mode ++ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) ++ return 0; ++ ++ // Flatten output? ++ if (enumerator->flatten) ++ return __loc_database_enumerator_next_network_flattened(enumerator, network); ++ ++ return __loc_database_enumerator_next_network(enumerator, network, 1); ++} ++ + LOC_EXPORT int loc_database_enumerator_next_country( + struct loc_database_enumerator* enumerator, struct loc_country** country) { + *country = NULL; diff --git a/src/libloc.sym b/src/libloc.sym -index 6139db6..453a1be 100644 +index b8296eb..ee333f1 100644 --- a/src/libloc.sym +++ b/src/libloc.sym -@@ -87,6 +87,7 @@ global: +@@ -37,6 +37,18 @@ global: + loc_as_set_name; + loc_as_unref; + ++ # AS List ++ loc_as_list_append; ++ loc_as_list_clear; ++ loc_as_list_contains; ++ loc_as_list_contains_number; ++ loc_as_list_empty; ++ loc_as_list_get; ++ loc_as_list_new; ++ loc_as_list_ref; ++ loc_as_list_size; ++ loc_as_list_unref; ++ + # Country + loc_country_cmp; + loc_country_code_is_valid; +@@ -49,6 +61,18 @@ global: + loc_country_set_name; + loc_country_unref; + ++ # Country List ++ loc_country_list_append; ++ loc_country_list_clear; ++ loc_country_list_contains; ++ loc_country_list_contains_code; ++ loc_country_list_empty; ++ loc_country_list_get; ++ loc_country_list_new; ++ loc_country_list_ref; ++ loc_country_list_size; ++ loc_country_list_unref; ++ + # Database + loc_database_add_as; + loc_database_count_as; +@@ -66,13 +90,15 @@ global: + loc_database_verify; + + # Database Enumerator ++ loc_database_enumerator_get_asns; ++ loc_database_enumerator_get_countries; + loc_database_enumerator_new; + loc_database_enumerator_next_as; + loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; +- loc_database_enumerator_set_asn; +- loc_database_enumerator_set_country_code; ++ loc_database_enumerator_set_asns; ++ loc_database_enumerator_set_countries; + loc_database_enumerator_set_family; + loc_database_enumerator_set_flag; + loc_database_enumerator_set_string; +@@ -80,24 +106,48 @@ global: + + # Network + loc_network_address_family; ++ loc_network_cmp; ++ loc_network_exclude; ++ loc_network_exclude_list; + loc_network_format_first_address; loc_network_format_last_address; loc_network_get_asn; loc_network_get_country_code; -+ loc_network_gt; ++ loc_network_get_first_address; ++ loc_network_get_last_address; loc_network_has_flag; - loc_network_is_subnet; - loc_network_is_subnet_of; -diff --git a/src/loc/network-list.h b/src/loc/network-list.h +- loc_network_is_subnet_of; ++ loc_network_is_subnet; ++ loc_network_match_address; + loc_network_match_asn; + loc_network_match_country_code; + loc_network_match_flag; + loc_network_new; + loc_network_new_from_string; ++ loc_network_overlaps; ++ loc_network_prefix; + loc_network_ref; + loc_network_set_asn; + loc_network_set_country_code; + loc_network_set_flag; + loc_network_str; ++ loc_network_subnets; + loc_network_unref; + ++ # Network List ++ loc_network_list_clear; ++ loc_network_list_contains; ++ loc_network_list_dump; ++ loc_network_list_empty; ++ loc_network_list_get; ++ loc_network_list_merge; ++ loc_network_list_new; ++ loc_network_list_pop; ++ loc_network_list_pop_first; ++ loc_network_list_push; ++ loc_network_list_ref; ++ loc_network_list_size; ++ loc_network_list_unref; ++ + # Writer + loc_writer_add_as; + loc_writer_add_country; +diff --git a/src/loc/as-list.h b/src/loc/as-list.h new file mode 100644 -index 0000000..af3b28d +index 0000000..7b5c4e8 --- /dev/null -+++ b/src/loc/network-list.h -@@ -0,0 +1,37 @@ ++++ b/src/loc/as-list.h +@@ -0,0 +1,41 @@ +/* + libloc - A library to determine the location of someone on the Internet + -+ Copyright (C) 2020 IPFire Development Team info@ipfire.org ++ Copyright (C) 2017 IPFire Development Team info@ipfire.org + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public @@ -5294,82 +1137,41 @@ index 0000000..af3b28d + Lesser General Public License for more details. +*/ + -+#ifndef LIBLOC_NETWORK_LIST_H -+#define LIBLOC_NETWORK_LIST_H ++#ifndef LIBLOC_AS_LIST_H ++#define LIBLOC_AS_LIST_H + -+struct loc_network_list; -+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); -+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); -+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); -+size_t loc_network_list_size(struct loc_network_list* list); -+int loc_network_list_empty(struct loc_network_list* list); -+void loc_network_list_clear(struct loc_network_list* list); -+void loc_network_list_dump(struct loc_network_list* list); -+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); -+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); -+struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); -+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); -+void loc_network_list_sort(struct loc_network_list* list); -+void loc_network_list_reverse(struct loc_network_list* list); -+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); ++#include <loc/as.h> ++#include <loc/libloc.h> ++ ++struct loc_as_list; ++ ++int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); ++struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); ++struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); ++ ++size_t loc_as_list_size(struct loc_as_list* list); ++int loc_as_list_empty(struct loc_as_list* list); ++void loc_as_list_clear(struct loc_as_list* list); ++ ++struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); ++int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); ++ ++int loc_as_list_contains( ++ struct loc_as_list* list, struct loc_as* as); ++int loc_as_list_contains_number( ++ struct loc_as_list* list, uint32_t number); + +#endif -diff --git a/src/loc/network.h b/src/loc/network.h -index 203e61c..d86b685 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -21,6 +21,7 @@ - - #include <loc/libloc.h> - #include <loc/format.h> -+#include <loc/network-list.h> - - enum loc_network_flags { - LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 -@@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_eq(struct loc_network* self, struct loc_network* other); -+int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); -@@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude( - struct loc_network_list* loc_network_exclude_list( - struct loc_network* network, struct loc_network_list* list); - --// List --struct loc_network_list; --int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); --struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); --struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); --size_t loc_network_list_size(struct loc_network_list* list); --int loc_network_list_empty(struct loc_network_list* list); --void loc_network_list_clear(struct loc_network_list* list); --void loc_network_list_dump(struct loc_network_list* list); --struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); --int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); --struct loc_network* loc_network_list_pop(struct loc_network_list* list); --struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); --int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); --void loc_network_list_sort(struct loc_network_list* list); --void loc_network_list_reverse(struct loc_network_list* list); --int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); -- - #ifdef LIBLOC_PRIVATE - - int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj); -diff --git a/src/network-list.c b/src/network-list.c +diff --git a/src/loc/country-list.h b/src/loc/country-list.h new file mode 100644 -index 0000000..1f6e80e +index 0000000..a7f818a --- /dev/null -+++ b/src/network-list.c -@@ -0,0 +1,224 @@ ++++ b/src/loc/country-list.h +@@ -0,0 +1,43 @@ +/* + libloc - A library to determine the location of someone on the Internet + -+ Copyright (C) 2020 IPFire Development Team info@ipfire.org ++ Copyright (C) 2017 IPFire Development Team info@ipfire.org + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public @@ -5378,511 +1180,166 @@ index 0000000..1f6e80e + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include <errno.h> -+#include <stdlib.h> -+ -+#include <loc/libloc.h> -+#include <loc/network.h> -+#include <loc/private.h> -+ -+struct loc_network_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_network* list[1024]; -+ size_t size; -+ size_t max_size; -+}; -+ -+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -+ struct loc_network_list** list) { -+ struct loc_network_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; -+ -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; -+ -+ DEBUG(l->ctx, "Network list allocated at %p\n", l); -+ *list = l; -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_network_list_free(struct loc_network_list* list) { -+ DEBUG(list->ctx, "Releasing network list at %p\n", list); -+ -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_network_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -+ return list->size == 0; -+} -+ -+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->list[i]); -+ -+ list->size = 0; -+} -+ -+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -+ struct loc_network* network; -+ char* s; -+ -+ for (unsigned int i = 0; i < list->size; i++) { -+ network = list->list[i]; -+ -+ s = loc_network_str(network); -+ -+ INFO(list->ctx, "%s\n", s); -+ free(s); -+ } -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_network_ref(list->list[index]); -+} -+ -+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -+ // Do not add networks that are already on the list -+ if (loc_network_list_contains(list, network)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); -+ return -ENOMEM; -+ } -+ -+ DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); -+ -+ list->list[list->size++] = loc_network_ref(network); -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->list[--list->size]; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->list[0]; ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ // Move all elements to the top of the stack -+ for (unsigned int i = 0; i < --list->size; i++) { -+ list->list[i] = list->list[i+1]; -+ } ++#ifndef LIBLOC_COUNTRY_LIST_H ++#define LIBLOC_COUNTRY_LIST_H + -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++#include <stdlib.h> + -+ return network; -+} ++#include <loc/libloc.h> ++#include <loc/country.h> + -+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_network_eq(list->list[i], network)) -+ return 1; -+ } ++struct loc_country_list; + -+ return 0; -+} ++int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); ++struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); ++struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); + -+static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -+ // Do nothing for invalid indices -+ if (i1 >= list->size || i2 >= list->size) -+ return; ++size_t loc_country_list_size(struct loc_country_list* list); ++int loc_country_list_empty(struct loc_country_list* list); ++void loc_country_list_clear(struct loc_country_list* list); + -+ struct loc_network* network1 = list->list[i1]; -+ struct loc_network* network2 = list->list[i2]; ++struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); ++int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); + -+ list->list[i1] = network2; -+ list->list[i2] = network1; -+} ++int loc_country_list_contains( ++ struct loc_country_list* list, struct loc_country* country); ++int loc_country_list_contains_code( ++ struct loc_country_list* list, const char* code); + -+LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -+ unsigned int i = 0; -+ unsigned int j = list->size - 1; ++#endif +diff --git a/src/loc/database.h b/src/loc/database.h +index 43173dd..70801f0 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -25,6 +25,7 @@ + #include <loc/network.h> + #include <loc/as.h> + #include <loc/country.h> ++#include <loc/country-list.h> + + struct loc_database; + int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); +@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode { + LOC_DB_ENUMERATE_COUNTRIES = 3, + }; + ++enum loc_database_enumerator_flags { ++ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), ++}; + -+ while (i < j) { -+ loc_network_list_swap(list, i++, j--); -+ } -+} + struct loc_database_enumerator; + int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, +- struct loc_database* db, enum loc_database_enumerator_mode mode); ++ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); + struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); + struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); + + int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); +-int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code); +-int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); ++struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_countries( ++ struct loc_database_enumerator* enumerator, struct loc_country_list* countries); ++struct loc_as_list* loc_database_enumerator_get_asns( ++ struct loc_database_enumerator* enumerator); ++int loc_database_enumerator_set_asns( ++ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); + int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); + int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); + int loc_database_enumerator_next_as( +diff --git a/src/loc/network-list.h b/src/loc/network-list.h +new file mode 100644 +index 0000000..bee21c4 +--- /dev/null ++++ b/src/loc/network-list.h +@@ -0,0 +1,37 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet + -+LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -+ unsigned int n = list->size; -+ int swapped; ++ Copyright (C) 2020 IPFire Development Team info@ipfire.org + -+ do { -+ swapped = 0; ++ This library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. + -+ for (unsigned int i = 1; i < n; i++) { -+ if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -+ loc_network_list_swap(list, i-1, i); -+ swapped = 1; -+ } -+ } ++ This library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++*/ + -+ n--; -+ } while (swapped); -+} ++#ifndef LIBLOC_NETWORK_LIST_H ++#define LIBLOC_NETWORK_LIST_H + -+LOC_EXPORT int loc_network_list_merge( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; ++#include <loc/network.h> + -+ for (unsigned int i = 0; i < other->size; i++) { -+ r = loc_network_list_push(self, other->list[i]); -+ if (r) -+ return r; -+ } ++struct loc_network_list; ++int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); ++struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); ++struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); ++size_t loc_network_list_size(struct loc_network_list* list); ++int loc_network_list_empty(struct loc_network_list* list); ++void loc_network_list_clear(struct loc_network_list* list); ++void loc_network_list_dump(struct loc_network_list* list); ++struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); ++int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); ++struct loc_network* loc_network_list_pop(struct loc_network_list* list); ++struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); ++int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); ++int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); + -+ return 0; -+} -diff --git a/src/network.c b/src/network.c -index d015579..28ca2df 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -29,6 +29,7 @@ - #include <loc/compat.h> - #include <loc/country.h> - #include <loc/network.h> ++#endif +diff --git a/src/loc/network.h b/src/loc/network.h +index 70c3803..af3dafd 100644 +--- a/src/loc/network.h ++++ b/src/loc/network.h +@@ -21,6 +21,7 @@ + + #include <loc/libloc.h> + #include <loc/format.h> +#include <loc/network-list.h> - #include <loc/private.h>
- struct loc_network { -@@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe - return 1; - } + enum loc_network_flags { + LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 +@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network); + struct loc_network* loc_network_unref(struct loc_network* network); + char* loc_network_str(struct loc_network* network); + int loc_network_address_family(struct loc_network* network); ++unsigned int loc_network_prefix(struct loc_network* network);
--static int loc_network_gt(struct loc_network* self, struct loc_network* other) { -+LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { - // Families must match - if (self->family != other->family) - return -1; -@@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { - struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { - return loc_network_ref(node->network); - } -- --// List -- --struct loc_network_list { -- struct loc_ctx* ctx; -- int refcount; -- -- struct loc_network* list[1024]; -- size_t size; -- size_t max_size; --}; -- --LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -- struct loc_network_list** list) { -- struct loc_network_list* l = calloc(1, sizeof(*l)); -- if (!l) -- return -ENOMEM; -- -- l->ctx = loc_ref(ctx); -- l->refcount = 1; -- -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- -- DEBUG(l->ctx, "Network list allocated at %p\n", l); -- *list = l; -- return 0; --} -- --LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { -- list->refcount++; -- -- return list; --} -- --static void loc_network_list_free(struct loc_network_list* list) { -- DEBUG(list->ctx, "Releasing network list at %p\n", list); -- -- for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -- -- loc_unref(list->ctx); -- free(list); --} -- --LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { -- if (!list) -- return NULL; -- -- if (--list->refcount > 0) -- return list; -- -- loc_network_list_free(list); -- return NULL; --} -- --LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -- return list->size; --} -- --LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -- return list->size == 0; --} -- --LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -- for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -- -- list->size = 0; --} -- --LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -- struct loc_network* network; -- char* s; -- -- for (unsigned int i = 0; i < list->size; i++) { -- network = list->list[i]; -- -- s = loc_network_str(network); -- -- INFO(list->ctx, "%s\n", s); -- free(s); -- } --} -- --LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { -- // Check index -- if (index >= list->size) -- return NULL; -- -- return loc_network_ref(list->list[index]); --} -- --LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -- // Do not add networks that are already on the list -- if (loc_network_list_contains(list, network)) -- return 0; -- -- // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); -- return -ENOMEM; -- } -- -- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); -- -- list->list[list->size++] = loc_network_ref(network); -- -- return 0; --} -- --LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { -- // Return nothing when empty -- if (loc_network_list_empty(list)) { -- DEBUG(list->ctx, "%p: Popped empty stack\n", list); -- return NULL; -- } -- -- struct loc_network* network = list->list[--list->size]; -- -- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -- -- return network; --} -- --LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { -- // Return nothing when empty -- if (loc_network_list_empty(list)) { -- DEBUG(list->ctx, "%p: Popped empty stack\n", list); -- return NULL; -- } -- -- struct loc_network* network = list->list[0]; -- -- // Move all elements to the top of the stack -- for (unsigned int i = 0; i < --list->size; i++) { -- list->list[i] = list->list[i+1]; -- } -- -- DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -- -- return network; --} -- --LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -- for (unsigned int i = 0; i < list->size; i++) { -- if (loc_network_eq(list->list[i], network)) -- return 1; -- } -- -- return 0; --} -- --static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -- // Do nothing for invalid indices -- if (i1 >= list->size || i2 >= list->size) -- return; -- -- struct loc_network* network1 = list->list[i1]; -- struct loc_network* network2 = list->list[i2]; -- -- list->list[i1] = network2; -- list->list[i2] = network1; --} -- --LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -- unsigned int i = 0; -- unsigned int j = list->size - 1; -- -- while (i < j) { -- loc_network_list_swap(list, i++, j--); -- } --} -- --LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -- unsigned int n = list->size; -- int swapped; -- -- do { -- swapped = 0; -- -- for (unsigned int i = 1; i < n; i++) { -- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -- loc_network_list_swap(list, i-1, i); -- swapped = 1; -- } -- } -- -- n--; -- } while (swapped); --} -- --LOC_EXPORT int loc_network_list_merge( -- struct loc_network_list* self, struct loc_network_list* other) { -- int r; -- -- for (unsigned int i = 0; i < other->size; i++) { -- r = loc_network_list_push(self, other->list[i]); -- if (r) -- return r; -- } -- -- return 0; --} -diff --git a/src/python/network.c b/src/python/network.c -index 11f672b..ed91d65 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -20,6 +20,7 @@ ++const struct in6_addr* loc_network_get_first_address(struct loc_network* network); + char* loc_network_format_first_address(struct loc_network* network); ++const struct in6_addr* loc_network_get_last_address(struct loc_network* network); + char* loc_network_format_last_address(struct loc_network* network); + int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
- #include <loc/libloc.h> - #include <loc/network.h> -+#include <loc/network-list.h> +@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); + int loc_network_set_flag(struct loc_network* network, uint32_t flag); + int loc_network_match_flag(struct loc_network* network, uint32_t flag); + +-int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); ++int loc_network_cmp(struct loc_network* self, struct loc_network* other); ++int loc_network_overlaps(struct loc_network* self, struct loc_network* other); ++int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); ++int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); ++struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other); ++struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list); + + #ifdef LIBLOC_PRIVATE
- #include "locationmodule.h" - #include "network.h" --- -2.20.1 - -From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Mon, 16 Nov 2020 15:13:28 +0000 -Subject: [PATCH 056/111] Implement filtering for multiple countries in the - enumerator - -This will allow us to speed up the export of the database -if only a few countries should be returned. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - Makefile.am | 2 + - src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++ - src/country.c | 3 + - src/database.c | 47 ++++++-------- - src/libloc.sym | 15 ++++- - src/loc/country-list.h | 43 +++++++++++++ - src/loc/database.h | 5 +- - src/python/database.c | 57 ++++++++++++++--- - 8 files changed, 274 insertions(+), 36 deletions(-) - create mode 100644 src/country-list.c - create mode 100644 src/loc/country-list.h - -diff --git a/Makefile.am b/Makefile.am -index f0d8c4c..f4ca3c8 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -93,6 +93,7 @@ pkginclude_HEADERS = \ - src/loc/as.h \ - src/loc/compat.h \ - src/loc/country.h \ -+ src/loc/country-list.h \ - src/loc/database.h \ - src/loc/format.h \ - src/loc/network.h \ -@@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \ - src/libloc.c \ - src/as.c \ - src/country.c \ -+ src/country-list.c \ - src/database.c \ - src/network.c \ - src/network-list.c \ -diff --git a/src/country-list.c b/src/country-list.c +diff --git a/src/network-list.c b/src/network-list.c new file mode 100644 -index 0000000..ae0d71a +index 0000000..e2b20f3 --- /dev/null -+++ b/src/country-list.c -@@ -0,0 +1,138 @@ ++++ b/src/network-list.c +@@ -0,0 +1,295 @@ +/* + libloc - A library to determine the location of someone on the Internet + @@ -5901,2068 +1358,1139 @@ index 0000000..ae0d71a + +#include <errno.h> +#include <stdlib.h> ++#include <time.h> + -+#include <loc/country.h> -+#include <loc/country-list.h> ++#include <loc/libloc.h> ++#include <loc/network.h> +#include <loc/private.h> + -+struct loc_country_list { ++struct loc_network_list { + struct loc_ctx* ctx; + int refcount; + -+ struct loc_country* list[1024]; ++ struct loc_network** elements; ++ size_t elements_size; ++ + size_t size; -+ size_t max_size; +}; + -+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, -+ struct loc_country_list** list) { -+ struct loc_country_list* l = calloc(1, sizeof(*l)); ++static int loc_network_list_grow(struct loc_network_list* list, size_t size) { ++ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", ++ list, size, list->elements_size + size); ++ ++ struct loc_network** elements = reallocarray(list->elements, ++ list->elements_size + size, sizeof(*list->elements)); ++ if (!elements) ++ return -errno; ++ ++ list->elements = elements; ++ list->elements_size += size; ++ ++ return 0; ++} ++ ++LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, ++ struct loc_network_list** list) { ++ struct loc_network_list* l = calloc(1, sizeof(*l)); + if (!l) + return -ENOMEM; + + l->ctx = loc_ref(ctx); + l->refcount = 1; + -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; -+ -+ DEBUG(l->ctx, "Country list allocated at %p\n", l); ++ DEBUG(l->ctx, "Network list allocated at %p\n", l); + *list = l; -+ + return 0; +} + -+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { ++LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { + list->refcount++; + + return list; +} + -+static void loc_country_list_free(struct loc_country_list* list) { -+ DEBUG(list->ctx, "Releasing country list at %p\n", list); ++static void loc_network_list_free(struct loc_network_list* list) { ++ DEBUG(list->ctx, "Releasing network list at %p\n", list); + -+ loc_country_list_clear(list); ++ // Remove all content ++ loc_network_list_clear(list); + + loc_unref(list->ctx); + free(list); +} + -+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { ++LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { + if (!list) + return NULL; + + if (--list->refcount > 0) + return list; + -+ loc_country_list_free(list); ++ loc_network_list_free(list); + return NULL; +} + -+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { ++LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { + return list->size; +} + -+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { ++LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { + return list->size == 0; +} + -+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_country_unref(list->list[i]); -+} -+ -+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_country_ref(list->list[index]); -+} -+ -+LOC_EXPORT int loc_country_list_append( -+ struct loc_country_list* list, struct loc_country* country) { -+ if (loc_country_list_contains(list, country)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); -+ return -ENOMEM; -+ } -+ -+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); -+ -+ list->list[list->size++] = loc_country_ref(country); -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_country_cmp(country, list->list[i]) == 0) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code) { -+ struct loc_country* country; -+ -+ int r = loc_country_new(list->ctx, &country, code); -+ if (r) -+ return -1; -+ -+ r = loc_country_list_contains(list, country); -+ loc_country_unref(country); -+ -+ return r; -+} -diff --git a/src/country.c b/src/country.c -index 2ba93e6..7aac0db 100644 ---- a/src/country.c -+++ b/src/country.c -@@ -34,6 +34,9 @@ struct loc_country { - }; - - LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { -+ if (!loc_country_code_is_valid(country_code)) -+ return -EINVAL; -+ - struct loc_country* c = calloc(1, sizeof(*c)); - if (!c) - return -ENOMEM; -diff --git a/src/database.c b/src/database.c -index b9d870f..29823b2 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -40,6 +40,7 @@ - #include <loc/as.h> - #include <loc/compat.h> - #include <loc/country.h> -+#include <loc/country-list.h> - #include <loc/database.h> - #include <loc/format.h> - #include <loc/network.h> -@@ -99,7 +100,7 @@ struct loc_database_enumerator { - - // Search string - char* string; -- char country_code[3]; -+ struct loc_country_list* countries; - uint32_t asn; - enum loc_network_flags flags; - int family; -@@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator - return 0; - } - --LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { -- // Set empty country code -- if (!country_code || !*country_code) { -- *enumerator->country_code = '\0'; -- return 0; -- } -+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->countries) -+ return NULL; - -- // Treat A1, A2, A3 as special country codes, -- // but perform search for flags instead -- if (strcmp(country_code, "A1") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); -- } else if (strcmp(country_code, "A2") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); -- } else if (strcmp(country_code, "A3") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANYCAST); -- } -+ return loc_country_list_ref(enumerator->countries); -+} - -- // Country codes must be two characters -- if (!loc_country_code_is_valid(country_code)) -- return -EINVAL; -+LOC_EXPORT int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); - -- for (unsigned int i = 0; i < 3; i++) { -- enumerator->country_code[i] = country_code[i]; -- } -+ enumerator->countries = loc_country_list_ref(countries); - - return 0; - } -@@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - -+static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { -+ const char* country_code = loc_network_get_country_code(network); -+ -+ return loc_country_list_contains_code(countries, country_code); -+} ++LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { ++ if (!list->elements) ++ return; + - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -@@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network( - return 1; - - // Skip if the country code does not match -- if (*enumerator->country_code && -- !loc_network_match_country_code(network, enumerator->country_code)) -+ if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) - return 1; - - // Skip if the ASN does not match -diff --git a/src/libloc.sym b/src/libloc.sym -index 453a1be..40e9f88 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -49,6 +49,18 @@ global: - loc_country_set_name; - loc_country_unref; - -+ # Country List -+ loc_country_list_append; -+ loc_country_list_clear; -+ loc_country_list_contains; -+ loc_country_list_contains_code; -+ loc_country_list_empty; -+ loc_country_list_get; -+ loc_country_list_new; -+ loc_country_list_ref; -+ loc_country_list_size; -+ loc_country_list_unref; ++ for (unsigned int i = 0; i < list->size; i++) ++ loc_network_unref(list->elements[i]); + - # Database - loc_database_add_as; - loc_database_count_as; -@@ -66,13 +78,14 @@ global: - loc_database_verify; - - # Database Enumerator -+ loc_database_enumerator_get_countries; - loc_database_enumerator_new; - loc_database_enumerator_next_as; - loc_database_enumerator_next_country; - loc_database_enumerator_next_network; - loc_database_enumerator_ref; - loc_database_enumerator_set_asn; -- loc_database_enumerator_set_country_code; -+ loc_database_enumerator_set_countries; - loc_database_enumerator_set_family; - loc_database_enumerator_set_flag; - loc_database_enumerator_set_string; -diff --git a/src/loc/country-list.h b/src/loc/country-list.h -new file mode 100644 -index 0000000..a7f818a ---- /dev/null -+++ b/src/loc/country-list.h -@@ -0,0 +1,43 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet ++ free(list->elements); ++ list->elements = NULL; ++ list->elements_size = 0; + -+ Copyright (C) 2017 IPFire Development Team info@ipfire.org ++ list->size = 0; ++} + -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. ++LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { ++ struct loc_network* network; ++ char* s; + -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ ++ for (unsigned int i = 0; i < list->size; i++) { ++ network = list->elements[i]; + -+#ifndef LIBLOC_COUNTRY_LIST_H -+#define LIBLOC_COUNTRY_LIST_H ++ s = loc_network_str(network); + -+#include <stdlib.h> ++ INFO(list->ctx, "%4d: %s\n", i, s); ++ free(s); ++ } ++} + -+#include <loc/libloc.h> -+#include <loc/country.h> ++LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { ++ // Check index ++ if (index >= list->size) ++ return NULL; + -+struct loc_country_list; ++ return loc_network_ref(list->elements[index]); ++} + -+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); -+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); -+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); ++static off_t loc_network_list_find(struct loc_network_list* list, ++ struct loc_network* network, int* found) { ++ off_t lo = 0; ++ off_t hi = list->size - 1; ++ int result; + -+size_t loc_country_list_size(struct loc_country_list* list); -+int loc_country_list_empty(struct loc_country_list* list); -+void loc_country_list_clear(struct loc_country_list* list); ++ // Since we are working on an ordered list, there is often a good chance that ++ // the network we are looking for is at the end or has to go to the end. ++ if (hi >= 0) { ++ result = loc_network_cmp(network, list->elements[hi]); + -+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); -+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); ++ // Match, so we are done ++ if (result == 0) { ++ *found = 1; + -+int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country); -+int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code); ++ return hi; ++ ++ // This needs to be added after the last one ++ } else if (result > 0) { ++ *found = 0; ++ ++ return hi + 1; ++ } ++ } + ++#ifdef ENABLE_DEBUG ++ // Save start time ++ clock_t start = clock(); +#endif -diff --git a/src/loc/database.h b/src/loc/database.h -index 14eb5ea..246e5c5 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -25,6 +25,7 @@ - #include <loc/network.h> - #include <loc/as.h> - #include <loc/country.h> -+#include <loc/country-list.h> - - struct loc_database; - int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); -@@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_ - struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); - - int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); --int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code); -+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); -+int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries); - int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); - int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); - int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); -diff --git a/src/python/database.c b/src/python/database.c -index d169547..e6f6f37 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { - } - - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -- char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL }; -- const char* country_code = NULL; -+ char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; -+ PyObject* country_codes = NULL; - unsigned int asn = 0; - int flags = 0; - int family = 0; - int flatten = 0; - -- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, -+ &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) - return NULL; - - struct loc_database_enumerator* enumerator; -@@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - } - - // Set country code we are searching for -- if (country_code) { -- r = loc_database_enumerator_set_country_code(enumerator, country_code); -- -+ if (country_codes) { -+ struct loc_country_list* countries; -+ r = loc_country_list_new(loc_ctx, &countries); - if (r) { -- PyErr_SetFromErrno(PyExc_SystemError); -+ PyErr_SetString(PyExc_SystemError, "Could not create country list"); - return NULL; - } + -+ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) { -+ PyObject* item = PyList_GetItem(country_codes, i); ++ off_t i = 0; + -+ if (!PyUnicode_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); -+ loc_country_list_unref(countries); -+ return NULL; -+ } ++ while (lo <= hi) { ++ i = (lo + hi) / 2; + -+ const char* country_code = PyUnicode_AsUTF8(item); ++ // Check if this is a match ++ result = loc_network_cmp(network, list->elements[i]); + -+ struct loc_country* country; -+ r = loc_country_new(loc_ctx, &country, country_code); -+ if (r) { -+ if (r == -EINVAL) { -+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); -+ } else { -+ PyErr_SetString(PyExc_SystemError, "Could not create country"); -+ } ++ if (result == 0) { ++ *found = 1; + -+ loc_country_list_unref(countries); -+ return NULL; -+ } ++#ifdef ENABLE_DEBUG ++ clock_t end = clock(); + -+ // Append it to the list -+ r = loc_country_list_append(countries, country); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); ++ // Log how fast this has been ++ DEBUG(list->ctx, "Found network in %.4fms at %jd\n", ++ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); ++#endif + -+ loc_country_list_unref(countries); -+ loc_country_unref(country); -+ return NULL; -+ } ++ return i; ++ } + -+ loc_country_unref(country); ++ if (result > 0) { ++ lo = i + 1; ++ i++; ++ } else { ++ hi = i - 1; + } ++ } + -+ loc_database_enumerator_set_countries(enumerator, countries); ++ *found = 0; + -+ Py_DECREF(country_codes); -+ loc_country_list_unref(countries); - } - - // Set the ASN we are searching for --- -2.20.1 - -From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Mon, 16 Nov 2020 15:20:50 +0000 -Subject: [PATCH 057/111] python: Only return country codes we want - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - -diff --git a/src/python/export.py b/src/python/export.py -index be4a68e..5e7fe53 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -184,8 +184,14 @@ class Exporter(object): - - writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) - -+ # Filter countries from special country codes -+ country_codes = [ -+ country_code for country_code in countries if not country_code in flags.values() -+ ] ++#ifdef ENABLE_DEBUG ++ clock_t end = clock(); + - # Get all networks that match the family -- networks = self.db.search_networks(family=family, flatten=True) -+ networks = self.db.search_networks(family=family, -+ country_codes=country_codes, flatten=True) - - # Walk through all networks - for network in networks: --- -2.20.1 - -From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Mon, 16 Nov 2020 15:25:15 +0000 -Subject: [PATCH 058/111] database: Filter flags in C - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 16 +++++++++++----- - 1 file changed, 11 insertions(+), 5 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 5e7fe53..739742f 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -29,7 +29,7 @@ import _location - log = logging.getLogger("location.export") - log.propagate = 1 - --flags = { -+FLAGS = { - _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", - _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", - _location.NETWORK_FLAG_ANYCAST : "A3", -@@ -186,12 +186,18 @@ class Exporter(object): - - # Filter countries from special country codes - country_codes = [ -- country_code for country_code in countries if not country_code in flags.values() -+ country_code for country_code in countries if not country_code in FLAGS.values() - ] - -+ # Collect flags -+ flags = 0 -+ for flag in FLAGS: -+ if FLAGS[flag] in countries: -+ flags |= flag ++ // Log how fast this has been ++ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n", ++ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); ++#endif + - # Get all networks that match the family - networks = self.db.search_networks(family=family, -- country_codes=country_codes, flatten=True) -+ country_codes=country_codes, flags=flags, flatten=True) - - # Walk through all networks - for network in networks: -@@ -208,10 +214,10 @@ class Exporter(object): - pass - - # Handle flags -- for flag in flags: -+ for flag in FLAGS: - if network.has_flag(flag): - # Fetch the "fake" country code -- country = flags[flag] -+ country = FLAGS[flag] - - try: - writers[country].write(network) --- -2.20.1 - -From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 16:46:48 +0000 -Subject: [PATCH 059/111] as: Add list for easier processing - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - Makefile.am | 2 + - src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++ - src/database.c | 29 +++++++-- - src/libloc.sym | 15 ++++- - src/loc/as-list.h | 41 +++++++++++++ - src/loc/database.h | 5 +- - src/python/database.c | 58 ++++++++++++++++-- - src/python/export.py | 2 +- - 8 files changed, 275 insertions(+), 15 deletions(-) - create mode 100644 src/as-list.c - create mode 100644 src/loc/as-list.h - -diff --git a/Makefile.am b/Makefile.am -index f4ca3c8..d0cc793 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -91,6 +91,7 @@ EXTRA_DIST += \ - pkginclude_HEADERS = \ - src/loc/libloc.h \ - src/loc/as.h \ -+ src/loc/as-list.h \ - src/loc/compat.h \ - src/loc/country.h \ - src/loc/country-list.h \ -@@ -109,6 +110,7 @@ lib_LTLIBRARIES = \ - src_libloc_la_SOURCES = \ - src/libloc.c \ - src/as.c \ -+ src/as-list.c \ - src/country.c \ - src/country-list.c \ - src/database.c \ -diff --git a/src/as-list.c b/src/as-list.c -new file mode 100644 -index 0000000..7c69eb0 ---- /dev/null -+++ b/src/as-list.c -@@ -0,0 +1,138 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet ++ return i; ++} ++ ++LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { ++ int found = 0; ++ ++ off_t index = loc_network_list_find(list, network, &found); ++ ++ // The network has been found on the list. Nothing to do. ++ if (found) ++ return 0; ++ ++ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n", ++ list, network, (intmax_t)index); + -+ Copyright (C) 2020 IPFire Development Team info@ipfire.org ++ // Check if we have space left ++ if (list->size >= list->elements_size) { ++ int r = loc_network_list_grow(list, 64); ++ if (r) ++ return r; ++ } + -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. ++ // The list is now larger ++ list->size++; + -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ ++ // Move all elements out of the way ++ for (unsigned int i = list->size - 1; i > index; i--) ++ list->elements[i] = list->elements[i - 1]; + -+#include <errno.h> -+#include <stdlib.h> ++ // Add the new element at the right place ++ list->elements[index] = loc_network_ref(network); + -+#include <loc/as.h> -+#include <loc/as-list.h> -+#include <loc/private.h> ++ return 0; ++} + -+struct loc_as_list { -+ struct loc_ctx* ctx; -+ int refcount; ++LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } + -+ struct loc_as* list[1024]; -+ size_t size; -+ size_t max_size; -+}; ++ struct loc_network* network = list->elements[--list->size]; + -+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, -+ struct loc_as_list** list) { -+ struct loc_as_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); + -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; ++ return network; ++} ++ ++LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { ++ // Return nothing when empty ++ if (loc_network_list_empty(list)) { ++ DEBUG(list->ctx, "%p: Popped empty stack\n", list); ++ return NULL; ++ } + -+ // Do not allow this list to grow larger than this -+ l->max_size = 1024; ++ struct loc_network* network = list->elements[0]; + -+ DEBUG(l->ctx, "AS list allocated at %p\n", l); -+ *list = l; ++ // Move all elements to the top of the stack ++ for (unsigned int i = 0; i < list->size - 1; i++) { ++ list->elements[i] = list->elements[i+1]; ++ } ++ ++ // The list is shorter now ++ --list->size; ++ ++ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); ++ ++ return network; ++} ++ ++LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { ++ int found = 0; ++ ++ loc_network_list_find(list, network, &found); ++ ++ return found; ++} ++ ++LOC_EXPORT int loc_network_list_merge( ++ struct loc_network_list* self, struct loc_network_list* other) { ++ int r; ++ ++ for (unsigned int i = 0; i < other->size; i++) { ++ r = loc_network_list_push(self, other->elements[i]); ++ if (r) ++ return r; ++ } + + return 0; +} +diff --git a/src/network.c b/src/network.c +index 366caa2..a6b679c 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -29,6 +29,7 @@ + #include <loc/compat.h> + #include <loc/country.h> + #include <loc/network.h> ++#include <loc/network-list.h> + #include <loc/private.h> + + struct loc_network { +@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s + return a; + } + ++static struct in6_addr address_increment(const struct in6_addr* address) { ++ struct in6_addr a = *address; + -+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { -+ list->refcount++; ++ for (int octet = 15; octet >= 0; octet--) { ++ if (a.s6_addr[octet] < 255) { ++ a.s6_addr[octet]++; ++ break; ++ } else { ++ a.s6_addr[octet] = 0; ++ } ++ } + -+ return list; ++ return a; +} + -+static void loc_as_list_free(struct loc_as_list* list) { -+ DEBUG(list->ctx, "Releasing AS list at %p\n", list); + LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, + struct in6_addr* address, unsigned int prefix) { + // Address cannot be unspecified +@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network + LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, + const char* address_string) { + struct in6_addr first_address; +- unsigned int prefix = 0; + char* prefix_string; +- int r = 1; ++ unsigned int prefix = 128; ++ int r = -EINVAL; + -+ loc_as_list_clear(list); ++ DEBUG(ctx, "Attempting to parse network %s\n", address_string); + + // Make a copy of the string to work on it + char* buffer = strdup(address_string); +@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo + // Split address and prefix + address_string = strsep(&prefix_string, "/"); + +- // Did we find a prefix? ++ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); + -+ loc_unref(list->ctx); -+ free(list); ++ // Parse the address ++ r = loc_parse_address(ctx, address_string, &first_address); ++ if (r) { ++ DEBUG(ctx, "The address could not be parsed\n"); ++ goto FAIL; ++ } ++ ++ // If a prefix was given, we will try to parse it + if (prefix_string) { + // Convert prefix to integer + prefix = strtol(prefix_string, NULL, 10); + +- if (prefix) { +- // Parse the address +- r = loc_parse_address(ctx, address_string, &first_address); +- +- // Map the prefix to IPv6 if needed +- if (IN6_IS_ADDR_V4MAPPED(&first_address)) +- prefix += 96; ++ if (!prefix) { ++ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); ++ goto FAIL; + } ++ ++ // Map the prefix to IPv6 if needed ++ if (IN6_IS_ADDR_V4MAPPED(&first_address)) ++ prefix += 96; + } + ++FAIL: + // Free temporary buffer + free(buffer); + +- if (r == 0) { +- r = loc_network_new(ctx, network, &first_address, prefix); +- } ++ // Exit if the parsing was unsuccessful ++ if (r) ++ return r; + +- return r; ++ // Create a new network ++ return loc_network_new(ctx, network, &first_address, prefix); + } + + LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { +@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) { + return network->family; + } + ++LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) { ++ switch (network->family) { ++ case AF_INET6: ++ return network->prefix; ++ ++ case AF_INET: ++ return network->prefix - 96; ++ } ++ ++ return 0; +} + -+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { -+ if (!list) -+ return NULL; + static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) { + const size_t length = INET6_ADDRSTRLEN; + +@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc + return string; + } + ++LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) { ++ return &network->first_address; ++} + -+ if (--list->refcount > 0) -+ return list; + LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) { + return loc_network_format_address(network, &network->first_address); + } + ++LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) { ++ return &network->last_address; ++} ++ + LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { + return loc_network_format_address(network, &network->last_address); + } +@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { + LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) { + // Address must be larger than the start address + if (in6_addr_cmp(&network->first_address, address) > 0) +- return 1; ++ return 0; + + // Address must be smaller than the last address + if (in6_addr_cmp(&network->last_address, address) < 0) +- return 1; ++ return 0; + + // The address is inside this network +- return 0; ++ return 1; + } + + LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) { +@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag + return loc_network_has_flag(network, flag); + } + +-LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { ++LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { ++ // Compare address ++ int r = in6_addr_cmp(&self->first_address, &other->first_address); ++ if (r) ++ return r; ++ ++ // Compare prefix ++ if (self->prefix > other->prefix) ++ return 1; ++ else if (self->prefix < other->prefix) ++ return -1; + -+ loc_as_list_free(list); -+ return NULL; ++ // Both networks are equal ++ return 0; +} + -+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { -+ return list->size; -+} ++LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { ++ // Either of the start addresses must be in the other subnet ++ if (loc_network_match_address(self, &other->first_address)) ++ return 1; + -+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { -+ return list->size == 0; -+} ++ if (loc_network_match_address(other, &self->first_address)) ++ return 1; + -+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_as_unref(list->list[i]); -+} ++ // Or either of the end addresses is in the other subnet ++ if (loc_network_match_address(self, &other->last_address)) ++ return 1; + -+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; ++ if (loc_network_match_address(other, &self->last_address)) ++ return 1; + -+ return loc_as_ref(list->list[index]); ++ return 0; +} + -+LOC_EXPORT int loc_as_list_append( -+ struct loc_as_list* list, struct loc_as* as) { -+ if (loc_as_list_contains(list, as)) ++LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { ++ // The prefix must be smaller (this avoids the more complex comparisons later) ++ if (self->prefix > other->prefix) + return 0; + -+ // Check if we have space left -+ if (list->size == list->max_size) { -+ ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); -+ return -ENOMEM; -+ } + // If the start address of the other network is smaller than this network, + // it cannot be a subnet. +- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) ++ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) + return 0; + + // If the end address of the other network is greater than this network, + // it cannot be a subnet. +- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) ++ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0) + return 0; + + return 1; + } + ++LOC_EXPORT int loc_network_subnets(struct loc_network* network, ++ struct loc_network** subnet1, struct loc_network** subnet2) { ++ int r; ++ *subnet1 = NULL; ++ *subnet2 = NULL; + -+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); ++ // New prefix length ++ unsigned int prefix = network->prefix + 1; ++ ++ // Check if the new prefix is valid ++ if (valid_prefix(&network->first_address, prefix)) ++ return -1; + -+ list->list[list->size++] = loc_as_ref(as); ++ // Create the first half of the network ++ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix); ++ if (r) ++ return r; + -+ return 0; -+} ++ // The next subnet starts after the first one ++ struct in6_addr first_address = address_increment(&(*subnet1)->last_address); + -+LOC_EXPORT int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_as_cmp(as, list->list[i]) == 0) -+ return 1; ++ // Create the second half of the network ++ r = loc_network_new(network->ctx, subnet2, &first_address, prefix); ++ if (r) ++ return r; ++ ++ // Copy country code ++ const char* country_code = loc_network_get_country_code(network); ++ if (country_code) { ++ loc_network_set_country_code(*subnet1, country_code); ++ loc_network_set_country_code(*subnet2, country_code); ++ } ++ ++ // Copy ASN ++ uint32_t asn = loc_network_get_asn(network); ++ if (asn) { ++ loc_network_set_asn(*subnet1, asn); ++ loc_network_set_asn(*subnet2, asn); + } + ++ // Copy flags ++ loc_network_set_flag(*subnet1, network->flags); ++ loc_network_set_flag(*subnet2, network->flags); ++ + return 0; +} + -+LOC_EXPORT int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number) { -+ struct loc_as* as; ++static int __loc_network_exclude(struct loc_network* network, ++ struct loc_network* other, struct loc_network_list* list) { ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; + -+ int r = loc_as_new(list->ctx, &as, number); ++ int r = loc_network_subnets(network, &subnet1, &subnet2); + if (r) -+ return -1; ++ goto ERROR; + -+ r = loc_as_list_contains(list, as); -+ loc_as_unref(as); ++ if (loc_network_cmp(other, subnet1) == 0) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; + -+ return r; -+} -diff --git a/src/database.c b/src/database.c -index 29823b2..51cb5cd 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -38,6 +38,7 @@ - - #include <loc/libloc.h> - #include <loc/as.h> -+#include <loc/as-list.h> - #include <loc/compat.h> - #include <loc/country.h> - #include <loc/country-list.h> -@@ -101,7 +102,7 @@ struct loc_database_enumerator { - // Search string - char* string; - struct loc_country_list* countries; -- uint32_t asn; -+ struct loc_as_list* asns; - enum loc_network_flags flags; - int family; - -@@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries( - return 0; - } - --LOC_EXPORT int loc_database_enumerator_set_asn( -- struct loc_database_enumerator* enumerator, unsigned int asn) { -- enumerator->asn = asn; -+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->asns) -+ return NULL; ++ } else if (loc_network_cmp(other, subnet2) == 0) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; + -+ return loc_as_list_ref(enumerator->asns); -+} ++ } else if (loc_network_is_subnet(subnet1, other)) { ++ r = loc_network_list_push(list, subnet2); ++ if (r) ++ goto ERROR; + -+LOC_EXPORT int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); ++ r = __loc_network_exclude(subnet1, other, list); ++ if (r) ++ goto ERROR; + -+ enumerator->asns = loc_as_list_ref(asns); - - return 0; - } -@@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c - return loc_country_list_contains_code(countries, country_code); - } - -+static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { -+ uint32_t asn = loc_network_get_asn(network); ++ } else if (loc_network_is_subnet(subnet2, other)) { ++ r = loc_network_list_push(list, subnet1); ++ if (r) ++ goto ERROR; + -+ return loc_as_list_contains_number(asns, asn); -+} ++ r = __loc_network_exclude(subnet2, other, list); ++ if (r) ++ goto ERROR; + - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -@@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network( - return 1; - - // Skip if the ASN does not match -- if (enumerator->asn && -- !loc_network_match_asn(network, enumerator->asn)) -+ if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) - return 1; - - // Skip if flags do not match -diff --git a/src/libloc.sym b/src/libloc.sym -index 40e9f88..53273cd 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -37,6 +37,18 @@ global: - loc_as_set_name; - loc_as_unref; - -+ # AS List -+ loc_as_list_append; -+ loc_as_list_clear; -+ loc_as_list_contains; -+ loc_as_list_contains_number; -+ loc_as_list_empty; -+ loc_as_list_get; -+ loc_as_list_new; -+ loc_as_list_ref; -+ loc_as_list_size; -+ loc_as_list_unref; ++ } else { ++ ERROR(network->ctx, "We should never get here\n"); ++ r = 1; ++ goto ERROR; ++ } + - # Country - loc_country_cmp; - loc_country_code_is_valid; -@@ -78,13 +90,14 @@ global: - loc_database_verify; - - # Database Enumerator -+ loc_database_enumerator_get_asns; - loc_database_enumerator_get_countries; - loc_database_enumerator_new; - loc_database_enumerator_next_as; - loc_database_enumerator_next_country; - loc_database_enumerator_next_network; - loc_database_enumerator_ref; -- loc_database_enumerator_set_asn; -+ loc_database_enumerator_set_asns; - loc_database_enumerator_set_countries; - loc_database_enumerator_set_family; - loc_database_enumerator_set_flag; -diff --git a/src/loc/as-list.h b/src/loc/as-list.h -new file mode 100644 -index 0000000..7b5c4e8 ---- /dev/null -+++ b/src/loc/as-list.h -@@ -0,0 +1,41 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet ++ERROR: ++ if (subnet1) ++ loc_network_unref(subnet1); + -+ Copyright (C) 2017 IPFire Development Team info@ipfire.org ++ if (subnet2) ++ loc_network_unref(subnet2); + -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. ++ return r; ++} + -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ ++static int __loc_network_exclude_to_list(struct loc_network* self, ++ struct loc_network* other, struct loc_network_list* list) { ++ // Other must be a subnet of self ++ if (!loc_network_is_subnet(self, other)) { ++ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); ++ ++ // Exit silently ++ return 0; ++ } ++ ++ // We cannot perform this operation if both networks equal ++ if (loc_network_cmp(self, other) == 0) { ++ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); ++ ++ // Exit silently ++ return 0; ++ } + -+#ifndef LIBLOC_AS_LIST_H -+#define LIBLOC_AS_LIST_H ++ return __loc_network_exclude(self, other, list); ++} + -+#include <loc/as.h> -+#include <loc/libloc.h> ++LOC_EXPORT struct loc_network_list* loc_network_exclude( ++ struct loc_network* self, struct loc_network* other) { ++ struct loc_network_list* list; + -+struct loc_as_list; ++#ifdef ENABLE_DEBUG ++ char* n1 = loc_network_str(self); ++ char* n2 = loc_network_str(other); + -+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); -+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); -+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); ++ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); + -+size_t loc_as_list_size(struct loc_as_list* list); -+int loc_as_list_empty(struct loc_as_list* list); -+void loc_as_list_clear(struct loc_as_list* list); ++ free(n1); ++ free(n2); ++#endif + -+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); -+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); ++ // Create a new list with the result ++ int r = loc_network_list_new(self->ctx, &list); ++ if (r) { ++ ERROR(self->ctx, "Could not create network list: %d\n", r); + -+int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as); -+int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number); ++ return NULL; ++ } + -+#endif -diff --git a/src/loc/database.h b/src/loc/database.h -index 246e5c5..70801f0 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato - struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); - int loc_database_enumerator_set_countries( - struct loc_database_enumerator* enumerator, struct loc_country_list* countries); --int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); -+struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator); -+int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); - int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); - int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); - int loc_database_enumerator_next_as( -diff --git a/src/python/database.c b/src/python/database.c -index e6f6f37..38a804c 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -17,6 +17,8 @@ - #include <Python.h> - - #include <loc/libloc.h> -+#include <loc/as.h> -+#include <loc/as-list.h> - #include <loc/database.h> - - #include "locationmodule.h" -@@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) { - } - - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -- char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL }; -+ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; - PyObject* country_codes = NULL; -- unsigned int asn = 0; -+ PyObject* asn_list = NULL; - int flags = 0; - int family = 0; - int flatten = 0; - -- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist, -- &PyList_Type, &country_codes, &asn, &flags, &family, &flatten)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, -+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) - return NULL; - - struct loc_database_enumerator* enumerator; -@@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - } - - // Set the ASN we are searching for -- if (asn) { -- r = loc_database_enumerator_set_asn(enumerator, asn); -+ if (asn_list) { -+ struct loc_as_list* asns; -+ r = loc_as_list_new(loc_ctx, &asns); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); -+ return NULL; -+ } ++ r = __loc_network_exclude_to_list(self, other, list); ++ if (r) { ++ loc_network_list_unref(list); + -+ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) { -+ PyObject* item = PyList_GetItem(asn_list, i); ++ return NULL; ++ } + -+ if (!PyLong_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); ++ // Return the result ++ return list; ++} + -+ loc_as_list_unref(asns); -+ return NULL; -+ } ++LOC_EXPORT struct loc_network_list* loc_network_exclude_list( ++ struct loc_network* network, struct loc_network_list* list) { ++ struct loc_network_list* to_check; + -+ unsigned long number = PyLong_AsLong(item); - -+ struct loc_as* as; -+ r = loc_as_new(loc_ctx, &as, number); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS"); ++ // Create a new list with all networks to look at ++ int r = loc_network_list_new(network->ctx, &to_check); ++ if (r) ++ return NULL; + -+ loc_as_list_unref(asns); -+ loc_as_unref(as); -+ return NULL; -+ } ++ struct loc_network* subnet = NULL; ++ struct loc_network_list* subnets = NULL; + -+ r = loc_as_list_append(asns, as); ++ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Find all excluded networks ++ if (!loc_network_list_contains(to_check, subnet)) { ++ r = __loc_network_exclude_to_list(network, subnet, to_check); + if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); ++ loc_network_list_unref(to_check); ++ loc_network_unref(subnet); + -+ loc_as_list_unref(asns); -+ loc_as_unref(as); + return NULL; + } -+ -+ loc_as_unref(as); + } + -+ r = loc_database_enumerator_set_asns(enumerator, asns); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_as_list_unref(asns); - return NULL; - } -+ -+ loc_as_list_unref(asns); - } - - // Set the flags we are searching for -diff --git a/src/python/export.py b/src/python/export.py -index 739742f..f675eb3 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -197,7 +197,7 @@ class Exporter(object): - - # Get all networks that match the family - networks = self.db.search_networks(family=family, -- country_codes=country_codes, flags=flags, flatten=True) -+ country_codes=country_codes, asns=asns, flags=flags, flatten=True) - - # Walk through all networks - for network in networks: --- -2.20.1 - -From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 16:56:43 +0000 -Subject: [PATCH 060/111] database: Simplify network matching code - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 51cb5cd..1a354f6 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - --static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) { -- const char* country_code = loc_network_get_country_code(network); -- -- return loc_country_list_contains_code(countries, country_code); --} -- - static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { - uint32_t asn = loc_network_get_asn(network); - -@@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network( - return 1; - - // Skip if the country code does not match -- if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries)) -- return 1; -+ if (enumerator->countries) { -+ if (!loc_country_list_empty(enumerator->countries)) { -+ const char* country_code = loc_network_get_country_code(network); -+ -+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) -+ return 1; -+ } ++ // Cleanup ++ loc_network_unref(subnet); + } - - // Skip if the ASN does not match - if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) --- -2.20.1 - -From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 16:58:55 +0000 -Subject: [PATCH 061/111] database: Simplify AS matching code - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 1a354f6..be93e00 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - --static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) { -- uint32_t asn = loc_network_get_asn(network); -- -- return loc_as_list_contains_number(asns, asn); --} -- - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -@@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network( - } - - // Skip if the ASN does not match -- if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns)) -- return 1; -+ if (enumerator->asns) { -+ if (!loc_as_list_empty(enumerator->asns)) { -+ uint32_t asn = loc_network_get_asn(network); + -+ if (!loc_as_list_contains_number(enumerator->asns, asn)) -+ return 1; -+ } ++ r = loc_network_list_new(network->ctx, &subnets); ++ if (r) { ++ loc_network_list_unref(to_check); ++ return NULL; + } - - // Skip if flags do not match - if (enumerator->flags && --- -2.20.1 - -From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 17:50:17 +0000 -Subject: [PATCH 062/111] countries: Make list grow dynamically - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/country-list.c | 38 ++++++++++++++++++++++++++------------ - 1 file changed, 26 insertions(+), 12 deletions(-) - -diff --git a/src/country-list.c b/src/country-list.c -index ae0d71a..1ce2d06 100644 ---- a/src/country-list.c -+++ b/src/country-list.c -@@ -25,11 +25,27 @@ struct loc_country_list { - struct loc_ctx* ctx; - int refcount; - -- struct loc_country* list[1024]; -+ struct loc_country** elements; -+ size_t elements_size; + - size_t size; -- size_t max_size; - }; - -+static int loc_country_list_grow(struct loc_country_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); ++ off_t smallest_subnet = 0; + -+ struct loc_country** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; ++ while (!loc_network_list_empty(to_check)) { ++ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check); + -+ list->elements = elements; -+ list->elements_size += size; ++ // Check whether the subnet to check is part of the input list ++ if (loc_network_list_contains(list, subnet_to_check)) { ++ loc_network_unref(subnet_to_check); ++ continue; ++ } + -+ return 0; -+} ++ // Marks whether this subnet passed all checks ++ int passed = 1; + - LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, - struct loc_country_list** list) { - struct loc_country_list* l = calloc(1, sizeof(*l)); -@@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, - l->ctx = loc_ref(ctx); - l->refcount = 1; - -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- - DEBUG(l->ctx, "Country list allocated at %p\n", l); - *list = l; - -@@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { - - LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { - for (unsigned int i = 0; i < list->size; i++) -- loc_country_unref(list->list[i]); -+ loc_country_unref(list->elements[i]); - } - - LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { -@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis - if (index >= list->size) - return NULL; - -- return loc_country_ref(list->list[index]); -+ return loc_country_ref(list->elements[index]); - } - - LOC_EXPORT int loc_country_list_append( -@@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append( - return 0; - - // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list); -- return -ENOMEM; -+ if (list->size >= list->elements_size) { -+ int r = loc_country_list_grow(list, 64); -+ if (r) -+ return r; - } - - DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); - -- list->list[list->size++] = loc_country_ref(country); -+ list->elements[list->size++] = loc_country_ref(country); - - return 0; - } -@@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append( - LOC_EXPORT int loc_country_list_contains( - struct loc_country_list* list, struct loc_country* country) { - for (unsigned int i = 0; i < list->size; i++) { -- if (loc_country_cmp(country, list->list[i]) == 0) -+ if (loc_country_cmp(country, list->elements[i]) == 0) - return 1; - } - --- -2.20.1 - -From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 17:55:51 +0000 -Subject: [PATCH 063/111] networks: Make list grow dynamically - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network-list.c | 60 ++++++++++++++++++++++++++++------------------ - 1 file changed, 37 insertions(+), 23 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index 1f6e80e..4912c02 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -25,11 +25,27 @@ struct loc_network_list { - struct loc_ctx* ctx; - int refcount; - -- struct loc_network* list[1024]; -+ struct loc_network** elements; -+ size_t elements_size; ++ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) { ++ subnet = loc_network_list_get(list, i); ++ ++ // Drop this subnet if is a subnet of another subnet ++ if (loc_network_is_subnet(subnet, subnet_to_check)) { ++ passed = 0; ++ loc_network_unref(subnet); ++ break; ++ } + - size_t size; -- size_t max_size; - }; - -+static int loc_network_list_grow(struct loc_network_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); ++ // Break it down if it overlaps ++ if (loc_network_overlaps(subnet, subnet_to_check)) { ++ passed = 0; + -+ struct loc_network** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; ++ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); + -+ list->elements = elements; -+ list->elements_size += size; ++ loc_network_unref(subnet); ++ break; ++ } + -+ return 0; -+} ++ // If the subnet is strictly greater, we do not need to continue the search ++ r = loc_network_cmp(subnet, subnet_to_check); ++ if (r > 0) { ++ loc_network_unref(subnet); ++ break; + - LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, - struct loc_network_list** list) { - struct loc_network_list* l = calloc(1, sizeof(*l)); -@@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, - l->ctx = loc_ref(ctx); - l->refcount = 1; - -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- - DEBUG(l->ctx, "Network list allocated at %p\n", l); - *list = l; - return 0; -@@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) { - DEBUG(list->ctx, "Releasing network list at %p\n", list); - - for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -+ loc_network_unref(list->elements[i]); - - loc_unref(list->ctx); - free(list); -@@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { - - LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { - for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->list[i]); -+ loc_network_unref(list->elements[i]); - - list->size = 0; - } -@@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { - char* s; - - for (unsigned int i = 0; i < list->size; i++) { -- network = list->list[i]; -+ network = list->elements[i]; - - s = loc_network_str(network); - -@@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - if (index >= list->size) - return NULL; - -- return loc_network_ref(list->list[index]); -+ return loc_network_ref(list->elements[index]); - } - - LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -@@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - return 0; - - // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list); -- return -ENOMEM; -+ if (list->size >= list->elements_size) { -+ int r = loc_network_list_grow(list, 64); -+ if (r) -+ return r; - } - - DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); - -- list->list[list->size++] = loc_network_ref(network); -+ list->elements[list->size++] = loc_network_ref(network); - - return 0; - } -@@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis - return NULL; - } - -- struct loc_network* network = list->list[--list->size]; -+ struct loc_network* network = list->elements[--list->size]; - - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); - -@@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis - return NULL; - } - -- struct loc_network* network = list->list[0]; -+ struct loc_network* network = list->elements[0]; - - // Move all elements to the top of the stack - for (unsigned int i = 0; i < --list->size; i++) { -- list->list[i] = list->list[i+1]; -+ list->elements[i] = list->elements[i+1]; - } - - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -@@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis - - LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { - for (unsigned int i = 0; i < list->size; i++) { -- if (loc_network_eq(list->list[i], network)) -+ if (loc_network_eq(list->elements[i], network)) - return 1; - } - -@@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1 - if (i1 >= list->size || i2 >= list->size) - return; - -- struct loc_network* network1 = list->list[i1]; -- struct loc_network* network2 = list->list[i2]; -+ struct loc_network* network1 = list->elements[i1]; -+ struct loc_network* network2 = list->elements[i2]; - -- list->list[i1] = network2; -- list->list[i2] = network1; -+ list->elements[i1] = network2; -+ list->elements[i2] = network1; - } - - LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -@@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { - swapped = 0; - - for (unsigned int i = 1; i < n; i++) { -- if (loc_network_gt(list->list[i-1], list->list[i]) > 0) { -+ if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) { - loc_network_list_swap(list, i-1, i); - swapped = 1; - } -@@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge( - int r; - - for (unsigned int i = 0; i < other->size; i++) { -- r = loc_network_list_push(self, other->list[i]); -+ r = loc_network_list_push(self, other->elements[i]); - if (r) - return r; - } --- -2.20.1 - -From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 17:57:55 +0000 -Subject: [PATCH 064/111] as: Make lists grow dynamically - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/as-list.c | 38 ++++++++++++++++++++++++++------------ - 1 file changed, 26 insertions(+), 12 deletions(-) - -diff --git a/src/as-list.c b/src/as-list.c -index 7c69eb0..17de23e 100644 ---- a/src/as-list.c -+++ b/src/as-list.c -@@ -25,11 +25,27 @@ struct loc_as_list { - struct loc_ctx* ctx; - int refcount; - -- struct loc_as* list[1024]; -+ struct loc_as** elements; -+ size_t elements_size; ++ // If it is strictly smaller, we can continue the search from here next ++ // time because all networks that are to be checked can only be larger ++ // than this one. ++ } else if (r < 0) { ++ smallest_subnet = i; ++ } + - size_t size; -- size_t max_size; - }; - -+static int loc_as_list_grow(struct loc_as_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); ++ loc_network_unref(subnet); ++ } + -+ struct loc_as** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; ++ if (passed) { ++ r = loc_network_list_push(subnets, subnet_to_check); ++ } + -+ list->elements = elements; -+ list->elements_size += size; ++ loc_network_unref(subnet_to_check); ++ } + -+ return 0; ++ loc_network_list_unref(to_check); ++ ++ return subnets; +} + - LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, - struct loc_as_list** list) { - struct loc_as_list* l = calloc(1, sizeof(*l)); -@@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, - l->ctx = loc_ref(ctx); - l->refcount = 1; - -- // Do not allow this list to grow larger than this -- l->max_size = 1024; -- - DEBUG(l->ctx, "AS list allocated at %p\n", l); - *list = l; - -@@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { - - LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { - for (unsigned int i = 0; i < list->size; i++) -- loc_as_unref(list->list[i]); -+ loc_as_unref(list->elements[i]); - } - - LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { -@@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index - if (index >= list->size) - return NULL; - -- return loc_as_ref(list->list[index]); -+ return loc_as_ref(list->elements[index]); - } - - LOC_EXPORT int loc_as_list_append( -@@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append( - return 0; - - // Check if we have space left -- if (list->size == list->max_size) { -- ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list); -- return -ENOMEM; -+ if (list->size >= list->elements_size) { -+ int r = loc_as_list_grow(list, 64); -+ if (r) -+ return r; - } - - DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); - -- list->list[list->size++] = loc_as_ref(as); -+ list->elements[list->size++] = loc_as_ref(as); + LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { + // Add country code + loc_country_code_copy(dbobj->country_code, network->country_code); +@@ -474,7 +813,7 @@ struct loc_network_tree_node { + struct loc_network* network; + };
+-LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { ++int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { + struct loc_network_tree* t = calloc(1, sizeof(*t)); + if (!t) + return -ENOMEM; +@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree return 0; } -@@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append( - LOC_EXPORT int loc_as_list_contains( - struct loc_as_list* list, struct loc_as* as) { - for (unsigned int i = 0; i < list->size; i++) { -- if (loc_as_cmp(as, list->list[i]) == 0) -+ if (loc_as_cmp(as, list->elements[i]) == 0) - return 1; - } - --- -2.20.1 - -From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 18:13:49 +0000 -Subject: [PATCH 065/111] export: Change back to use Network objects - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index f675eb3..67e437f 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter): - def _write_network(self, network): - for address in (network.first_address, network.last_address): - # Convert this into a string of bits -- bytes = socket.inet_pton( -- socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address, -- ) -+ bytes = socket.inet_pton(network.family, address) - - self.f.write(bytes) - --- -2.20.1 - -From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 18:14:15 +0000 -Subject: [PATCH 066/111] Actually clear all lists - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/as-list.c | 8 ++++++++ - src/country-list.c | 8 ++++++++ - src/network-list.c | 6 ++++++ - 3 files changed, 22 insertions(+) - -diff --git a/src/as-list.c b/src/as-list.c -index 17de23e..76620c7 100644 ---- a/src/as-list.c -+++ b/src/as-list.c -@@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { - } - - LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { -+ if (!list->elements) -+ return; -+ - for (unsigned int i = 0; i < list->size; i++) - loc_as_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements_size = 0; -+ -+ list->size = 0; - }
- LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { -diff --git a/src/country-list.c b/src/country-list.c -index 1ce2d06..1c49c47 100644 ---- a/src/country-list.c -+++ b/src/country-list.c -@@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { ++struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { + return loc_network_tree_node_ref(tree->root); }
- LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { -+ if (!list->elements) -+ return; -+ - for (unsigned int i = 0; i < list->size; i++) - loc_country_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements_size = 0; -+ -+ list->size = 0; +@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_ + return 0; }
- LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { -diff --git a/src/network-list.c b/src/network-list.c -index 4912c02..9cb4547 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { +-LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, ++int loc_network_tree_walk(struct loc_network_tree* tree, + int(*filter_callback)(struct loc_network* network, void* data), + int(*callback)(struct loc_network* network, void* data), void* data) { + return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); +@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) { + free(tree); }
- LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ if (!list->elements) -+ return; -+ - for (unsigned int i = 0; i < list->size; i++) - loc_network_unref(list->elements[i]); +-LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { ++struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { + if (--tree->refcount > 0) + return tree;
-+ free(list->elements); -+ list->elements_size = 0; -+ - list->size = 0; +@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) { + return 0; }
--- -2.20.1 - -From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 18:34:51 +0000 -Subject: [PATCH 067/111] database: Do not clean up python list - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/database.c | 9 +++++++-- - 1 file changed, 7 insertions(+), 2 deletions(-) - -diff --git a/src/python/database.c b/src/python/database.c -index 38a804c..ed22275 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - loc_country_unref(country); - } - -- loc_database_enumerator_set_countries(enumerator, countries); -+ r = loc_database_enumerator_set_countries(enumerator, countries); -+ if (r) { -+ PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_as_list_unref(countries); -+ return NULL; -+ } - -- Py_DECREF(country_codes); - loc_country_list_unref(countries); - } - --- -2.20.1 - -From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 17 Nov 2020 19:01:04 +0000 -Subject: [PATCH 068/111] database: Free filter lists in enumerator - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/src/database.c b/src/database.c -index be93e00..ca35fe1 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera - if (enumerator->string) - free(enumerator->string); - -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); -+ -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); -+ - // Free network search - free(enumerator->networks_visited); - --- -2.20.1 - -From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 18 Nov 2020 13:18:52 +0000 -Subject: [PATCH 069/111] database: Add debug output to filtering - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 29 ++++++++++++++++------------- - 1 file changed, 16 insertions(+), 13 deletions(-) - -diff --git a/src/database.c b/src/database.c -index ca35fe1..83dd752 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node( - static int loc_database_enumerator_filter_network( - struct loc_database_enumerator* enumerator, struct loc_network* network) { - // Skip if the family does not match -- if (enumerator->family && loc_network_address_family(network) != enumerator->family) -+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); - return 1; -+ } - - // Skip if the country code does not match -- if (enumerator->countries) { -- if (!loc_country_list_empty(enumerator->countries)) { -- const char* country_code = loc_network_get_country_code(network); -+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { -+ const char* country_code = loc_network_get_country_code(network); - -- if (!loc_country_list_contains_code(enumerator->countries, country_code)) -- return 1; -+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); -+ return 1; - } - } - - // Skip if the ASN does not match -- if (enumerator->asns) { -- if (!loc_as_list_empty(enumerator->asns)) { -- uint32_t asn = loc_network_get_asn(network); -+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { -+ uint32_t asn = loc_network_get_asn(network); - -- if (!loc_as_list_contains_number(enumerator->asns, asn)) -- return 1; -+ if (!loc_as_list_contains_number(enumerator->asns, asn)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); -+ return 1; - } - } +-LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { ++int loc_network_tree_dump(struct loc_network_tree* tree) { + DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
- // Skip if flags do not match -- if (enumerator->flags && -- !loc_network_match_flag(network, enumerator->flags)) -+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); - return 1; -+ } + return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); + } + +-LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { ++int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { + DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
- // Do not filter + struct loc_network_tree_node* node = loc_network_tree_get_path(tree, +@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) { return 0; --- -2.20.1 - -From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 18 Nov 2020 13:19:04 +0000 -Subject: [PATCH 070/111] export: Remove filtering for flags - -The filter is an AND filter and if we set the flags from -the special country codes, we won't get back much. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 8 +------- - 1 file changed, 1 insertion(+), 7 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 67e437f..4219957 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -187,15 +187,9 @@ class Exporter(object): - country_code for country_code in countries if not country_code in FLAGS.values() - ] - -- # Collect flags -- flags = 0 -- for flag in FLAGS: -- if FLAGS[flag] in countries: -- flags |= flag -- - # Get all networks that match the family - networks = self.db.search_networks(family=family, -- country_codes=country_codes, asns=asns, flags=flags, flatten=True) -+ country_codes=country_codes, asns=asns, flatten=True) + }
- # Walk through all networks - for network in networks: --- -2.20.1 - -From 627bf1daaae1510cfd4016297ed16b82df209aae Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 18 Nov 2020 13:33:45 +0000 -Subject: [PATCH 071/111] python: Remove unnecessary db object from writers - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 12 ++++++------ - 1 file changed, 6 insertions(+), 6 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 4219957..5bc9f30 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -39,8 +39,8 @@ class OutputWriter(object): - suffix = "networks" - mode = "w" +-LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { + size_t counter = 0;
-- def __init__(self, db, f, prefix=None, flatten=True): -- self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten -+ def __init__(self, f, prefix=None, flatten=True): -+ self.f, self.prefix, self.flatten = f, prefix, flatten + int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); +@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) + return counter; + }
- # The previously written network - self._last_network = None -@@ -49,13 +49,13 @@ class OutputWriter(object): - self._write_header() +-LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { ++size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { + return __loc_network_tree_count_nodes(tree->root); + }
- @classmethod -- def open(cls, db, filename, **kwargs): -+ def open(cls, filename, **kwargs): - """ - Convenience function to open a file - """ - f = open(filename, cls.mode) +-LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { ++int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { + struct loc_network_tree_node* n = calloc(1, sizeof(*n)); + if (!n) + return -ENOMEM; +@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network + return 0; + }
-- return cls(db, f, **kwargs) -+ return cls(f, **kwargs) +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { + if (node) + node->refcount++;
- def __repr__(self): - return "<%s f=%s>" % (self.__class__.__name__, self.f) -@@ -172,7 +172,7 @@ class Exporter(object): - directory, prefix=country_code, suffix=self.writer.suffix, family=family, - ) +@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { + free(node); + }
-- writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code) -+ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { ++struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { + if (!node) + return NULL;
- # Create writers for ASNs - for asn in asns: -@@ -180,7 +180,7 @@ class Exporter(object): - directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, - ) +@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_ + return NULL; + }
-- writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn) -+ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) - - # Filter countries from special country codes - country_codes = [ --- -2.20.1 - -From 9cb56ac9adafafa6e452009c2fa2d42e94474e11 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 18 Nov 2020 13:34:50 +0000 -Subject: [PATCH 072/111] location: End lookup after an invalid IP address was - passed - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/location.in | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/python/location.in b/src/python/location.in -index 0d09210..6885ea0 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -253,6 +253,7 @@ class CLI(object): - network = db.lookup(address) - except ValueError: - print(_("Invalid IP address: %s") % address, file=sys.stderr) -+ return 2 +-LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { ++struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { + if (index == 0) + node = node->zero; + else +@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne + return loc_network_tree_node_ref(node); + }
- args = { - "address" : address, --- -2.20.1 - -From 2a550d12208f8bc8002e05ac08613312df26b20e Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 19 Nov 2020 12:03:33 +0000 -Subject: [PATCH 073/111] python: Fix download of database - -This was all messed up in 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 -when the change of type for the timestamp wasn't changed everywhere -else. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/downloader.py | 6 +++--- - src/python/location.in | 2 +- - 2 files changed, 4 insertions(+), 4 deletions(-) - -diff --git a/src/python/downloader.py b/src/python/downloader.py -index 87bbb68..05f7872 100644 ---- a/src/python/downloader.py -+++ b/src/python/downloader.py -@@ -119,8 +119,8 @@ class Downloader(object): +-LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { ++int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { + return (!!node->network); + }
- headers = {} - if timestamp: -- headers["If-Modified-Since"] = timestamp.strftime( -- "%a, %d %b %Y %H:%M:%S GMT", -+ headers["If-Modified-Since"] = time.strftime( -+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp), - ) +-LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { ++struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { + return loc_network_ref(node->network); + } +diff --git a/src/perl/Location.xs b/src/perl/Location.xs +index dcf3f0d..b7676d2 100644 +--- a/src/perl/Location.xs ++++ b/src/perl/Location.xs +@@ -125,7 +125,7 @@ database_countries(db) + PPCODE: + // Create Database enumerator + struct loc_database_enumerator* enumerator; +- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES); ++ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
- t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) -@@ -195,7 +195,7 @@ class Downloader(object): - db = Database(f.name) + if (err) { + croak("Could not create a database enumerator\n"); +diff --git a/src/python/database.c b/src/python/database.c +index 1013a58..f385c61 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -17,6 +17,8 @@ + #include <Python.h>
- # Database is not recent -- if timestamp and db.created_at < timestamp.timestamp(): -+ if timestamp and db.created_at < timestamp: - return False + #include <loc/libloc.h> ++#include <loc/as.h> ++#include <loc/as-list.h> + #include <loc/database.h>
- log.info("Downloaded new database from %s" % (time.strftime( -diff --git a/src/python/location.in b/src/python/location.in -index 6885ea0..b30beae 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -433,7 +433,7 @@ class CLI(object): + #include "locationmodule.h" +@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database + return (PyObject*)self; + }
- # Try downloading a new database - try: -- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) -+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir) +-static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) { ++static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) { + struct loc_database_enumerator* enumerator;
- # If no file could be downloaded, log a message - except FileNotFoundError as e: --- -2.20.1 - -From a1a00053300cff3c0f690d52377c76c83c2a08b2 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 19 Nov 2020 12:34:11 +0000 -Subject: [PATCH 074/111] python: Add property to return IP addresses as bytes - -This avoids calling inet_pton to parse IP addresses from string - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 2 ++ - src/loc/network.h | 2 ++ - src/network.c | 8 ++++++++ - src/python/database.c | 2 +- - src/python/export.py | 7 ++----- - src/python/network.c | 40 ++++++++++++++++++++++++++++++++++++++++ - 6 files changed, 55 insertions(+), 6 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 53273cd..406dd15 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -113,6 +113,8 @@ global: - loc_network_format_last_address; - loc_network_get_asn; - loc_network_get_country_code; -+ loc_network_get_first_address; -+ loc_network_get_last_address; - loc_network_gt; - loc_network_has_flag; - loc_network_is_subnet; -diff --git a/src/loc/network.h b/src/loc/network.h -index d86b685..4b7410c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -39,7 +39,9 @@ struct loc_network* loc_network_unref(struct loc_network* network); - char* loc_network_str(struct loc_network* network); - int loc_network_address_family(struct loc_network* network); +- int r = loc_database_enumerator_new(&enumerator, self->db, what); ++ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en + }
-+const struct in6_addr* loc_network_get_first_address(struct loc_network* network); - char* loc_network_format_first_address(struct loc_network* network); -+const struct in6_addr* loc_network_get_last_address(struct loc_network* network); - char* loc_network_format_last_address(struct loc_network* network); - int loc_network_match_address(struct loc_network* network, const struct in6_addr* address); + static PyObject* Database_ases(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); + }
-diff --git a/src/network.c b/src/network.c -index 28ca2df..4c8787a 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -343,10 +343,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc - return string; + static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { +@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { + + struct loc_database_enumerator* enumerator; + +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; +@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { }
-+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) { -+ return &network->first_address; + static PyObject* Database_networks(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); +} + - LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) { - return loc_network_format_address(network, &network->first_address); - } - -+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) { -+ return &network->last_address; -+} ++static PyObject* Database_networks_flattened(DatabaseObject *self) { ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN); + } + + static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { +- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL }; +- const char* country_code = NULL; +- unsigned int asn = 0; ++ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; ++ PyObject* country_codes = NULL; ++ PyObject* asn_list = NULL; + int flags = 0; + int family = 0; ++ int flatten = 0; + +- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family)) ++ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, ++ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) + return NULL; + + struct loc_database_enumerator* enumerator; +- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS); ++ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, ++ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); + return NULL; + } + + // Set country code we are searching for +- if (country_code) { +- r = loc_database_enumerator_set_country_code(enumerator, country_code); ++ if (country_codes) { ++ struct loc_country_list* countries; ++ r = loc_country_list_new(loc_ctx, &countries); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create country list"); ++ return NULL; ++ } ++ ++ for (unsigned int i = 0; i < PyList_Size(country_codes); i++) { ++ PyObject* item = PyList_GetItem(country_codes, i); ++ ++ if (!PyUnicode_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ const char* country_code = PyUnicode_AsUTF8(item); ++ ++ struct loc_country* country; ++ r = loc_country_new(loc_ctx, &country, country_code); ++ if (r) { ++ if (r == -EINVAL) { ++ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); ++ } else { ++ PyErr_SetString(PyExc_SystemError, "Could not create country"); ++ } ++ ++ loc_country_list_unref(countries); ++ return NULL; ++ } ++ ++ // Append it to the list ++ r = loc_country_list_append(countries, country); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); + - LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { - return loc_network_format_address(network, &network->last_address); - } -diff --git a/src/python/database.c b/src/python/database.c -index ed22275..f385c61 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -329,7 +329,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, ++ loc_country_list_unref(countries); ++ loc_country_unref(country); ++ return NULL; ++ } ++ ++ loc_country_unref(country); ++ } + ++ r = loc_database_enumerator_set_countries(enumerator, countries); if (r) { PyErr_SetFromErrno(PyExc_SystemError); - -- loc_as_list_unref(countries); ++ + loc_country_list_unref(countries); return NULL; } ++ ++ loc_country_list_unref(countries); + }
-diff --git a/src/python/export.py b/src/python/export.py -index 5bc9f30..6b39878 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -142,11 +142,8 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" - - def _write_network(self, network): -- for address in (network.first_address, network.last_address): -- # Convert this into a string of bits -- bytes = socket.inet_pton(network.family, address) -- -- self.f.write(bytes) -+ for address in (network._first_address, network._last_address): -+ self.f.write(address) - - - formats = { -diff --git a/src/python/network.c b/src/python/network.c -index ed91d65..742b472 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -215,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) { - return obj; - } + // Set the ASN we are searching for +- if (asn) { +- r = loc_database_enumerator_set_asn(enumerator, asn); ++ if (asn_list) { ++ struct loc_as_list* asns; ++ r = loc_as_list_new(loc_ctx, &asns); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); ++ return NULL; ++ } ++ ++ for (unsigned int i = 0; i < PyList_Size(asn_list); i++) { ++ PyObject* item = PyList_GetItem(asn_list, i); ++ ++ if (!PyLong_Check(item)) { ++ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
-+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) { -+ struct in_addr address4; ++ loc_as_list_unref(asns); ++ return NULL; ++ } + -+ // Convert IPv4 addresses to struct in_addr -+ if (IN6_IS_ADDR_V4MAPPED(address6)) { -+ address4.s_addr = address6->s6_addr32[3]; ++ unsigned long number = PyLong_AsLong(item); + -+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4)); -+ } ++ struct loc_as* as; ++ r = loc_as_new(loc_ctx, &as, number); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not create AS"); + -+ // Return IPv6 addresses as they are -+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6)); -+} ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } + -+static PyObject* Network_get__first_address(NetworkObject* self) { -+ const struct in6_addr* address = loc_network_get_first_address(self->network); ++ r = loc_as_list_append(asns, as); ++ if (r) { ++ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); + -+ return PyBytes_FromAddress(address); -+} ++ loc_as_list_unref(asns); ++ loc_as_unref(as); ++ return NULL; ++ } + - static PyObject* Network_get_last_address(NetworkObject* self) { - char* address = loc_network_format_last_address(self->network); ++ loc_as_unref(as); ++ } ++ ++ r = loc_database_enumerator_set_asns(enumerator, asns); + if (r) { + PyErr_SetFromErrno(PyExc_SystemError); ++ ++ loc_as_list_unref(asns); + return NULL; + } ++ ++ loc_as_list_unref(asns); + }
-@@ -224,6 +244,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) { - return obj; + // Set the flags we are searching for +@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, }
-+static PyObject* Network_get__last_address(NetworkObject* self) { -+ const struct in6_addr* address = loc_network_get_last_address(self->network); -+ -+ return PyBytes_FromAddress(address); -+} -+ - static struct PyMethodDef Network_methods[] = { - { - "exclude", -@@ -281,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = { + static PyObject* Database_countries(DatabaseObject* self) { +- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); + } + + static struct PyMethodDef Database_methods[] = { +@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = { NULL, NULL, }, + { -+ "_first_address", -+ (getter)Network_get__first_address, ++ "networks_flattened", ++ (getter)Database_networks_flattened, + NULL, + NULL, + NULL, + }, { - "last_address", - (getter)Network_get_last_address, -@@ -288,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "_last_address", -+ (getter)Network_get__last_address, -+ NULL, -+ NULL, -+ NULL, -+ }, - { NULL }, - }; + "vendor", + (getter)Database_get_vendor, +diff --git a/src/python/downloader.py b/src/python/downloader.py +index 87bbb68..05f7872 100644 +--- a/src/python/downloader.py ++++ b/src/python/downloader.py +@@ -119,8 +119,8 @@ class Downloader(object): + + headers = {} + if timestamp: +- headers["If-Modified-Since"] = timestamp.strftime( +- "%a, %d %b %Y %H:%M:%S GMT", ++ headers["If-Modified-Since"] = time.strftime( ++ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp), + ) + + t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) +@@ -195,7 +195,7 @@ class Downloader(object): + db = Database(f.name) + + # Database is not recent +- if timestamp and db.created_at < timestamp.timestamp(): ++ if timestamp and db.created_at < timestamp: + return False
--- -2.20.1 - -From 90d2194a876c223f9124ce9e27bdee6a6b49ff6a Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 19 Nov 2020 12:40:01 +0000 -Subject: [PATCH 075/111] export: Remove old flattening feature - -The database enumerator now only returns networks that will -never overlap. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 34 ++++++---------------------------- - 1 file changed, 6 insertions(+), 28 deletions(-) - + log.info("Downloaded new database from %s" % (time.strftime( diff --git a/src/python/export.py b/src/python/export.py -index 6b39878..4702bcf 100644 +index d15c6f0..f0eae26 100644 --- a/src/python/export.py +++ b/src/python/export.py +@@ -29,7 +29,7 @@ import _location + log = logging.getLogger("location.export") + log.propagate = 1 + +-flags = { ++FLAGS = { + _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", + _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", + _location.NETWORK_FLAG_ANYCAST : "A3", @@ -39,11 +39,8 @@ class OutputWriter(object): suffix = "networks" mode = "w" @@ -7996,7 +2524,7 @@ index 6b39878..4702bcf 100644 def _write_header(self): """ The header of the file -@@ -84,15 +69,8 @@ class OutputWriter(object): +@@ -84,16 +69,8 @@ class OutputWriter(object): """ pass
@@ -8005,15 +2533,16 @@ index 6b39878..4702bcf 100644 - def write(self, network): - if self.flatten and self._flatten(network): -- log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network)) +- log.debug("Skipping writing network %s" % network) - return - -- return self._write_network(network) +- # Write the network to file +- self._write_network(network) + self.f.write("%s\n" % network)
def finish(self): """ -@@ -113,7 +91,7 @@ class IpsetOutputWriter(OutputWriter): +@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter): def _write_header(self): self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
@@ -8022,1756 +2551,875 @@ index 6b39878..4702bcf 100644 self.f.write("add %s %s\n" % (self.prefix, network))
-@@ -129,7 +107,7 @@ class NftablesOutputWriter(OutputWriter): +@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter): def _write_footer(self): self.f.write("}\n")
- def _write_network(self, network): + def write(self, network): - self.f.write(" %s,\n" % network) - - -@@ -141,7 +119,7 @@ class XTGeoIPOutputWriter(OutputWriter): - suffix = "iv" - mode = "wb" - -- def _write_network(self, network): -+ def write(self, network): - for address in (network._first_address, network._last_address): - self.f.write(address) - --- -2.20.1 - -From 90188dad86223380b0854b523b63ec024117c5f2 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 19 Nov 2020 12:41:19 +0000 -Subject: [PATCH 076/111] export: Speed-up export in xt_geoip format - -Removing the loop avoids creating a tuple and then iterating over it - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/python/export.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/python/export.py b/src/python/export.py -index 4702bcf..f0eae26 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -120,8 +120,8 @@ class XTGeoIPOutputWriter(OutputWriter): - mode = "wb" - - def write(self, network): -- for address in (network._first_address, network._last_address): -- self.f.write(address) -+ self.f.write(network._first_address) -+ self.f.write(network._last_address) - - - formats = { --- -2.20.1 - -From 6661692f3bc8f788af8b75ae25561f4cc4a2acb5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Thu, 19 Nov 2020 12:48:46 +0000 -Subject: [PATCH 077/111] database: Disable some useless code when not running - in debug mode - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 16 +++++++++++++++- - 1 file changed, 15 insertions(+), 1 deletion(-) - -diff --git a/src/database.c b/src/database.c -index 83dd752..ef4f505 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -619,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { - } - - clock_t end = clock(); -- DEBUG(db->ctx, "Signature checked in %.4fms\n", -+ INFO(db->ctx, "Signature checked in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); - - CLEANUP: -@@ -679,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, - off_t lo = 0; - off_t hi = db->as_count - 1; - -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif - - while (lo <= hi) { - off_t i = (lo + hi) / 2; -@@ -693,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, - // Check if this is a match - uint32_t as_number = loc_as_get_number(*as); - if (as_number == number) { -+#ifdef ENABLE_DEBUG - clock_t end = clock(); - - // Log how fast this has been - DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif - - return 0; - } -@@ -741,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ - return -1; - } - -+#ifdef ENABLE_DEBUG - if (r == 0) { - char* string = loc_network_str(*network); - DEBUG(db->ctx, "Got network %s\n", string); - free(string); - } -+#endif - - return r; - } -@@ -840,17 +846,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db, - - *network = NULL; - -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif + self.f.write(" %s,\n" % network)
- int r = __loc_database_lookup(db, address, network, &network_address, - db->network_nodes_v1, 0);
-+#ifdef ENABLE_DEBUG - clock_t end = clock(); +@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter): + suffix = "iv" + mode = "wb"
- // Log how fast this has been - DEBUG(db->ctx, "Executed network search in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif +- def _write_network(self, network): +- for address in (network.first_address, network.last_address): +- # Convert this into a string of bits +- bytes = socket.inet_pton( +- network.family, address, +- ) +- +- self.f.write(bytes) ++ def write(self, network): ++ self.f.write(network._first_address) ++ self.f.write(network._last_address)
- return r; - } -@@ -897,8 +907,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - off_t lo = 0; - off_t hi = db->countries_count - 1;
-+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif + formats = { +@@ -185,8 +157,14 @@ class Exporter(object):
- while (lo <= hi) { - off_t i = (lo + hi) / 2; -@@ -913,11 +925,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - int result = strcmp(code, cc); + writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
- if (result == 0) { -+#ifdef ENABLE_DEBUG - clock_t end = clock(); ++ # Filter countries from special country codes ++ country_codes = [ ++ country_code for country_code in countries if not country_code in FLAGS.values() ++ ] ++ + # Get all networks that match the family +- networks = self.db.search_networks(family=family) ++ networks = self.db.search_networks(family=family, ++ country_codes=country_codes, asns=asns, flatten=True)
- // Log how fast this has been - DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif + # Walk through all networks + for network in networks: +@@ -203,10 +181,10 @@ class Exporter(object): + pass
- return 0; - } --- -2.20.1 - -From a17c353bce55a9f38f67b4b7d2425194cfa208e7 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 14:43:31 +0000 -Subject: [PATCH 078/111] test: Add tests to network-lists - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - Makefile.am | 10 ++++ - src/.gitignore | 1 + - src/test-network-list.c | 126 ++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 137 insertions(+) - create mode 100644 src/test-network-list.c - -diff --git a/Makefile.am b/Makefile.am -index d0cc793..ebd7e17 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -318,6 +318,7 @@ check_PROGRAMS = \ - src/test-database \ - src/test-as \ - src/test-network \ -+ src/test-network-list \ - src/test-country \ - src/test-signature + # Handle flags +- for flag in flags: ++ for flag in FLAGS: + if network.has_flag(flag): + # Fetch the "fake" country code +- country = flags[flag] ++ country = FLAGS[flag]
-@@ -357,6 +358,15 @@ src_test_network_CFLAGS = \ - src_test_network_LDADD = \ - src/libloc.la + try: + writers[country].write(network) +diff --git a/src/python/importer.py b/src/python/importer.py +index f19db4b..5f46bc3 100644 +--- a/src/python/importer.py ++++ b/src/python/importer.py +@@ -64,7 +64,7 @@ EXTENDED_SOURCES = ( + "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-+src_test_network_list_SOURCES = \ -+ src/test-network-list.c -+ -+src_test_network_list_CFLAGS = \ -+ $(TESTS_CFLAGS) -+ -+src_test_network_list_LDADD = \ -+ src/libloc.la -+ - src_test_stringpool_SOURCES = \ - src/test-stringpool.c + # Latin America and Caribbean Network Information Centre +- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", ++ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-diff --git a/src/.gitignore b/src/.gitignore -index caf80b5..3ccbdb8 100644 ---- a/src/.gitignore -+++ b/src/.gitignore -@@ -10,5 +10,6 @@ test-libloc - test-database - test-country - test-network -+test-network-list - test-signature - test-stringpool -diff --git a/src/test-network-list.c b/src/test-network-list.c -new file mode 100644 -index 0000000..3061d63 ---- /dev/null -+++ b/src/test-network-list.c -@@ -0,0 +1,126 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team info@ipfire.org -+ -+ This program is free software; you can redistribute it and/or modify -+ it under the terms of the GNU General Public License as published by -+ the Free Software Foundation; either version 2 of the License, or -+ (at your option) any later version. -+ -+ This program is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ GNU General Public License for more details. -+*/ -+ -+#include <errno.h> -+#include <stdio.h> -+#include <stddef.h> -+#include <stdlib.h> -+#include <string.h> -+#include <syslog.h> -+ -+#include <loc/libloc.h> -+#include <loc/network.h> -+#include <loc/network-list.h> -+ -+int main(int argc, char** argv) { -+ int err; -+ -+ struct loc_ctx* ctx; -+ err = loc_new(&ctx); -+ if (err < 0) -+ exit(EXIT_FAILURE); -+ -+ // Enable debug logging -+ loc_set_log_priority(ctx, LOG_DEBUG); -+ -+ // Create a network -+ struct loc_network* network1; -+ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32"); -+ if (err) { -+ fprintf(stderr, "Could not create the network1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet1; -+ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet2; -+ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet2\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Make a list with both subnets -+ struct loc_network_list* subnets; -+ err = loc_network_list_new(ctx, &subnets); -+ if (err) { -+ fprintf(stderr, "Could not create subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ size_t size = loc_network_list_size(subnets); -+ if (size > 0) { -+ fprintf(stderr, "The list is not empty: %zu\n", size); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_list_push(subnets, subnet1); -+ if (err) { -+ fprintf(stderr, "Could not add subnet1 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (loc_network_list_empty(subnets)) { -+ fprintf(stderr, "The subnets list reports that it is empty\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_list_push(subnets, subnet2); -+ if (err) { -+ fprintf(stderr, "Could not add subnet2 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ size = loc_network_list_size(subnets); -+ if (size != 2) { -+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Exclude subnet1 from network1 -+ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ // Exclude all subnets from network1 -+ excluded = loc_network_exclude_list(network1, subnets); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ if (excluded) -+ loc_network_list_unref(excluded); -+ -+ loc_network_list_unref(subnets); -+ loc_network_unref(network1); -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ loc_unref(ctx); -+ -+ return EXIT_SUCCESS; -+} --- -2.20.1 - -From 1209ff0cd4be6b2f52a5e82168db8a8ac68e628a Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 16:24:40 +0000 -Subject: [PATCH 079/111] network: Fix loc_network_is_subnet() - -This function always returned false. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 16 +++------------- - 1 file changed, 3 insertions(+), 13 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 4c8787a..b440d76 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -491,12 +491,12 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) - return 0; + # Réseaux IP Européens + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", +diff --git a/src/python/location-importer.in b/src/python/location-importer.in +index 1467923..2dec89e 100644 +--- a/src/python/location-importer.in ++++ b/src/python/location-importer.in +@@ -152,6 +152,7 @@ class CLI(object): + last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); + CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); + CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); ++ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); + + -- autnums + CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); +@@ -165,6 +166,7 @@ class CLI(object): + -- networks + CREATE TABLE IF NOT EXISTS networks(network inet, country text); + CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); ++ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); + CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
- // If the end address of the other network is greater than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0) - return 0; + -- overrides +@@ -188,6 +190,8 @@ class CLI(object): + ); + CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network + ON network_overrides(network); ++ CREATE INDEX IF NOT EXISTS network_overrides_search ++ ON network_overrides USING GIST(network inet_ops); + """)
- return 1; -@@ -504,17 +504,7 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ + return db +@@ -234,32 +238,24 @@ class CLI(object):
- // XXX DEPRECATED - I find this too difficult to use - LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { -- // If the start address of the other network is smaller than this network, -- // it cannot be a subnet. -- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -- return 0; -- -- // If the end address of the other network is greater than this network, -- // it cannot be a subnet. -- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -- return 0; + # Select all known networks + rows = self.db.query(""" +- -- Get a (sorted) list of all known networks +- WITH known_networks AS ( +- SELECT network FROM announcements +- UNION +- SELECT network FROM networks +- ORDER BY network +- ) - -- return 1; -+ return loc_network_is_subnet(other, self); - } - - LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { --- -2.20.1 - -From 92349c14876f8e4711739c82d1a389a53bcb27a5 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 16:25:27 +0000 -Subject: [PATCH 080/111] network: Remove debugging output - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 4 ---- - 1 file changed, 4 deletions(-) - -diff --git a/src/network.c b/src/network.c -index b440d76..3271272 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -670,10 +670,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude( - loc_network_unref(subnet2); - } + -- Return a list of those networks enriched with all + -- other information that we store in the database + SELECT +- DISTINCT ON (known_networks.network) +- known_networks.network AS network, +- announcements.autnum AS autnum, ++ DISTINCT ON (network) ++ network, ++ autnum,
--#ifdef ENABLE_DEBUG -- loc_network_list_dump(list); --#endif + -- Country + COALESCE( + ( + SELECT country FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT country FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + networks.country + ) AS country, +@@ -268,50 +264,67 @@ class CLI(object): + COALESCE( + ( + SELECT is_anonymous_proxy FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anonymous_proxy FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_anonymous_proxy, + COALESCE( + ( + SELECT is_satellite_provider FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_satellite_provider FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_satellite_provider, + COALESCE( + ( + SELECT is_anycast FROM network_overrides overrides +- WHERE announcements.network <<= overrides.network ++ WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anycast FROM autnum_overrides overrides +- WHERE announcements.autnum = overrides.number ++ WHERE networks.autnum = overrides.number + ), + FALSE +- ) AS is_anycast, - - // Return the result - return list; - --- -2.20.1 - -From ed0f53df0f3ebb915faf25138cc09df7555415a3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 16:25:56 +0000 -Subject: [PATCH 081/111] database: Flatten out code due to compiler errors - -It looks like GCC could not follow the code as it was -written. Therefore I clean up more often now instead -of having a GOTO block where I do that in. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 59 ++++++++++++++++++++++++++++++++++---------------- - 1 file changed, 40 insertions(+), 19 deletions(-) - -diff --git a/src/database.c b/src/database.c -index ef4f505..c21b957 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1305,8 +1305,12 @@ static int __loc_database_enumerator_next_network_flattened( - while (1) { - // Fetch the next network in line - r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); -- if (r) -- goto END; -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); +- -- Must be part of returned values for ORDER BY clause +- masklen(announcements.network) AS sort_a, +- masklen(networks.network) AS sort_b +- FROM known_networks +- LEFT JOIN announcements ON known_networks.network <<= announcements.network +- LEFT JOIN networks ON known_networks.network <<= networks.network +- ORDER BY known_networks.network, sort_a DESC, sort_b DESC ++ ) AS is_anycast ++ FROM ( ++ SELECT ++ known_networks.network AS network, ++ announcements.autnum AS autnum, ++ networks.country AS country, + -+ return r; -+ } ++ -- Must be part of returned values for ORDER BY clause ++ masklen(announcements.network) AS sort_a, ++ masklen(networks.network) AS sort_b ++ FROM ( ++ SELECT network FROM announcements ++ UNION ALL ++ SELECT network FROM networks ++ UNION ALL ++ SELECT network FROM network_overrides ++ ) known_networks ++ LEFT JOIN ++ announcements ON known_networks.network <<= announcements.network ++ LEFT JOIN ++ networks ON known_networks.network <<= networks.network ++ ORDER BY ++ known_networks.network, ++ sort_a DESC, ++ sort_b DESC ++ ) networks + """)
- // End if we did not receive another subnet - if (!subnet) -@@ -1315,8 +1319,12 @@ static int __loc_database_enumerator_next_network_flattened( - // Collect all subnets in a list - if (loc_network_is_subnet(*network, subnet)) { - r = loc_network_list_push(subnets, subnet); -- if (r) -- goto END; -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); + for row in rows: +@@ -363,6 +376,16 @@ class CLI(object): + CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) + ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); + -+ return r; -+ } ++ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) ++ ON COMMIT DROP; ++ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); ++ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); ++ """) ++ ++ # Remove all previously imported content ++ self.db.execute(""" ++ TRUNCATE TABLE networks; + """)
- loc_network_unref(subnet); - continue; -@@ -1324,8 +1332,12 @@ static int __loc_database_enumerator_next_network_flattened( + for source in location.importer.WHOIS_SOURCES: +@@ -370,31 +393,72 @@ class CLI(object): + for block in f: + self._parse_block(block)
- // If this is not a subnet, we push it back onto the stack and break - r = loc_network_list_push(enumerator->stack, subnet); -- if (r) -- goto END; -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); ++ # Process all parsed networks from every RIR we happen to have access to, ++ # insert the largest network chunks into the networks table immediately... ++ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") + -+ return r; -+ } - - loc_network_unref(subnet); - break; -@@ -1343,29 +1355,38 @@ static int __loc_database_enumerator_next_network_flattened( - // If the network has any subnets, we will break it into smaller parts - // without the subnets. - struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); -- if (!excluded || loc_network_list_empty(excluded)) { -- r = 1; -- goto END; -+ if (!excluded) { -+ loc_network_list_unref(subnets); -+ return -1; -+ } ++ for family in (row.family for row in families): ++ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) + -+ // Merge excluded list with subnets -+ r = loc_network_list_merge(subnets, excluded); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); ++ self.db.execute("INSERT INTO networks(network, country) \ ++ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) + -+ return r; - } - -+ // We no longer need the excluded list -+ loc_network_list_unref(excluded); ++ # ... determine any other prefixes for this network family, ... ++ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ ++ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) + -+ // Sort all subnets -+ loc_network_list_sort(subnets); ++ # ... and insert networks with this prefix in case they provide additional ++ # information (i. e. subnet of a larger chunk with a different country) ++ for prefix in (row.prefix for row in prefixes): ++ self.db.execute(""" ++ WITH candidates AS ( ++ SELECT ++ _rirdata.network, ++ _rirdata.country ++ FROM ++ _rirdata ++ WHERE ++ family(_rirdata.network) = %s ++ AND ++ masklen(_rirdata.network) = %s ++ ), ++ filtered AS ( ++ SELECT ++ DISTINCT ON (c.network) ++ c.network, ++ c.country, ++ masklen(networks.network), ++ networks.country AS parent_country ++ FROM ++ candidates c ++ LEFT JOIN ++ networks ++ ON ++ c.network << networks.network ++ ORDER BY ++ c.network, ++ masklen(networks.network) DESC NULLS LAST ++ ) ++ INSERT INTO ++ networks(network, country) ++ SELECT ++ network, ++ country ++ FROM ++ filtered ++ WHERE ++ parent_country IS NULL ++ OR ++ country <> parent_country ++ ON CONFLICT DO NOTHING""", ++ family, prefix, ++ ) + - // Replace network with the first one - loc_network_unref(*network); - -- *network = loc_network_list_pop_first(excluded); -+ *network = loc_network_list_pop_first(subnets); - - // Push the rest onto the stack -- loc_network_list_reverse(excluded); -- loc_network_list_merge(enumerator->stack, excluded); + self.db.execute(""" + INSERT INTO autnums(number, name) + SELECT _autnums.number, _organizations.name FROM _autnums + JOIN _organizations ON _autnums.organization = _organizations.handle +- ON CONFLICT (number) DO UPDATE SET name = excluded.name +- """) +- +- self.db.execute(""" +- --- Purge any redundant entries +- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP +- AS +- SELECT network FROM networks candidates +- WHERE EXISTS ( +- SELECT FROM networks +- WHERE +- networks.network << candidates.network +- AND +- networks.country = candidates.country +- ); - -- loc_network_list_unref(excluded); +- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); - --END: -- if (subnet) -- loc_network_unref(subnet); -+ loc_network_list_reverse(subnets); -+ loc_network_list_merge(enumerator->stack, subnets); - - loc_network_list_unref(subnets); - -- return r; -+ return 0; - } - - LOC_EXPORT int loc_database_enumerator_next_network( --- -2.20.1 - -From a1024390795d60fab9f697fa230c9901ebd7d221 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 16:42:55 +0000 -Subject: [PATCH 082/111] network-list: Implement merging in reverse in one - step - -This will save us some time because we do not need to -change the list in place first and then merge it. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 3 +-- - src/libloc.sym | 1 + - src/loc/network-list.h | 1 + - src/network-list.c | 13 +++++++++++++ - 4 files changed, 16 insertions(+), 2 deletions(-) - -diff --git a/src/database.c b/src/database.c -index c21b957..ba4f98a 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1381,8 +1381,7 @@ static int __loc_database_enumerator_next_network_flattened( - *network = loc_network_list_pop_first(subnets); - - // Push the rest onto the stack -- loc_network_list_reverse(subnets); -- loc_network_list_merge(enumerator->stack, subnets); -+ loc_network_list_merge_reverse(enumerator->stack, subnets); - - loc_network_list_unref(subnets); - -diff --git a/src/libloc.sym b/src/libloc.sym -index 406dd15..56cada8 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -140,6 +140,7 @@ global: - loc_network_list_empty; - loc_network_list_get; - loc_network_list_merge; -+ loc_network_list_merge_reverse; - loc_network_list_new; - loc_network_list_pop; - loc_network_list_pop_first; -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -index af3b28d..89776a6 100644 ---- a/src/loc/network-list.h -+++ b/src/loc/network-list.h -@@ -33,5 +33,6 @@ int loc_network_list_contains(struct loc_network_list* list, struct loc_network* - void loc_network_list_sort(struct loc_network_list* list); - void loc_network_list_reverse(struct loc_network_list* list); - int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); -+int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other); +- DELETE FROM networks WHERE EXISTS ( +- SELECT FROM _garbage WHERE networks.network = _garbage.network +- ); ++ ON CONFLICT (number) DO UPDATE SET name = excluded.name; + """)
- #endif -diff --git a/src/network-list.c b/src/network-list.c -index 9cb4547..b455caf 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -242,3 +242,16 @@ LOC_EXPORT int loc_network_list_merge( + # Download all extended sources +@@ -405,6 +469,69 @@ class CLI(object): + for line in f: + self._parse_line(line)
- return 0; - } ++ def _check_parsed_network(self, network): ++ """ ++ Assistive function to detect and subsequently sort out parsed ++ networks from RIR data (both Whois and so-called "extended sources"), ++ which are or have... + -+LOC_EXPORT int loc_network_list_merge_reverse( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; ++ (a) not globally routable (RFC 1918 space, et al.) ++ (b) covering a too large chunk of the IP address space (prefix length ++ is < 7 for IPv4 networks, and < 10 for IPv6) ++ (c) "0.0.0.0" or "::" as a network address ++ (d) are too small for being publicly announced (we have decided not to ++ process them at the moment, as they significantly enlarge our ++ database without providing very helpful additional information) + -+ for (int i = other->size - 1; i >= 0; i--) { -+ r = loc_network_list_push(self, other->elements[i]); -+ if (r) -+ return r; -+ } ++ This unfortunately is necessary due to brain-dead clutter across ++ various RIR databases, causing mismatches and eventually disruptions. + -+ return 0; -+} --- -2.20.1 - -From a5967330d530504db401540d4bcd5474fe00e421 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 18:36:48 +0000 -Subject: [PATCH 083/111] network: Optimise _subnet function - -This function used to create a network list which always -had exactly two elements. Since splitting a network in half -always returns two parts, we can simply return them as a -pointer. - -This improves returning the network tree by about 17%. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/loc/network.h | 2 +- - src/network.c | 158 ++++++++++++++++++--------------------------- - src/test-network.c | 51 ++++++++------- - 3 files changed, 91 insertions(+), 120 deletions(-) - -diff --git a/src/loc/network.h b/src/loc/network.h -index 4b7410c..d67ec54 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -62,7 +62,7 @@ int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); --struct loc_network_list* loc_network_subnets(struct loc_network* network); -+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); - struct loc_network_list* loc_network_exclude( - struct loc_network* self, struct loc_network* other); - struct loc_network_list* loc_network_exclude_list( -diff --git a/src/network.c b/src/network.c -index 3271272..ff9c1eb 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -507,66 +507,91 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net - return loc_network_is_subnet(other, self); - } - --LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) { -- struct loc_network_list* list; -+LOC_EXPORT int loc_network_subnets(struct loc_network* network, -+ struct loc_network** subnet1, struct loc_network** subnet2) { -+ int r; -+ *subnet1 = NULL; -+ *subnet2 = NULL; - - // New prefix length - unsigned int prefix = network->prefix + 1; - - // Check if the new prefix is valid - if (valid_prefix(&network->first_address, prefix)) -- return NULL; -- -- // Create a new list with the result -- int r = loc_network_list_new(network->ctx, &list); -- if (r) { -- ERROR(network->ctx, "Could not create network list: %d\n", r); -- return NULL; -- } -- -- struct loc_network* subnet1 = NULL; -- struct loc_network* subnet2 = NULL; -+ return -1; - - // Create the first half of the network -- r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix); -+ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix); - if (r) -- goto ERROR; -+ return r; - - // The next subnet starts after the first one -- struct in6_addr first_address = address_increment(&subnet1->last_address); -+ struct in6_addr first_address = address_increment(&(*subnet1)->last_address); - - // Create the second half of the network -- r = loc_network_new(network->ctx, &subnet2, &first_address, prefix); -- if (r) -- goto ERROR; -- -- // Push the both onto the stack (in reverse order) -- r = loc_network_list_push(list, subnet2); -+ r = loc_network_new(network->ctx, subnet2, &first_address, prefix); - if (r) -- goto ERROR; -- -- r = loc_network_list_push(list, subnet1); -- if (r) -- goto ERROR; -+ return r; - - // Copy country code - const char* country_code = loc_network_get_country_code(network); - if (country_code) { -- loc_network_set_country_code(subnet1, country_code); -- loc_network_set_country_code(subnet2, country_code); -+ loc_network_set_country_code(*subnet1, country_code); -+ loc_network_set_country_code(*subnet2, country_code); - } - - // Copy ASN - uint32_t asn = loc_network_get_asn(network); - if (asn) { -- loc_network_set_asn(subnet1, asn); -- loc_network_set_asn(subnet2, asn); -+ loc_network_set_asn(*subnet1, asn); -+ loc_network_set_asn(*subnet2, asn); - } - -- loc_network_unref(subnet1); -- loc_network_unref(subnet2); -+ return 0; -+} - -- return list; -+static int __loc_network_exclude(struct loc_network* network, -+ struct loc_network* other, struct loc_network_list* list) { -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; ++ We will return False in case a network is not suitable for adding ++ it to our database, and True otherwise. ++ """ + -+ int r = loc_network_subnets(network, &subnet1, &subnet2); -+ if (r) -+ goto ERROR; ++ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): ++ return False + -+ if (loc_network_eq(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; ++ if not network.is_global: ++ log.warning("Skipping non-globally routable network: %s" % network) ++ return False + -+ } else if (loc_network_eq(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; ++ if network.version == 4: ++ if network.prefixlen < 7: ++ log.warning("Skipping too big IP chunk: %s" % network) ++ return False + -+ } else if (loc_network_is_subnet_of(other, subnet1)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; ++ if network.prefixlen > 24: ++ log.debug("Skipping network too small to be publicly announced: %s" % network) ++ return False + -+ r = __loc_network_exclude(subnet1, other, list); -+ if (r) -+ goto ERROR; ++ if str(network.network_address) == "0.0.0.0": ++ log.warning("Skipping network based on 0.0.0.0: %s" % network) ++ return False + -+ } else if (loc_network_is_subnet_of(other, subnet2)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; ++ elif network.version == 6: ++ if network.prefixlen < 10: ++ log.warning("Skipping too big IP chunk: %s" % network) ++ return False + -+ r = __loc_network_exclude(subnet2, other, list); -+ if (r) -+ goto ERROR; ++ if network.prefixlen > 48: ++ log.debug("Skipping network too small to be publicly announced: %s" % network) ++ return False + -+ } else { -+ ERROR(network->ctx, "We should never get here\n"); -+ r = 1; -+ goto ERROR; -+ } - - ERROR: - if (subnet1) -@@ -575,10 +600,7 @@ ERROR: - if (subnet2) - loc_network_unref(subnet2); - -- if (list) -- loc_network_list_unref(list); -- -- return NULL; -+ return r; - } - - LOC_EXPORT struct loc_network_list* loc_network_exclude( -@@ -623,67 +645,15 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude( - return NULL; - } - -- struct loc_network_list* subnets = loc_network_subnets(self); -- -- struct loc_network* subnet1 = NULL; -- struct loc_network* subnet2 = NULL; -- -- while (subnets) { -- // Fetch both subnets -- subnet1 = loc_network_list_get(subnets, 0); -- subnet2 = loc_network_list_get(subnets, 1); -- -- // Free list -- loc_network_list_unref(subnets); -- subnets = NULL; -- -- if (loc_network_eq(other, subnet1)) { -- r = loc_network_list_push(list, subnet2); -- if (r) -- goto ERROR; -- -- } else if (loc_network_eq(other, subnet2)) { -- r = loc_network_list_push(list, subnet1); -- if (r) -- goto ERROR; -- -- } else if (loc_network_is_subnet_of(other, subnet1)) { -- r = loc_network_list_push(list, subnet2); -- if (r) -- goto ERROR; -- -- subnets = loc_network_subnets(subnet1); -- -- } else if (loc_network_is_subnet_of(other, subnet2)) { -- r = loc_network_list_push(list, subnet1); -- if (r) -- goto ERROR; -- -- subnets = loc_network_subnets(subnet2); -- -- } else { -- ERROR(self->ctx, "We should never get here\n"); -- goto ERROR; -- } -+ r = __loc_network_exclude(self, other, list); -+ if (r) { -+ loc_network_list_unref(list); ++ if str(network.network_address) == "::": ++ log.warning("Skipping network based on '::': %s" % network) ++ return False ++ ++ else: ++ # This should not happen... ++ log.warning("Skipping network of unknown family, this should not happen: %s" % network) ++ return False ++ ++ # In case we have made it here, the network is considered to ++ # be suitable for libloc consumption... ++ return True ++ + def _parse_block(self, block): + # Get first line to find out what type of block this is + line = block[0] +@@ -433,7 +560,7 @@ class CLI(object): + autnum["asn"] = m.group(2)
-- loc_network_unref(subnet1); -- loc_network_unref(subnet2); -+ return NULL; - } + elif key == "org": +- autnum[key] = val ++ autnum[key] = val.upper()
- // Return the result - return list; -- --ERROR: -- if (subnet1) -- loc_network_unref(subnet1); -- -- if (subnet2) -- loc_network_unref(subnet2); -- -- if (list) -- loc_network_list_unref(list); -- -- return NULL; - } + # Skip empty objects + if not autnum: +@@ -447,15 +574,22 @@ class CLI(object): + )
- LOC_EXPORT struct loc_network_list* loc_network_exclude_list( -diff --git a/src/test-network.c b/src/test-network.c -index 7c90224..79c2967 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -138,47 +138,48 @@ int main(int argc, char** argv) { - } + def _parse_inetnum_block(self, block): +- logging.debug("Parsing inetnum block:") ++ log.debug("Parsing inetnum block:")
- // Check subnet function -- err = loc_network_is_subnet_of(network1, network2); -- if (err != 0) { -+ err = loc_network_is_subnet(network1, network2); -+ if (!err) { - fprintf(stderr, "Subnet check 1 failed: %d\n", err); - exit(EXIT_FAILURE); - } + inetnum = {} + for line in block: +- logging.debug(line) ++ log.debug(line)
-- err = loc_network_is_subnet_of(network2, network1); -- if (err != 1) { -+ err = loc_network_is_subnet(network2, network1); -+ if (err) { - fprintf(stderr, "Subnet check 2 failed: %d\n", err); - exit(EXIT_FAILURE); - } + # Split line + key, val = split_line(line)
-- // Make list of subnets -- struct loc_network_list* subnets = loc_network_subnets(network1); -- if (!subnets) { -- fprintf(stderr, "Could not find subnets of network\n"); -+ // Make subnets -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; ++ # Filter any inetnum records which are only referring to IP space ++ # not managed by that specific RIR... ++ if key == "netname": ++ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): ++ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) ++ return + -+ err = loc_network_subnets(network1, &subnet1, &subnet2); -+ if (err || !subnet1 || !subnet2) { -+ fprintf(stderr, "Could not find subnets of network: %d\n", err); - exit(EXIT_FAILURE); - } + if key == "inetnum": + start_address, delim, end_address = val.partition("-") + +@@ -467,7 +601,7 @@ class CLI(object): + start_address = ipaddress.ip_address(start_address) + end_address = ipaddress.ip_address(end_address) + except ValueError: +- logging.warning("Could not parse line: %s" % line) ++ log.warning("Could not parse line: %s" % line) + return + + # Set prefix to default +@@ -484,23 +618,24 @@ class CLI(object): + inetnum[key] = val
-- loc_network_list_dump(subnets); + elif key == "country": +- if val == "UNITED STATES": +- val = "US" - -- while (!loc_network_list_empty(subnets)) { -- struct loc_network* subnet = loc_network_list_pop(subnets); -- if (!subnet) { -- fprintf(stderr, "Received an empty subnet\n"); -- exit(EXIT_FAILURE); -- } -+ char* s = loc_network_str(subnet1); -+ printf("Received subnet1 = %s\n", s); -+ free(s); + inetnum[key] = val.upper()
-- char* s = loc_network_str(subnet); -- printf("Received subnet %s\n", s); -- free(s); -+ s = loc_network_str(subnet2); -+ printf("Received subnet2 = %s\n", s); -+ free(s); + # Skip empty objects +- if not inetnum: ++ if not inetnum or not "country" in inetnum: ++ return ++ ++ # Skip objects with bogus country code 'ZZ' ++ if inetnum.get("country") == "ZZ": ++ log.warning("Skipping network with bogus country 'ZZ': %s" % \ ++ (inetnum.get("inet6num") or inetnum.get("inetnum"))) + return
-- if (!loc_network_is_subnet_of(subnet, network1)) { -- fprintf(stderr, "Not a subnet\n"); -- exit(EXIT_FAILURE); -- } -+ if (!loc_network_is_subnet(network1, subnet1)) { -+ fprintf(stderr, "Subnet1 is not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-- loc_network_unref(subnet); -+ if (!loc_network_is_subnet(network1, subnet2)) { -+ fprintf(stderr, "Subnet2 is not a subnet\n"); -+ exit(EXIT_FAILURE); - } +- # Bail out in case we have processed a non-public IP network +- if network.is_private: +- logging.warning("Skipping non-globally routable network: %s" % network) ++ if not self._check_parsed_network(network): + return
-- loc_network_list_unref(subnets); -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); +- self.db.execute("INSERT INTO networks(network, country) \ ++ self.db.execute("INSERT INTO _rirdata(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % network, inetnum.get("country"), + ) +@@ -511,7 +646,9 @@ class CLI(object): + # Split line + key, val = split_line(line)
- struct loc_network_list* excluded = loc_network_exclude(network1, network2); - if (!excluded) { --- -2.20.1 - -From 7fe6a21845edf6692d239f228117bc95620d0419 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 18:38:47 +0000 -Subject: [PATCH 084/111] network: Add function to return the prefix - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network.c | 12 ++++++++++++ - 3 files changed, 14 insertions(+) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 56cada8..9f41c89 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -125,6 +125,7 @@ global: - loc_network_new; - loc_network_new_from_string; - loc_network_overlaps; -+ loc_network_prefix; - loc_network_ref; - loc_network_set_asn; - loc_network_set_country_code; -diff --git a/src/loc/network.h b/src/loc/network.h -index d67ec54..7b2ae4c 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -38,6 +38,7 @@ struct loc_network* loc_network_ref(struct loc_network* network); - struct loc_network* loc_network_unref(struct loc_network* network); - char* loc_network_str(struct loc_network* network); - int loc_network_address_family(struct loc_network* network); -+unsigned int loc_network_prefix(struct loc_network* network); +- if key in ("organisation", "org-name"): ++ if key == "organisation": ++ org[key] = val.upper() ++ elif key == "org-name": + org[key] = val
- const struct in6_addr* loc_network_get_first_address(struct loc_network* network); - char* loc_network_format_first_address(struct loc_network* network); -diff --git a/src/network.c b/src/network.c -index ff9c1eb..4720503 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -310,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) { - return network->family; - } + # Skip empty objects +@@ -581,6 +718,9 @@ class CLI(object): + log.warning("Invalid IP address: %s" % address) + return
-+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) { -+ switch (network->family) { -+ case AF_INET6: -+ return network->prefix; -+ -+ case AF_INET: -+ return network->prefix - 96; -+ } -+ -+ return 0; -+} ++ if not self._check_parsed_network(network): ++ return + - static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) { - const size_t length = INET6_ADDRSTRLEN; - --- -2.20.1 - -From cf8d3c6454843943e3bc81eb85522779d1c11f9b Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 18:39:13 +0000 -Subject: [PATCH 085/111] network: Speed up subnet check - -There is no point in checking different address families -with each other and we do not need to compare addresses -when the prefix of the subnet does not fit into the -network to check. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/network.c b/src/network.c -index 4720503..e52b58c 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -501,6 +501,14 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - } + self.db.execute("INSERT INTO networks(network, country) \ + VALUES(%s, %s) ON CONFLICT (network) DO \ + UPDATE SET country = excluded.country", +diff --git a/src/python/location.in b/src/python/location.in +index 44ad726..b30beae 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -253,6 +253,7 @@ class CLI(object): + network = db.lookup(address) + except ValueError: + print(_("Invalid IP address: %s") % address, file=sys.stderr) ++ return 2
- LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { -+ // Check family -+ if (self->family != other->family) -+ return 0; -+ -+ // The prefix must be smaller (this avoids the more complex comparisons later) -+ if (self->prefix > other->prefix) -+ return 0; -+ - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. - if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) --- -2.20.1 - -From d6a5092f969bb3bd50d130d4ba64b4e4be2e61f6 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Fri, 20 Nov 2020 18:44:42 +0000 -Subject: [PATCH 086/111] network: Remove deprecated loc_network_is_subnet_of - function - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 - - src/loc/network.h | 1 - - src/network.c | 13 ++++--------- - src/python/network.c | 2 +- - 4 files changed, 5 insertions(+), 12 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 9f41c89..b2d8a31 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -118,7 +118,6 @@ global: - loc_network_gt; - loc_network_has_flag; - loc_network_is_subnet; -- loc_network_is_subnet_of; - loc_network_match_asn; - loc_network_match_country_code; - loc_network_match_flag; -diff --git a/src/loc/network.h b/src/loc/network.h -index 7b2ae4c..b31c8a2 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -62,7 +62,6 @@ int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); --int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); - int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); - struct loc_network_list* loc_network_exclude( - struct loc_network* self, struct loc_network* other); -diff --git a/src/network.c b/src/network.c -index e52b58c..72b77e6 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -522,11 +522,6 @@ LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_networ - return 1; - } + args = { + "address" : address, +@@ -398,10 +399,7 @@ class CLI(object):
--// XXX DEPRECATED - I find this too difficult to use --LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { -- return loc_network_is_subnet(other, self); --} + def handle_update(self, db, ns): + if ns.cron and db: +- now = datetime.datetime.utcnow() - - LOC_EXPORT int loc_network_subnets(struct loc_network* network, - struct loc_network** subnet1, struct loc_network** subnet2) { - int r; -@@ -589,7 +584,7 @@ static int __loc_network_exclude(struct loc_network* network, - if (r) - goto ERROR; - -- } else if (loc_network_is_subnet_of(other, subnet1)) { -+ } else if (loc_network_is_subnet(subnet1, other)) { - r = loc_network_list_push(list, subnet2); - if (r) - goto ERROR; -@@ -598,7 +593,7 @@ static int __loc_network_exclude(struct loc_network* network, - if (r) - goto ERROR; - -- } else if (loc_network_is_subnet_of(other, subnet2)) { -+ } else if (loc_network_is_subnet(subnet2, other)) { - r = loc_network_list_push(list, subnet1); - if (r) - goto ERROR; -@@ -645,7 +640,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude( - } - - // Other must be a subnet of self -- if (!loc_network_is_subnet_of(other, self)) { -+ if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); +- # Parse the database timestamp +- t = datetime.datetime.utcfromtimestamp(db.created_at) ++ now = time.time()
- return NULL; -@@ -726,7 +721,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - } + if ns.cron == "daily": + delta = datetime.timedelta(days=1) +@@ -410,22 +408,20 @@ class CLI(object): + elif ns.cron == "monthly": + delta = datetime.timedelta(days=30)
- // Drop this subnet if is a subnet of another subnet -- if (loc_network_is_subnet_of(subnet, subnet_to_check)) { -+ if (loc_network_is_subnet(subnet_to_check, subnet)) { - passed = 0; - loc_network_unref(subnet); - break; -diff --git a/src/python/network.c b/src/python/network.c -index 742b472..b6e92fb 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -194,7 +194,7 @@ static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { - if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) - return NULL; ++ delta = delta.total_seconds() ++ + # Check if the database has recently been updated +- if t >= (now - delta): ++ if db.created_at >= (now - delta): + log.info( +- _("The database has been updated recently (%s)") % \ +- format_timedelta(now - t), ++ _("The database has been updated recently"), + ) + return 3
-- if (loc_network_is_subnet_of(self->network, other->network)) -+ if (loc_network_is_subnet(other->network, self->network)) - Py_RETURN_TRUE; + # Fetch the timestamp we need from DNS + t = location.discover_latest_version()
- Py_RETURN_FALSE; --- -2.20.1 - -From af4689bf5c56ad79e9e90396b41be460e49ef384 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 14:55:08 +0000 -Subject: [PATCH 087/111] network-list: Make this a sorted list - -The list is now organised so that it is always ordered. - -This allows us to find if a network is on the list a lot -quicker using binary search as well as inserting a new -network at the right place. - -This will benefit loc_network_exclude with very large networks. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 + - src/loc/network.h | 1 + - src/network-list.c | 84 +++++++++++++++++++++++++++++++++++++++++----- - src/network.c | 22 ++++++++++++ - 4 files changed, 99 insertions(+), 9 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index b2d8a31..28cc8e8 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -106,6 +106,7 @@ global: +- # Parse timestamp into datetime format +- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None +- + # Check the version of the local database +- if db and timestamp and db.created_at >= timestamp.timestamp(): ++ if db and t and db.created_at >= t: + log.info("Already on the latest version") + return
- # Network - loc_network_address_family; -+ loc_network_cmp; - loc_network_eq; - loc_network_exclude; - loc_network_exclude_list; -diff --git a/src/loc/network.h b/src/loc/network.h -index b31c8a2..8ab1562 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -58,6 +58,7 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); - int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); +@@ -437,7 +433,7 @@ class CLI(object):
-+int loc_network_cmp(struct loc_network* self, struct loc_network* other); - int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); -diff --git a/src/network-list.c b/src/network-list.c -index b455caf..6e9cd37 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -16,6 +16,7 @@ + # Try downloading a new database + try: +- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) ++ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
- #include <errno.h> - #include <stdlib.h> -+#include <time.h> + # If no file could be downloaded, log a message + except FileNotFoundError as e: +@@ -453,13 +449,7 @@ class CLI(object):
- #include <loc/libloc.h> - #include <loc/network.h> -@@ -130,11 +131,71 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - return loc_network_ref(list->elements[index]); - } + return 0
-+//MOVE FUNCTION GOES HERE -+ -+static off_t loc_network_list_find(struct loc_network_list* list, -+ struct loc_network* network, int* found) { -+ off_t lo = 0; -+ off_t hi = list->size - 1; -+ -+ *found = 0; -+ -+#ifdef ENABLE_DEBUG -+ // Save start time -+ clock_t start = clock(); -+#endif -+ -+ off_t i = 0; -+ -+ while (lo <= hi) { -+ i = (lo + hi) / 2; -+ -+ // Check if this is a match -+ int result = loc_network_cmp(network, list->elements[i]); -+ -+ if (result == 0) { -+ *found = 1; -+ -+#ifdef ENABLE_DEBUG -+ clock_t end = clock(); +- def handle_verify(self, ns): +- try: +- db = location.Database(ns.database) +- except FileNotFoundError as e: +- log.error("%s: %s" % (ns.database, e)) +- return 127 +- ++ def handle_verify(self, db, ns): + # Verify the database + with open(ns.public_key, "r") as f: + if not db.verify(f): +diff --git a/src/python/network.c b/src/python/network.c +index 5496d1e..b6e92fb 100644 +--- a/src/python/network.c ++++ b/src/python/network.c +@@ -20,10 +20,29 @@ + + #include <loc/libloc.h> + #include <loc/network.h> ++#include <loc/network-list.h> + + #include "locationmodule.h" + #include "network.h" + ++static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { ++ PyObject* list = PyList_New(0); ++ if (!networks) ++ return list; + -+ // Log how fast this has been -+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif ++ while (!loc_network_list_empty(networks)) { ++ struct loc_network* network = loc_network_list_pop(networks); + -+ return i; -+ } ++ PyObject* n = new_network(&NetworkType, network); ++ PyList_Append(list, n); + -+ if (result > 0) -+ lo = i + 1; -+ else -+ hi = i - 1; ++ loc_network_unref(network); ++ Py_DECREF(n); + } + -+#ifdef ENABLE_DEBUG -+ clock_t end = clock(); ++ return list; ++} + -+ // Log how fast this has been -+ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif + PyObject* new_network(PyTypeObject* type, struct loc_network* network) { + NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); + if (self) { +@@ -154,13 +173,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { + Py_RETURN_NONE; + } + ++static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { ++ NetworkObject* other = NULL; + -+ return i; -+} ++ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) ++ return NULL; + - LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -- // Do not add networks that are already on the list -- if (loc_network_list_contains(list, network)) -+ int found = 0; ++ struct loc_network_list* list = loc_network_exclude(self->network, other->network); + -+ off_t index = loc_network_list_find(list, network, &found); ++ // Convert to Python objects ++ PyObject* obj = PyList_FromNetworkList(list); ++ loc_network_list_unref(list); + -+ // The network has been found on the list. Nothing to do. -+ if (found) - return 0; - -+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n", -+ list, network, (intmax_t)index); ++ return obj; ++} + - // Check if we have space left - if (list->size >= list->elements_size) { - int r = loc_network_list_grow(list, 64); -@@ -142,9 +203,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n - return r; - } + static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { + NetworkObject* other = NULL;
-- DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network); -+ // The list is now larger -+ list->size++; -+ -+ // Move all elements out of the way -+ for (unsigned int i = list->size - 1; i > index; i--) -+ list->elements[i] = list->elements[i - 1]; + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) + return NULL;
-- list->elements[list->size++] = loc_network_ref(network); -+ // Add the new element at the right place -+ list->elements[index] = loc_network_ref(network); +- if (loc_network_is_subnet_of(self->network, other->network)) ++ if (loc_network_is_subnet(other->network, self->network)) + Py_RETURN_TRUE;
- return 0; - } -@@ -183,12 +250,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis + Py_RETURN_FALSE; +@@ -181,6 +215,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) { + return obj; }
- LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -- for (unsigned int i = 0; i < list->size; i++) { -- if (loc_network_eq(list->elements[i], network)) -- return 1; -- } -+ int found = 0; - -- return 0; -+ loc_network_list_find(list, network, &found); ++static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) { ++ struct in_addr address4; + -+ return found; - } - - static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -diff --git a/src/network.c b/src/network.c -index 72b77e6..38d557a 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -441,6 +441,28 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - return loc_network_has_flag(network, flag); - } - -+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { -+ // Compare family -+ if (self->family > other->family) -+ return 1; -+ else if (self->family < other->family) -+ return -1; ++ // Convert IPv4 addresses to struct in_addr ++ if (IN6_IS_ADDR_V4MAPPED(address6)) { ++ address4.s_addr = address6->s6_addr32[3]; + -+ // Compare address -+ int r = in6_addr_cmp(&self->first_address, &other->first_address); -+ if (r) -+ return r; ++ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4)); ++ } + -+ // Compare prefix -+ if (self->prefix > other->prefix) -+ return 1; -+ else if (self->prefix < other->prefix) -+ return -1; ++ // Return IPv6 addresses as they are ++ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6)); ++} + -+ // Both networks are equal -+ return 0; ++static PyObject* Network_get__first_address(NetworkObject* self) { ++ const struct in6_addr* address = loc_network_get_first_address(self->network); ++ ++ return PyBytes_FromAddress(address); +} + - LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { - // Family must be the same - if (self->family != other->family) --- -2.20.1 - -From 39cbbc63aee362d82f69a9b4722b59153ce799a0 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 15:04:03 +0000 -Subject: [PATCH 088/111] network-list: Drop sorting functions - -Since the list is always sorted, there is no point in -sorting it again... - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 6 +---- - src/libloc.sym | 3 --- - src/loc/network-list.h | 3 --- - src/network-list.c | 52 ------------------------------------------ - src/test-network.c | 9 -------- - 5 files changed, 1 insertion(+), 72 deletions(-) - -diff --git a/src/database.c b/src/database.c -index ba4f98a..5546091 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1372,17 +1372,13 @@ static int __loc_database_enumerator_next_network_flattened( - // We no longer need the excluded list - loc_network_list_unref(excluded); - -- // Sort all subnets -- loc_network_list_sort(subnets); -- - // Replace network with the first one - loc_network_unref(*network); - - *network = loc_network_list_pop_first(subnets); - - // Push the rest onto the stack -- loc_network_list_merge_reverse(enumerator->stack, subnets); -- -+ loc_network_list_merge(enumerator->stack, subnets); - loc_network_list_unref(subnets); - - return 0; -diff --git a/src/libloc.sym b/src/libloc.sym -index 28cc8e8..4b0ce45 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -141,15 +141,12 @@ global: - loc_network_list_empty; - loc_network_list_get; - loc_network_list_merge; -- loc_network_list_merge_reverse; - loc_network_list_new; - loc_network_list_pop; - loc_network_list_pop_first; - loc_network_list_push; - loc_network_list_ref; -- loc_network_list_reverse; - loc_network_list_size; -- loc_network_list_sort; - loc_network_list_unref; - - # Writer -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -index 89776a6..21c7402 100644 ---- a/src/loc/network-list.h -+++ b/src/loc/network-list.h -@@ -30,9 +30,6 @@ int loc_network_list_push(struct loc_network_list* list, struct loc_network* net - struct loc_network* loc_network_list_pop(struct loc_network_list* list); - struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); - int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); --void loc_network_list_sort(struct loc_network_list* list); --void loc_network_list_reverse(struct loc_network_list* list); - int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); --int loc_network_list_merge_reverse(struct loc_network_list* self, struct loc_network_list* other); - - #endif -diff --git a/src/network-list.c b/src/network-list.c -index 6e9cd37..7e8b5f3 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -257,45 +257,6 @@ LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct l - return found; - } - --static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) { -- // Do nothing for invalid indices -- if (i1 >= list->size || i2 >= list->size) -- return; -- -- struct loc_network* network1 = list->elements[i1]; -- struct loc_network* network2 = list->elements[i2]; -- -- list->elements[i1] = network2; -- list->elements[i2] = network1; --} -- --LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) { -- unsigned int i = 0; -- unsigned int j = list->size - 1; -- -- while (i < j) { -- loc_network_list_swap(list, i++, j--); -- } --} -- --LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) { -- unsigned int n = list->size; -- int swapped; -- -- do { -- swapped = 0; -- -- for (unsigned int i = 1; i < n; i++) { -- if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) { -- loc_network_list_swap(list, i-1, i); -- swapped = 1; -- } -- } -- -- n--; -- } while (swapped); --} -- - LOC_EXPORT int loc_network_list_merge( - struct loc_network_list* self, struct loc_network_list* other) { - int r; -@@ -308,16 +269,3 @@ LOC_EXPORT int loc_network_list_merge( + static PyObject* Network_get_last_address(NetworkObject* self) { + char* address = loc_network_format_last_address(self->network);
- return 0; +@@ -190,7 +244,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) { + return obj; } -- --LOC_EXPORT int loc_network_list_merge_reverse( -- struct loc_network_list* self, struct loc_network_list* other) { -- int r; -- -- for (int i = other->size - 1; i >= 0; i--) { -- r = loc_network_list_push(self, other->elements[i]); -- if (r) -- return r; -- } -- -- return 0; --} -diff --git a/src/test-network.c b/src/test-network.c -index 79c2967..8a6763c 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -188,15 +188,6 @@ int main(int argc, char** argv) { - } - - loc_network_list_dump(excluded); -- -- // Reverse it -- loc_network_list_reverse(excluded); -- loc_network_list_dump(excluded); -- -- // Sort them and dump them again -- loc_network_list_sort(excluded); -- loc_network_list_dump(excluded); -- - loc_network_list_unref(excluded); - - // Create a database --- -2.20.1 - -From 5dacb45afceac2d05ea597755c1ca5a1b62cc0fd Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 15:15:59 +0000 -Subject: [PATCH 089/111] database: Avoid merging the same data twice - -When finish splitting networks into many parts, we have -a list of subnets with the excluded subnets and merge them -together first and put them on the stack again. - -This is slower than pushing it all onto the stack first -and then popping the first element. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 24 ++++++++++++++---------- - 1 file changed, 14 insertions(+), 10 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 5546091..f1f6ae0 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1360,8 +1360,8 @@ static int __loc_database_enumerator_next_network_flattened( - return -1; - } - -- // Merge excluded list with subnets -- r = loc_network_list_merge(subnets, excluded); -+ // Merge subnets onto the stack -+ r = loc_network_list_merge(enumerator->stack, subnets); - if (r) { - loc_network_list_unref(subnets); - loc_network_list_unref(excluded); -@@ -1369,17 +1369,21 @@ static int __loc_database_enumerator_next_network_flattened( - return r; - } - -- // We no longer need the excluded list -- loc_network_list_unref(excluded); -- -- // Replace network with the first one -- loc_network_unref(*network); -+ // Push excluded list onto the stack -+ r = loc_network_list_merge(enumerator->stack, excluded); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); - -- *network = loc_network_list_pop_first(subnets); -+ return r; -+ }
-- // Push the rest onto the stack -- loc_network_list_merge(enumerator->stack, subnets); - loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); ++static PyObject* Network_get__last_address(NetworkObject* self) { ++ const struct in6_addr* address = loc_network_get_last_address(self->network); + -+ // Replace network with the first one from the stack -+ loc_network_unref(*network); -+ *network = loc_network_list_pop_first(enumerator->stack); - - return 0; - } --- -2.20.1 - -From 04cbd2bfa892fc7374ad506ec7ba81727c5a4b96 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 15:22:02 +0000 -Subject: [PATCH 090/111] database: Read the first element from the list - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/database.c b/src/database.c -index f1f6ae0..914ed3e 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1191,7 +1191,7 @@ static int __loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { - // Return top element from the stack - while (1) { -- *network = loc_network_list_pop(enumerator->stack); -+ *network = loc_network_list_pop_first(enumerator->stack); - - // Stack is empty - if (!*network) --- -2.20.1 - -From c650008e2dd9af5fd1eadba6354aa0b615047f84 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 15:41:53 +0000 -Subject: [PATCH 091/111] network: Add excluded networks to to_check list - -This is now being done immediately instead of creating a new -list which is being merged later. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 55 ++++++++++++++++++++++++++++++--------------------- - 1 file changed, 32 insertions(+), 23 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 38d557a..7ab22f8 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -640,49 +640,55 @@ ERROR: - return r; - } - --LOC_EXPORT struct loc_network_list* loc_network_exclude( -- struct loc_network* self, struct loc_network* other) { -- struct loc_network_list* list; -- --#ifdef ENABLE_DEBUG -- char* n1 = loc_network_str(self); -- char* n2 = loc_network_str(other); -- -- DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); -- -- free(n1); -- free(n2); --#endif -- -+static int __loc_network_exclude_to_list(struct loc_network* self, -+ struct loc_network* other, struct loc_network_list* list) { - // Family must match - if (self->family != other->family) { - DEBUG(self->ctx, "Family mismatch\n"); - -- return NULL; -+ return 1; - } - - // Other must be a subnet of self - if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); ++ return PyBytes_FromAddress(address); ++} ++ + static struct PyMethodDef Network_methods[] = { ++ { ++ "exclude", ++ (PyCFunction)Network_exclude, ++ METH_VARARGS, ++ NULL, ++ }, + { + "has_flag", + (PyCFunction)Network_has_flag, +@@ -241,6 +307,13 @@ static struct PyGetSetDef Network_getsetters[] = { + NULL, + NULL, + }, ++ { ++ "_first_address", ++ (getter)Network_get__first_address, ++ NULL, ++ NULL, ++ NULL, ++ }, + { + "last_address", + (getter)Network_get_last_address, +@@ -248,6 +321,13 @@ static struct PyGetSetDef Network_getsetters[] = { + NULL, + NULL, + }, ++ { ++ "_last_address", ++ (getter)Network_get__last_address, ++ NULL, ++ NULL, ++ NULL, ++ }, + { NULL }, + };
-- return NULL; -+ return 1; - } +diff --git a/src/test-as.c b/src/test-as.c +index 839a04c..2d61675 100644 +--- a/src/test-as.c ++++ b/src/test-as.c +@@ -95,7 +95,7 @@ int main(int argc, char** argv) { + // Enumerator
- // We cannot perform this operation if both networks equal - if (loc_network_eq(self, other)) { - DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); + struct loc_database_enumerator* enumerator; +- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES); ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); + if (err) { + fprintf(stderr, "Could not create a database enumerator\n"); + exit(EXIT_FAILURE); +diff --git a/src/test-database.c b/src/test-database.c +index b4a75c4..da4b11c 100644 +--- a/src/test-database.c ++++ b/src/test-database.c +@@ -38,6 +38,14 @@ const char* DESCRIPTION = + "Maecenas ut venenatis nunc."; + const char* LICENSE = "CC";
-- return NULL; -+ return 1; ++const char* networks[] = { ++ "2001:db8::/32", ++ "2001:db8:1000::/48", ++ "2001:db8:2000::/48", ++ "2001:db8:2020::/48", ++ NULL, ++}; ++ + static int attempt_to_open(struct loc_ctx* ctx, char* path) { + FILE* f = fopen(path, "r"); + if (!f) +@@ -139,6 +147,24 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); }
-+ return __loc_network_exclude(self, other, list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other) { -+ struct loc_network_list* list; -+ -+#ifdef ENABLE_DEBUG -+ char* n1 = loc_network_str(self); -+ char* n2 = loc_network_str(other); ++ struct loc_network* network = NULL; + -+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); ++ // Add some networks ++ const char** n = networks; ++ while (*n) { ++ err = loc_writer_add_network(writer, &network, *n); ++ if (err) { ++ fprintf(stderr, "Could not add network %s\n", *n); ++ exit(EXIT_FAILURE); ++ } + -+ free(n1); -+ free(n2); -+#endif ++ // Set a country ++ loc_network_set_country_code(network, "XX"); + - // Create a new list with the result - int r = loc_network_list_new(self->ctx, &list); - if (r) { - ERROR(self->ctx, "Could not create network list: %d\n", r); ++ // Next one ++ n++; ++ } + - return NULL; + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +@@ -170,6 +196,33 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); }
-- r = __loc_network_exclude(self, other, list); -+ r = __loc_network_exclude_to_list(self, other, list); - if (r) { - loc_network_list_unref(list); - -@@ -709,11 +715,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - subnet = loc_network_list_get(list, i); - - // Find all excluded networks -- struct loc_network_list* excluded = loc_network_exclude(network, subnet); -- if (excluded) { -- // Add them all to the "to check" list -- loc_network_list_merge(to_check, excluded); -- loc_network_list_unref(excluded); -+ if (!loc_network_list_contains(to_check, subnet)) { -+ r = __loc_network_exclude_to_list(network, subnet, to_check); -+ if (r) { -+ loc_network_list_unref(to_check); -+ loc_network_unref(subnet); ++ // Enumerator ++ struct loc_database_enumerator* enumerator; ++ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); ++ if (err) { ++ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); ++ exit(EXIT_FAILURE); ++ } + -+ return NULL; -+ } - } - - // Cleanup --- -2.20.1 - -From 7c6983ad52724d395446bdbd24d36b2ce22aecfd Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 15:42:54 +0000 -Subject: [PATCH 092/111] test: Add more networks to list to see the algorithm - work - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/test-network-list.c | 32 +++++++++++++++++++++++++++++++- - 1 file changed, 31 insertions(+), 1 deletion(-) - ++ // Walk through all networks ++ while (1) { ++ err = loc_database_enumerator_next_network(enumerator, &network); ++ if (err) { ++ fprintf(stderr, "Error fetching the next network: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (!network) ++ break; ++ ++ char* s = loc_network_str(network); ++ printf("Got network: %s\n", s); ++ free(s); ++ } ++ ++ // Free the enumerator ++ loc_database_enumerator_unref(enumerator); ++ + // Close the database + loc_database_unref(db); + loc_unref(ctx); diff --git a/src/test-network-list.c b/src/test-network-list.c -index 3061d63..8253fc7 100644 ---- a/src/test-network-list.c +new file mode 100644 +index 0000000..6f32ff7 +--- /dev/null +++ b/src/test-network-list.c -@@ -58,6 +58,20 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - +@@ -0,0 +1,183 @@ ++/* ++ libloc - A library to determine the location of someone on the Internet ++ ++ Copyright (C) 2017 IPFire Development Team info@ipfire.org ++ ++ This program is free software; you can redistribute it and/or modify ++ it under the terms of the GNU General Public License as published by ++ the Free Software Foundation; either version 2 of the License, or ++ (at your option) any later version. ++ ++ This program is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ GNU General Public License for more details. ++*/ ++ ++#include <errno.h> ++#include <stdio.h> ++#include <stddef.h> ++#include <stdlib.h> ++#include <string.h> ++#include <syslog.h> ++ ++#include <loc/libloc.h> ++#include <loc/network.h> ++#include <loc/network-list.h> ++ ++int main(int argc, char** argv) { ++ int err; ++ ++ struct loc_ctx* ctx; ++ err = loc_new(&ctx); ++ if (err < 0) ++ exit(EXIT_FAILURE); ++ ++ // Enable debug logging ++ loc_set_log_priority(ctx, LOG_DEBUG); ++ ++ // Create a network ++ struct loc_network* network1; ++ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32"); ++ if (err) { ++ fprintf(stderr, "Could not create the network1\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ struct loc_network* subnet1; ++ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet1\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ struct loc_network* subnet2; ++ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet2\n"); ++ exit(EXIT_FAILURE); ++ } ++ + struct loc_network* subnet3; + err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48"); + if (err) { @@ -9786,325 +3434,117 @@ index 3061d63..8253fc7 100644 + exit(EXIT_FAILURE); + } + - // Make a list with both subnets - struct loc_network_list* subnets; - err = loc_network_list_new(ctx, &subnets); -@@ -89,8 +103,24 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Add the fourth one next -+ err = loc_network_list_push(subnets, subnet4); -+ if (err) { -+ fprintf(stderr, "Could not add subnet4 to subnets list\n"); ++ struct loc_network* subnet5; ++ err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet5\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ struct loc_network* subnet6; ++ err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48"); ++ if (err) { ++ fprintf(stderr, "Could not create the subnet6\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Make a list with both subnets ++ struct loc_network_list* subnets; ++ err = loc_network_list_new(ctx, &subnets); ++ if (err) { ++ fprintf(stderr, "Could not create subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ size_t size = loc_network_list_size(subnets); ++ if (size > 0) { ++ fprintf(stderr, "The list is not empty: %zu\n", size); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_list_push(subnets, subnet1); ++ if (err) { ++ fprintf(stderr, "Could not add subnet1 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (loc_network_list_empty(subnets)) { ++ fprintf(stderr, "The subnets list reports that it is empty\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_list_push(subnets, subnet2); ++ if (err) { ++ fprintf(stderr, "Could not add subnet2 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Add the fourth one next ++ err = loc_network_list_push(subnets, subnet4); ++ if (err) { ++ fprintf(stderr, "Could not add subnet4 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Add the third one ++ err = loc_network_list_push(subnets, subnet3); ++ if (err) { ++ fprintf(stderr, "Could not add subnet3 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Add more subnets ++ err = loc_network_list_push(subnets, subnet5); ++ if (err) { ++ fprintf(stderr, "Could not add subnet5 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_list_push(subnets, subnet6); ++ if (err) { ++ fprintf(stderr, "Could not add subnet6 to subnets list\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ loc_network_list_dump(subnets); ++ ++ size = loc_network_list_size(subnets); ++ if (size != 6) { ++ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Exclude subnet1 from network1 ++ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1); ++ if (!excluded) { ++ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n"); + exit(EXIT_FAILURE); + } + -+ // Add the third one -+ err = loc_network_list_push(subnets, subnet3); -+ if (err) { -+ fprintf(stderr, "Could not add subnet3 to subnets list\n"); ++ loc_network_list_dump(excluded); ++ ++ // Exclude all subnets from network1 ++ excluded = loc_network_exclude_list(network1, subnets); ++ if (!excluded) { ++ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n"); + exit(EXIT_FAILURE); + } + -+ loc_network_list_dump(subnets); ++ loc_network_list_dump(excluded); + - size = loc_network_list_size(subnets); -- if (size != 2) { -+ if (size != 4) { - fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); - exit(EXIT_FAILURE); - } --- -2.20.1 - -From da101d55e66ebaeeed8b0b16829a2022a1af0678 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 15:48:55 +0000 -Subject: [PATCH 093/111] Drop loc_network_eq in favour of loc_network_cmp - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 - - src/loc/network.h | 1 - - src/network.c | 24 ++++-------------------- - src/test-network.c | 8 ++++---- - 4 files changed, 8 insertions(+), 26 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index 4b0ce45..d8e8f14 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -107,7 +107,6 @@ global: - # Network - loc_network_address_family; - loc_network_cmp; -- loc_network_eq; - loc_network_exclude; - loc_network_exclude_list; - loc_network_format_first_address; -diff --git a/src/loc/network.h b/src/loc/network.h -index 8ab1562..d5d0ccd 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_cmp(struct loc_network* self, struct loc_network* other); --int loc_network_eq(struct loc_network* self, struct loc_network* other); - int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); -diff --git a/src/network.c b/src/network.c -index 7ab22f8..503bf3d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -463,22 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth - return 0; - } - --LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) { -- // Family must be the same -- if (self->family != other->family) -- return 0; -- -- // The start address must be the same -- if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) -- return 0; -- -- // The prefix length must be the same -- if (self->prefix != other->prefix) -- return 0; -- -- return 1; --} -- - LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { - // Families must match - if (self->family != other->family) -@@ -596,12 +580,12 @@ static int __loc_network_exclude(struct loc_network* network, - if (r) - goto ERROR; - -- if (loc_network_eq(other, subnet1)) { -+ if (loc_network_cmp(other, subnet1) == 0) { - r = loc_network_list_push(list, subnet2); - if (r) - goto ERROR; - -- } else if (loc_network_eq(other, subnet2)) { -+ } else if (loc_network_cmp(other, subnet2) == 0) { - r = loc_network_list_push(list, subnet1); - if (r) - goto ERROR; -@@ -657,7 +641,7 @@ static int __loc_network_exclude_to_list(struct loc_network* self, - } - - // We cannot perform this operation if both networks equal -- if (loc_network_eq(self, other)) { -+ if (loc_network_cmp(self, other) == 0) { - DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); - - return 1; -@@ -745,7 +729,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - subnet = loc_network_list_get(list, i); - - // Drop this subnet if is is already in list -- if (loc_network_eq(subnet_to_check, subnet)) { -+ if (loc_network_cmp(subnet_to_check, subnet) == 0) { - passed = 0; - loc_network_unref(subnet); - break; -diff --git a/src/test-network.c b/src/test-network.c -index 8a6763c..f4cf97b 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -125,14 +125,14 @@ int main(int argc, char** argv) { - #endif - - // Check equals function -- err = loc_network_eq(network1, network1); -- if (!err) { -+ err = loc_network_cmp(network1, network1); -+ if (err) { - fprintf(stderr, "Network is not equal with itself\n"); - exit(EXIT_FAILURE); - } - -- err = loc_network_eq(network1, network2); -- if (err) { -+ err = loc_network_cmp(network1, network2); -+ if (!err) { - fprintf(stderr, "Networks equal unexpectedly\n"); - exit(EXIT_FAILURE); - } --- -2.20.1 - -From 61a6f6e4bf4493236d796e94f3d721bcf5ba68aa Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 15:50:39 +0000 -Subject: [PATCH 094/111] Drop loc_network_gt which is now unused - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/libloc.sym | 1 - - src/loc/network.h | 1 - - src/network.c | 27 --------------------------- - 3 files changed, 29 deletions(-) - -diff --git a/src/libloc.sym b/src/libloc.sym -index d8e8f14..cb5e8ef 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -115,7 +115,6 @@ global: - loc_network_get_country_code; - loc_network_get_first_address; - loc_network_get_last_address; -- loc_network_gt; - loc_network_has_flag; - loc_network_is_subnet; - loc_network_match_asn; -diff --git a/src/loc/network.h b/src/loc/network.h -index d5d0ccd..af3dafd 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -59,7 +59,6 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - - int loc_network_cmp(struct loc_network* self, struct loc_network* other); --int loc_network_gt(struct loc_network* self, struct loc_network* other); - int loc_network_overlaps(struct loc_network* self, struct loc_network* other); - int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); - int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); -diff --git a/src/network.c b/src/network.c -index 503bf3d..ac478d5 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -463,33 +463,6 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth - return 0; - } - --LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) { -- // Families must match -- if (self->family != other->family) -- return -1; -- -- int r = in6_addr_cmp(&self->first_address, &other->first_address); -- -- switch (r) { -- // Smaller -- case -1: -- return 0; -- -- // Larger -- case 1: -- return 1; -- -- default: -- break; -- } -- -- if (self->prefix > other->prefix) -- return 1; -- -- // Dunno -- return 0; --} -- - LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { - if (loc_network_match_address(self, &other->first_address) == 0) - return 1; --- -2.20.1 - -From fc692a58d9f4ca958f88cfa202250c572a0af6ea Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 16:50:17 +0000 -Subject: [PATCH 095/111] network: Adjust return codes of - loc_network_match_address and add test - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/database.c | 3 +-- - src/libloc.sym | 1 + - src/network.c | 16 +++++++++------- - src/test-network.c | 14 ++++++++++++++ - 4 files changed, 25 insertions(+), 9 deletions(-) - -diff --git a/src/database.c b/src/database.c -index 914ed3e..1871b74 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -776,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru - } - - // Check if the given IP address is inside the network -- r = loc_network_match_address(*network, address); -- if (r) { -+ if (!loc_network_match_address(*network, address)) { - DEBUG(db->ctx, "Searched address is not part of the network\n"); - - loc_network_unref(*network); -diff --git a/src/libloc.sym b/src/libloc.sym -index cb5e8ef..ee333f1 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -117,6 +117,7 @@ global: - loc_network_get_last_address; - loc_network_has_flag; - loc_network_is_subnet; -+ loc_network_match_address; - loc_network_match_asn; - loc_network_match_country_code; - loc_network_match_flag; -diff --git a/src/network.c b/src/network.c -index ac478d5..febab95 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -374,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { - LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) { - // Address must be larger than the start address - if (in6_addr_cmp(&network->first_address, address) > 0) -- return 1; -+ return 0; - - // Address must be smaller than the last address - if (in6_addr_cmp(&network->last_address, address) < 0) -- return 1; -+ return 0; - - // The address is inside this network -- return 0; -+ return 1; - } - - LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) { -@@ -464,16 +464,18 @@ LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* oth - } - - LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { -- if (loc_network_match_address(self, &other->first_address) == 0) -+ // Either of the start addresses must be in the other subnet -+ if (loc_network_match_address(self, &other->first_address)) - return 1; - -- if (loc_network_match_address(self, &other->last_address) == 0) -+ if (loc_network_match_address(other, &self->first_address)) - return 1; - -- if (loc_network_match_address(other, &self->first_address) == 0) -+ // Or either of the end addresses is in the other subnet -+ if (loc_network_match_address(self, &other->last_address)) - return 1; - -- if (loc_network_match_address(other, &self->last_address) == 0) -+ if (loc_network_match_address(other, &self->last_address)) - return 1; - - return 0; ++ if (excluded) ++ loc_network_list_unref(excluded); ++ ++ loc_network_list_unref(subnets); ++ loc_network_unref(network1); ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); ++ loc_unref(ctx); ++ ++ return EXIT_SUCCESS; ++} diff --git a/src/test-network.c b/src/test-network.c -index f4cf97b..339743d 100644 +index d38f13d..dde13f1 100644 --- a/src/test-network.c +++ b/src/test-network.c @@ -14,6 +14,7 @@ @@ -10115,280 +3555,128 @@ index f4cf97b..339743d 100644 #include <errno.h> #include <stdio.h> #include <stddef.h> -@@ -46,6 +47,13 @@ int main(int argc, char** argv) { - } - #endif +@@ -37,12 +38,21 @@ int main(int argc, char** argv) { + // Enable debug logging + loc_set_log_priority(ctx, LOG_DEBUG);
++#if 0 + struct loc_network_tree* tree; + err = loc_network_tree_new(ctx, &tree); + if (err) { + fprintf(stderr, "Could not create the network tree\n"); + exit(EXIT_FAILURE); + } ++#endif ++ + struct in6_addr address; + err = inet_pton(AF_INET6, "2001:db8::1", &address); + if (err != 1) { + fprintf(stderr, "Could not parse IP address\n"); + exit(EXIT_FAILURE); + } -+ + // Create a network struct loc_network* network1; - err = loc_network_new_from_string(ctx, &network1, "2001:db8::1/32"); -@@ -92,6 +100,12 @@ int main(int argc, char** argv) { +@@ -58,12 +68,14 @@ int main(int argc, char** argv) { exit(EXIT_FAILURE); }
-+ err = loc_network_match_address(network1, &address); -+ if (!err) { -+ fprintf(stderr, "Network1 does not match address\n"); -+ exit(EXIT_FAILURE); -+ } -+ - struct loc_network* network2; - err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48"); ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network1); if (err) { --- -2.20.1 - -From 06177d8c6004bf8b54322d92926152a7656f9c2a Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 16:57:28 +0000 -Subject: [PATCH 096/111] network-list: Use binary search to find if a network - is a subnet - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 13 ++++++------- - 1 file changed, 6 insertions(+), 7 deletions(-) - -diff --git a/src/network.c b/src/network.c -index febab95..394bafc 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -697,19 +697,18 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - while (!loc_network_list_empty(to_check)) { - struct loc_network* subnet_to_check = loc_network_list_pop(to_check); - -+ // Check whether the subnet to check is part of the input list -+ if (loc_network_list_contains(list, subnet_to_check)) { -+ loc_network_unref(subnet_to_check); -+ continue; -+ } -+ - // Marks whether this subnet passed all checks - int passed = 1; - - for (unsigned int i = 0; i < loc_network_list_size(list); i++) { - subnet = loc_network_list_get(list, i); - -- // Drop this subnet if is is already in list -- if (loc_network_cmp(subnet_to_check, subnet) == 0) { -- passed = 0; -- loc_network_unref(subnet); -- break; -- } -- - // Drop this subnet if is a subnet of another subnet - if (loc_network_is_subnet(subnet_to_check, subnet)) { - passed = 0; --- -2.20.1 - -From 77e6d5379ea0de3264be5b8918d620d16e6bcbd3 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Tue, 24 Nov 2020 19:39:35 +0000 -Subject: [PATCH 097/111] network-list: Check last element before doing binary - search - -This is helpful because very often we walk through a list in -order and are most interested in the last element. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network-list.c | 24 ++++++++++++++++++++++-- - 1 file changed, 22 insertions(+), 2 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index 7e8b5f3..c86ad07 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -137,8 +137,26 @@ static off_t loc_network_list_find(struct loc_network_list* list, - struct loc_network* network, int* found) { - off_t lo = 0; - off_t hi = list->size - 1; -+ int result; - -- *found = 0; -+ // Since we are working on an ordered list, there is often a good chance that -+ // the network we are looking for is at the end or has to go to the end. -+ if (hi >= 0) { -+ result = loc_network_cmp(network, list->elements[hi]); -+ -+ // Match, so we are done -+ if (result == 0) { -+ *found = 1; -+ -+ return hi; -+ -+ // This needs to be added after the last one -+ } else if (result > 0) { -+ *found = 0; -+ -+ return hi + 1; -+ } -+ } - - #ifdef ENABLE_DEBUG - // Save start time -@@ -151,7 +169,7 @@ static off_t loc_network_list_find(struct loc_network_list* list, - i = (lo + hi) / 2; - - // Check if this is a match -- int result = loc_network_cmp(network, list->elements[i]); -+ result = loc_network_cmp(network, list->elements[i]); - - if (result == 0) { - *found = 1; -@@ -173,6 +191,8 @@ static off_t loc_network_list_find(struct loc_network_list* list, - hi = i - 1; + fprintf(stderr, "Could not add network to the tree\n"); + exit(EXIT_FAILURE); } ++#endif
-+ *found = 0; -+ - #ifdef ENABLE_DEBUG - clock_t end = clock(); - --- -2.20.1 - -From d42f34ce9b0aa2382ae9b852882c62b34d367cfe Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 14:41:39 +0000 -Subject: [PATCH 098/111] network-list: Set elements pointer to NULL so that we - know it is empty - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network-list.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/network-list.c b/src/network-list.c -index c86ad07..2c4edb3 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -104,6 +104,7 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { - loc_network_unref(list->elements[i]); - - free(list->elements); -+ list->elements = NULL; - list->elements_size = 0; - - list->size = 0; --- -2.20.1 - -From 673e03f7bfc807248a769cb00ec80f0fa2af7a25 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 14:42:26 +0000 -Subject: [PATCH 099/111] network-list: Do not half list when popping the first - element - -The list was unfortunately halved in size every time an element -was taken from it, which was great for performance, but shortened -the result substantially. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network-list.c | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/src/network-list.c b/src/network-list.c -index 2c4edb3..f4a9d05 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -261,10 +261,13 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis - struct loc_network* network = list->elements[0]; - - // Move all elements to the top of the stack -- for (unsigned int i = 0; i < --list->size; i++) { -+ for (unsigned int i = 0; i < list->size - 1; i++) { - list->elements[i] = list->elements[i+1]; + // Check if the first and last addresses are correct + char* string = loc_network_format_first_address(network1); +@@ -88,6 +100,12 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); }
-+ // The list is shorter now -+ --list->size; ++ err = loc_network_match_address(network1, &address); ++ if (!err) { ++ fprintf(stderr, "Network1 does not match address\n"); ++ exit(EXIT_FAILURE); ++ } + - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); - - return network; --- -2.20.1 - -From 9446c7538ab8b27486d74f6dc0dac8bb5c1bbe92 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 14:43:58 +0000 -Subject: [PATCH 100/111] network-list: Show index when listing networks - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network-list.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/network-list.c b/src/network-list.c -index f4a9d05..cf9459d 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -119,7 +119,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { + struct loc_network* network2; + err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48"); + if (err) { +@@ -101,6 +119,7 @@ int main(int argc, char** argv) { + exit(EXIT_FAILURE); + }
- s = loc_network_str(network); ++#if 0 + // Adding network to the tree + err = loc_network_tree_add_network(tree, network2); + if (err) { +@@ -117,20 +136,84 @@ int main(int argc, char** argv) {
-- INFO(list->ctx, "%s\n", s); -+ INFO(list->ctx, "%4d: %s\n", i, s); - free(s); + size_t nodes = loc_network_tree_count_nodes(tree); + printf("The tree has %zu nodes\n", nodes); ++#endif ++ ++ // Check equals function ++ err = loc_network_cmp(network1, network1); ++ if (err) { ++ fprintf(stderr, "Network is not equal with itself\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ err = loc_network_cmp(network1, network2); ++ if (!err) { ++ fprintf(stderr, "Networks equal unexpectedly\n"); ++ exit(EXIT_FAILURE); ++ } + + // Check subnet function +- err = loc_network_is_subnet_of(network1, network2); +- if (err != 0) { ++ err = loc_network_is_subnet(network1, network2); ++ if (!err) { + fprintf(stderr, "Subnet check 1 failed: %d\n", err); + exit(EXIT_FAILURE); } - } --- -2.20.1 - -From dc31666be416dbb3bd2bc04127104a3f75b42d5c Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 14:44:23 +0000 -Subject: [PATCH 101/111] network-list: Remove useless comment - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network-list.c | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index cf9459d..bd3d64e 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -132,8 +132,6 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis - return loc_network_ref(list->elements[index]); - }
--//MOVE FUNCTION GOES HERE -- - static off_t loc_network_list_find(struct loc_network_list* list, - struct loc_network* network, int* found) { - off_t lo = 0; --- -2.20.1 - -From 82fa4c92c88cc172953198637bccc375c4d25d20 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 14:44:56 +0000 -Subject: [PATCH 102/111] networks: Add tests for overlaps function - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/test-network.c | 10 ++++++++++ - 1 file changed, 10 insertions(+) - -diff --git a/src/test-network.c b/src/test-network.c -index 339743d..dde13f1 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -192,6 +192,16 @@ int main(int argc, char** argv) { +- err = loc_network_is_subnet_of(network2, network1); +- if (err != 1) { ++ err = loc_network_is_subnet(network2, network1); ++ if (err) { + fprintf(stderr, "Subnet check 2 failed: %d\n", err); exit(EXIT_FAILURE); }
++ // Make subnets ++ struct loc_network* subnet1 = NULL; ++ struct loc_network* subnet2 = NULL; ++ ++ err = loc_network_subnets(network1, &subnet1, &subnet2); ++ if (err || !subnet1 || !subnet2) { ++ fprintf(stderr, "Could not find subnets of network: %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ char* s = loc_network_str(subnet1); ++ printf("Received subnet1 = %s\n", s); ++ free(s); ++ ++ s = loc_network_str(subnet2); ++ printf("Received subnet2 = %s\n", s); ++ free(s); ++ ++ if (!loc_network_is_subnet(network1, subnet1)) { ++ fprintf(stderr, "Subnet1 is not a subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (!loc_network_is_subnet(network1, subnet2)) { ++ fprintf(stderr, "Subnet2 is not a subnet\n"); ++ exit(EXIT_FAILURE); ++ } ++ + if (!loc_network_overlaps(network1, subnet1)) { + fprintf(stderr, "Network1 does not seem to contain subnet1\n"); + exit(EXIT_FAILURE); @@ -10399,340 +3687,84 @@ index 339743d..dde13f1 100644 + exit(EXIT_FAILURE); + } + - loc_network_unref(subnet1); - loc_network_unref(subnet2); - --- -2.20.1 - -From 0a39d5499ae0c343a6600ea30de7d95e384ef911 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:11:21 +0000 -Subject: [PATCH 103/111] networks: Remove comparing family - -Everything is encoded in IPv6 anyways... - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 17 ----------------- - 1 file changed, 17 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 394bafc..66b460f 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -442,12 +442,6 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - } - - LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { -- // Compare family -- if (self->family > other->family) -- return 1; -- else if (self->family < other->family) -- return -1; -- - // Compare address - int r = in6_addr_cmp(&self->first_address, &other->first_address); - if (r) -@@ -482,10 +476,6 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network - } - - LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { -- // Check family -- if (self->family != other->family) -- return 0; -- - // The prefix must be smaller (this avoids the more complex comparisons later) - if (self->prefix > other->prefix) - return 0; -@@ -601,13 +591,6 @@ ERROR: - - static int __loc_network_exclude_to_list(struct loc_network* self, - struct loc_network* other, struct loc_network_list* list) { -- // Family must match -- if (self->family != other->family) { -- DEBUG(self->ctx, "Family mismatch\n"); -- -- return 1; -- } -- - // Other must be a subnet of self - if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); --- -2.20.1 - -From abf559267696061a0c9723d8f8e09c9d1aa8fb75 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:13:08 +0000 -Subject: [PATCH 104/111] network: Do not execute with an error when the - excluded result will be empty - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 66b460f..9aa802f 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -595,14 +595,16 @@ static int __loc_network_exclude_to_list(struct loc_network* self, - if (!loc_network_is_subnet(self, other)) { - DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); - -- return 1; -+ // Exit silently -+ return 0; - } - - // We cannot perform this operation if both networks equal - if (loc_network_cmp(self, other) == 0) { - DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); - -- return 1; -+ // Exit silently -+ return 0; - } - - return __loc_network_exclude(self, other, list); --- -2.20.1 - -From 4fc034e20fdc36267f6ba18e27776fed747fe983 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:14:31 +0000 -Subject: [PATCH 105/111] network: Add more excluded networks straight to the - to_check list - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 6 +----- - 1 file changed, 1 insertion(+), 5 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 9aa802f..19c387d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -705,11 +705,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - if (loc_network_overlaps(subnet_to_check, subnet)) { - passed = 0; - -- struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet); -- if (excluded) { -- loc_network_list_merge(to_check, excluded); -- loc_network_list_unref(excluded); -- } -+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); - - loc_network_unref(subnet); - break; --- -2.20.1 - -From 4de8ff8e5435225d3375a168054490d2a66b1baf Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:15:33 +0000 -Subject: [PATCH 106/111] network: Call subnet function with the correct order - of arguments - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 19c387d..a96ce6d 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -695,14 +695,14 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - subnet = loc_network_list_get(list, i); - - // Drop this subnet if is a subnet of another subnet -- if (loc_network_is_subnet(subnet_to_check, subnet)) { -+ if (loc_network_is_subnet(subnet, subnet_to_check)) { - passed = 0; - loc_network_unref(subnet); - break; - } - - // Break it down if it overlaps -- if (loc_network_overlaps(subnet_to_check, subnet)) { -+ if (loc_network_overlaps(subnet, subnet_to_check)) { - passed = 0; - - __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); --- -2.20.1 - -From 058e7800e56150bcf0fb8dceaae6fbab1ed239e4 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:16:06 +0000 -Subject: [PATCH 107/111] network: Massively improve performance on exclude - -When we check the result for any overlaps, we can cut this short -by walking through both lists from start to end and remember the -last network that we checked. - -The next one will by definition be strictly greater and therefore -we do not need to check anything before this any more. - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 19 +++++++++++++++++-- - 1 file changed, 17 insertions(+), 2 deletions(-) - -diff --git a/src/network.c b/src/network.c -index a96ce6d..10fa997 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -679,8 +679,10 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - return NULL; - } - -+ off_t smallest_subnet = 0; ++ loc_network_unref(subnet1); ++ loc_network_unref(subnet2); + - while (!loc_network_list_empty(to_check)) { -- struct loc_network* subnet_to_check = loc_network_list_pop(to_check); -+ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check); - - // Check whether the subnet to check is part of the input list - if (loc_network_list_contains(list, subnet_to_check)) { -@@ -691,7 +693,7 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - // Marks whether this subnet passed all checks - int passed = 1; - -- for (unsigned int i = 0; i < loc_network_list_size(list); i++) { -+ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) { - subnet = loc_network_list_get(list, i); - - // Drop this subnet if is a subnet of another subnet -@@ -711,6 +713,19 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - break; - } - -+ // If the subnet is strictly greater, we do not need to continue the search -+ r = loc_network_cmp(subnet, subnet_to_check); -+ if (r > 0) { -+ loc_network_unref(subnet); -+ break; ++ struct loc_network_list* excluded = loc_network_exclude(network1, network2); ++ if (!excluded) { ++ fprintf(stderr, "Could not create excluded list\n"); ++ exit(EXIT_FAILURE); ++ } + -+ // If it is strictly smaller, we can continue the search from here next -+ // time because all networks that are to be checked can only be larger -+ // than this one. -+ } else if (r < 0) { -+ smallest_subnet = i; -+ } ++ loc_network_list_dump(excluded); ++ loc_network_list_unref(excluded); + - loc_network_unref(subnet); - } - --- -2.20.1 - -From 9caf6cf5f0fd5eb56ad1678910dbc8017f48863c Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:17:21 +0000 -Subject: [PATCH 108/111] network: Remove deprecated sort call - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network.c | 3 --- - 1 file changed, 3 deletions(-) - -diff --git a/src/network.c b/src/network.c -index 10fa997..4d7ac79 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -738,9 +738,6 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list( - - loc_network_list_unref(to_check); + // Create a database + struct loc_writer* writer; + err = loc_writer_new(ctx, &writer, NULL, NULL); +@@ -160,6 +243,28 @@ int main(int argc, char** argv) { + // Set ASN + loc_network_set_asn(network4, 1024);
-- // Sort the result -- loc_network_list_sort(subnets); -- - return subnets; - } ++ // Try adding an invalid network ++ struct loc_network* network; ++ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Try adding a single address ++ err = loc_writer_add_network(writer, &network, "2001:db8::"); ++ if (err) { ++ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); ++ exit(EXIT_FAILURE); ++ } ++ ++ // Try adding localhost ++ err = loc_writer_add_network(writer, &network, "::1/128"); ++ if (err != -EINVAL) { ++ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); ++ exit(EXIT_FAILURE); ++ } ++ + FILE* f = tmpfile(); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); +@@ -177,7 +282,10 @@ int main(int argc, char** argv) { + loc_network_unref(network2); + loc_network_unref(network3); + loc_network_unref(network4); ++ ++#if 0 + loc_network_tree_unref(tree); ++#endif
--- -2.20.1 - -From 8ebf1d3912020d5c0ed98bb32a0640952bd6447c Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:17:42 +0000 -Subject: [PATCH 109/111] network-list: Include network header - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/loc/network-list.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -index 21c7402..bee21c4 100644 ---- a/src/loc/network-list.h -+++ b/src/loc/network-list.h -@@ -17,6 +17,8 @@ - #ifndef LIBLOC_NETWORK_LIST_H - #define LIBLOC_NETWORK_LIST_H + // And open it again from disk + struct loc_database* db; +diff --git a/src/writer.c b/src/writer.c +index 5939cff..2f09b56 100644 +--- a/src/writer.c ++++ b/src/writer.c +@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) { + EVP_PKEY_free(writer->private_key2);
-+#include <loc/network.h> + // Unref all AS +- for (unsigned int i = 0; i < writer->as_count; i++) { +- loc_as_unref(writer->as[i]); ++ if (writer->as) { ++ for (unsigned int i = 0; i < writer->as_count; i++) { ++ loc_as_unref(writer->as[i]); ++ } ++ free(writer->as); ++ } + - struct loc_network_list; - int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); - struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); --- -2.20.1 - -From fff093b6827a31ff84a6d5150bfede140b696d25 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 15:24:43 +0000 -Subject: [PATCH 110/111] network-list: Use clear function to tidy up - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - src/network-list.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/network-list.c b/src/network-list.c -index bd3d64e..f3458b4 100644 ---- a/src/network-list.c -+++ b/src/network-list.c -@@ -70,8 +70,8 @@ LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list - static void loc_network_list_free(struct loc_network_list* list) { - DEBUG(list->ctx, "Releasing network list at %p\n", list); - -- for (unsigned int i = 0; i < list->size; i++) -- loc_network_unref(list->elements[i]); -+ // Remove all content -+ loc_network_list_clear(list); ++ // Unref all countries ++ if (writer->countries) { ++ for (unsigned int i = 0; i < writer->countries_count; i++) { ++ loc_country_unref(writer->countries[i]); ++ } ++ free(writer->countries); + }
- loc_unref(list->ctx); - free(list); --- -2.20.1 - -From adbec5c0317a1c007a897fe3f63f0f86ff448124 Mon Sep 17 00:00:00 2001 -From: Michael Tremer michael.tremer@ipfire.org -Date: Wed, 25 Nov 2020 20:00:46 +0000 -Subject: [PATCH 111/111] configure: Bump version to 0.9.5 - -Signed-off-by: Michael Tremer michael.tremer@ipfire.org ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 2364dfd..012d8ca 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1,6 +1,6 @@ - AC_PREREQ(2.60) - AC_INIT([libloc], -- [0.9.4], -+ [0.9.5], - [location@lists.ipfire.org], - [libloc], - [https://location.ipfire.org/]) --- -2.20.1 - + // Release network tree
hooks/post-receive -- IPFire 2.x development tree
-
Michael Tremer