From mboxrd@z Thu Jan 1 00:00:00 1970 From: Peter =?utf-8?q?M=C3=BCller?= To: development@lists.ipfire.org Subject: [PATCH] libloc: update to 0.9.5 and backport fix for #12554 Date: Wed, 23 Dec 2020 15:03:32 +0100 Message-ID: MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============2053765717606856132==" List-Id: --===============2053765717606856132== Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable This patch updates libloc to 0.9.5, deletes the upstream patchset from version 0.9.4, and includes a latest upstream patch to backport a fix for #12554. Cc: Michael Tremer Signed-off-by: Peter M=C3=BCller --- lfs/libloc | 14 +- src/patches/libloc-0.9.4-upstream.patch | 3850 ----------------- ...9.5-location-Fix-list-networks-by-as.patch | 27 + 3 files changed, 34 insertions(+), 3857 deletions(-) delete mode 100644 src/patches/libloc-0.9.4-upstream.patch create mode 100644 src/patches/libloc-0.9.5-location-Fix-list-networks-by-as= .patch diff --git a/lfs/libloc b/lfs/libloc index 010ddf1cb..279840535 100644 --- a/lfs/libloc +++ b/lfs/libloc @@ -24,8 +24,8 @@ =20 include Config =20 -VER =3D 0.9.4 -DB_DATE =3D 2020-09-21 +VER =3D 0.9.5 +DB_DATE =3D 2020-12-23 =20 THISAPP =3D libloc-$(VER) DL_FILE =3D $(THISAPP).tar.gz @@ -40,11 +40,11 @@ TARGET =3D $(DIR_INFO)/$(THISAPP) objects =3D $(DL_FILE) \ location-$(DB_DATE).db.xz =20 -$(DL_FILE) =3D https://source.ipfire.org/releases/libloc/= /$(DL_FILE) +$(DL_FILE) =3D https://source.ipfire.org/releases/libloc/= $(DL_FILE) location-$(DB_DATE).db.xz =3D https://location.ipfire.org/databases/1/ar= chive/location-$(DB_DATE).db.xz =20 -$(DL_FILE)_MD5 =3D 82770e9eba20f636c96e6fa42ff234b5 -location-$(DB_DATE).db.xz_MD5 =3D fa3069bf31170629d638317e283913c0 +$(DL_FILE)_MD5 =3D 41d8dc3fb4e498db958b7696cadd61f5 +location-$(DB_DATE).db.xz_MD5 =3D 3a71931555623fc3665f280c8e5c292a =20 install : $(TARGET) =20 @@ -78,8 +78,8 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) @$(PREBUILD) @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE) =20 - # Import changes from upstream - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.4-upstream= .patch + # Import recent patches from upstream + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.5-location= -Fix-list-networks-by-as.patch =20 # Add patch for i586 to disable strong stack protector. ifeq "$(BUILD_ARCH)" "i586" diff --git a/src/patches/libloc-0.9.4-upstream.patch b/src/patches/libloc-0.9= .4-upstream.patch deleted file mode 100644 index a6ec1066b..000000000 --- a/src/patches/libloc-0.9.4-upstream.patch +++ /dev/null @@ -1,3850 +0,0 @@ -diff --git a/Makefile.am b/Makefile.am -index a0431a6..dc594f8 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -91,11 +91,14 @@ EXTRA_DIST +=3D \ - pkginclude_HEADERS =3D \ - src/loc/libloc.h \ - src/loc/as.h \ -+ src/loc/as-list.h \ - src/loc/compat.h \ - src/loc/country.h \ -+ src/loc/country-list.h \ - src/loc/database.h \ - src/loc/format.h \ - src/loc/network.h \ -+ src/loc/network-list.h \ - src/loc/private.h \ - src/loc/stringpool.h \ - src/loc/resolv.h \ -@@ -107,9 +110,12 @@ lib_LTLIBRARIES =3D \ - src_libloc_la_SOURCES =3D \ - src/libloc.c \ - src/as.c \ -+ src/as-list.c \ - src/country.c \ -+ src/country-list.c \ - src/database.c \ - src/network.c \ -+ src/network-list.c \ - src/resolv.c \ - src/stringpool.c \ - src/writer.c -@@ -312,6 +318,7 @@ check_PROGRAMS =3D \ - src/test-database \ - src/test-as \ - src/test-network \ -+ src/test-network-list \ - src/test-country \ - src/test-signature -=20 -@@ -351,6 +358,15 @@ src_test_network_CFLAGS =3D \ - src_test_network_LDADD =3D \ - src/libloc.la -=20 -+src_test_network_list_SOURCES =3D \ -+ src/test-network-list.c -+ -+src_test_network_list_CFLAGS =3D \ -+ $(TESTS_CFLAGS) -+ -+src_test_network_list_LDADD =3D \ -+ src/libloc.la -+ - src_test_stringpool_SOURCES =3D \ - src/test-stringpool.c -=20 -@@ -390,7 +406,7 @@ MANPAGES_XML =3D $(patsubst %.txt,%.xml,$(MANPAGES_TXT)) - .PHONY: man - man: $(MANPAGES) $(MANPAGES_HTML) -=20 --if ENABLE_MANPAGES -+if ENABLE_MAN_PAGES - man_MANS =3D \ - $(MANPAGES) - endif -diff --git a/configure.ac b/configure.ac -index 2364dfd..9eb9012 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1,6 +1,6 @@ - AC_PREREQ(2.60) - AC_INIT([libloc], -- [0.9.4], -+ [0.9.5], - [location(a)lists.ipfire.org], - [libloc], - [https://location.ipfire.org/]) -@@ -43,16 +43,16 @@ AC_PROG_MKDIR_P -=20 - # - man -------------------------------------------------------------------= ----- -=20 --have_manpages=3Dno --AC_ARG_ENABLE(manpages, AS_HELP_STRING([--disable-man-pages], -+have_man_pages=3Dno -+AC_ARG_ENABLE(man_pages, AS_HELP_STRING([--disable-man-pages], - [do not install man pages])) --AS_IF([test "x$enable_manpages" !=3D xno], [have_manpages=3Dyes]) --AM_CONDITIONAL(ENABLE_MANPAGES, [test "x$have_manpages" =3D "xyes"]) -+AS_IF([test "x$enable_man_pages" !=3D xno], [have_man_pages=3Dyes]) -+AM_CONDITIONAL(ENABLE_MAN_PAGES, [test "x$have_man_pages" =3D "xyes"]) -=20 - AC_PATH_PROG([XSLTPROC], [xsltproc]) -=20 - AC_CHECK_PROGS(ASCIIDOC, [asciidoc]) --if test "${have_manpages}" =3D "yes" && test -z "${ASCIIDOC}"; then -+if test "${have_man_pages}" =3D "yes" && test -z "${ASCIIDOC}"; then - AC_MSG_ERROR([Required program 'asciidoc' not found]) - fi - # - debug -----------------------------------------------------------------= ----- -diff --git a/src/.gitignore b/src/.gitignore -index caf80b5..3ccbdb8 100644 ---- a/src/.gitignore -+++ b/src/.gitignore -@@ -10,5 +10,6 @@ test-libloc - test-database - test-country - test-network -+test-network-list - test-signature - test-stringpool -diff --git a/src/as-list.c b/src/as-list.c -new file mode 100644 -index 0000000..5acbb8a ---- /dev/null -+++ b/src/as-list.c -@@ -0,0 +1,161 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include -+#include -+ -+#include -+#include -+#include -+ -+struct loc_as_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_as** elements; -+ size_t elements_size; -+ -+ size_t size; -+}; -+ -+static int loc_as_list_grow(struct loc_as_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); -+ -+ struct loc_as** elements =3D reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; -+ -+ list->elements =3D elements; -+ list->elements_size +=3D size; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, -+ struct loc_as_list** list) { -+ struct loc_as_list* l =3D calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx =3D loc_ref(ctx); -+ l->refcount =3D 1; -+ -+ DEBUG(l->ctx, "AS list allocated at %p\n", l); -+ *list =3D l; -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_as_list_free(struct loc_as_list* list) { -+ DEBUG(list->ctx, "Releasing AS list at %p\n", list); -+ -+ loc_as_list_clear(list); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_as_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { -+ return list->size =3D=3D 0; -+} -+ -+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { -+ if (!list->elements) -+ return; -+ -+ for (unsigned int i =3D 0; i < list->size; i++) -+ loc_as_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements =3D NULL; -+ list->elements_size =3D 0; -+ -+ list->size =3D 0; -+} -+ -+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t = index) { -+ // Check index -+ if (index >=3D list->size) -+ return NULL; -+ -+ return loc_as_ref(list->elements[index]); -+} -+ -+LOC_EXPORT int loc_as_list_append( -+ struct loc_as_list* list, struct loc_as* as) { -+ if (loc_as_list_contains(list, as)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size >=3D list->elements_size) { -+ int r =3D loc_as_list_grow(list, 64); -+ if (r) -+ return r; -+ } -+ -+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); -+ -+ list->elements[list->size++] =3D loc_as_ref(as); -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as) { -+ for (unsigned int i =3D 0; i < list->size; i++) { -+ if (loc_as_cmp(as, list->elements[i]) =3D=3D 0) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number) { -+ struct loc_as* as; -+ -+ int r =3D loc_as_new(list->ctx, &as, number); -+ if (r) -+ return -1; -+ -+ r =3D loc_as_list_contains(list, as); -+ loc_as_unref(as); -+ -+ return r; -+} -diff --git a/src/as.c b/src/as.c -index e1fbb01..757bf3d 100644 ---- a/src/as.c -+++ b/src/as.c -@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as)= { - } -=20 - LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) { -- as->name =3D strdup(name); -+ if (as->name) -+ free(as->name); -+ -+ if (name) -+ as->name =3D strdup(name); -+ else -+ as->name =3D NULL; -=20 - return 0; - } -@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* = string) { - if (!string) - return 1; -=20 -+ // Cannot match anything when name is not set -+ if (!as->name) -+ return 1; -+ - // Search if string is in name - if (strcasestr(as->name, string) !=3D NULL) - return 1; -diff --git a/src/country-list.c b/src/country-list.c -new file mode 100644 -index 0000000..cc36740 ---- /dev/null -+++ b/src/country-list.c -@@ -0,0 +1,161 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include -+#include -+ -+#include -+#include -+#include -+ -+struct loc_country_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_country** elements; -+ size_t elements_size; -+ -+ size_t size; -+}; -+ -+static int loc_country_list_grow(struct loc_country_list* list, size_t size= ) { -+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); -+ -+ struct loc_country** elements =3D reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; -+ -+ list->elements =3D elements; -+ list->elements_size +=3D size; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, -+ struct loc_country_list** list) { -+ struct loc_country_list* l =3D calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx =3D loc_ref(ctx); -+ l->refcount =3D 1; -+ -+ DEBUG(l->ctx, "Country list allocated at %p\n", l); -+ *list =3D l; -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country= _list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_country_list_free(struct loc_country_list* list) { -+ DEBUG(list->ctx, "Releasing country list at %p\n", list); -+ -+ loc_country_list_clear(list); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_count= ry_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_country_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { -+ return list->size =3D=3D 0; -+} -+ -+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { -+ if (!list->elements) -+ return; -+ -+ for (unsigned int i =3D 0; i < list->size; i++) -+ loc_country_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements =3D NULL; -+ list->elements_size =3D 0; -+ -+ list->size =3D 0; -+} -+ -+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list= * list, size_t index) { -+ // Check index -+ if (index >=3D list->size) -+ return NULL; -+ -+ return loc_country_ref(list->elements[index]); -+} -+ -+LOC_EXPORT int loc_country_list_append( -+ struct loc_country_list* list, struct loc_country* country) { -+ if (loc_country_list_contains(list, country)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size >=3D list->elements_size) { -+ int r =3D loc_country_list_grow(list, 64); -+ if (r) -+ return r; -+ } -+ -+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); -+ -+ list->elements[list->size++] =3D loc_country_ref(country); -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country) { -+ for (unsigned int i =3D 0; i < list->size; i++) { -+ if (loc_country_cmp(country, list->elements[i]) =3D=3D 0) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code) { -+ struct loc_country* country; -+ -+ int r =3D loc_country_new(list->ctx, &country, code); -+ if (r) -+ return -1; -+ -+ r =3D loc_country_list_contains(list, country); -+ loc_country_unref(country); -+ -+ return r; -+} -diff --git a/src/country.c b/src/country.c -index 2ba93e6..7aac0db 100644 ---- a/src/country.c -+++ b/src/country.c -@@ -34,6 +34,9 @@ struct loc_country { - }; -=20 - LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** co= untry, const char* country_code) { -+ if (!loc_country_code_is_valid(country_code)) -+ return -EINVAL; -+ - struct loc_country* c =3D calloc(1, sizeof(*c)); - if (!c) - return -ENOMEM; -diff --git a/src/database.c b/src/database.c -index fa1dad0..4b8bf1d 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -38,8 +38,10 @@ -=20 - #include - #include -+#include - #include - #include -+#include - #include - #include - #include -@@ -99,11 +101,14 @@ struct loc_database_enumerator { -=20 - // Search string - char* string; -- char country_code[3]; -- uint32_t asn; -+ struct loc_country_list* countries; -+ struct loc_as_list* asns; - enum loc_network_flags flags; - int family; -=20 -+ // Flatten output? -+ int flatten; -+ - // Index of the AS we are looking at - unsigned int as_index; -=20 -@@ -115,6 +120,9 @@ struct loc_database_enumerator { - struct loc_node_stack network_stack[MAX_STACK_DEPTH]; - int network_stack_depth; - unsigned int* networks_visited; -+ -+ // For subnet search -+ struct loc_network_list* stack; - }; -=20 - static int loc_database_read_magic(struct loc_database* db) { -@@ -242,11 +250,11 @@ static int loc_database_read_signature(struct loc_data= base* db, - char** dst, char* src, size_t length) { - // Check for a plausible signature length - if (length > LOC_SIGNATURE_MAX_LENGTH) { -- ERROR(db->ctx, "Signature too long: %ld\n", length); -+ ERROR(db->ctx, "Signature too long: %zu\n", length); - return -EINVAL; - } -=20 -- DEBUG(db->ctx, "Reading signature of %ld bytes\n", length); -+ DEBUG(db->ctx, "Reading signature of %zu bytes\n", length); -=20 - // Allocate space - *dst =3D malloc(length); -@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* = db, FILE* f) { - } -=20 - clock_t end =3D clock(); -- DEBUG(db->ctx, "Signature checked in %.4fms\n", -+ INFO(db->ctx, "Signature checked in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); -=20 - CLEANUP: -@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database*= db, struct loc_as** as, - off_t lo =3D 0; - off_t hi =3D db->as_count - 1; -=20 -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start =3D clock(); -+#endif -=20 - while (lo <=3D hi) { - off_t i =3D (lo + hi) / 2; -@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database= * db, struct loc_as** as, - // Check if this is a match - uint32_t as_number =3D loc_as_get_number(*as); - if (as_number =3D=3D number) { -+#ifdef ENABLE_DEBUG - clock_t end =3D clock(); -=20 - // Log how fast this has been - DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif -=20 - return 0; - } -@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_datab= ase* db, struct loc_networ - return -1; - } -=20 -+#ifdef ENABLE_DEBUG - if (r =3D=3D 0) { - char* string =3D loc_network_str(*network); - DEBUG(db->ctx, "Got network %s\n", string); - free(string); - } -+#endif -=20 - return r; - } -@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_= database* db, const stru - } -=20 - // Check if the given IP address is inside the network -- r =3D loc_network_match_address(*network, address); -- if (r) { -+ if (!loc_network_match_address(*network, address)) { - DEBUG(db->ctx, "Searched address is not part of the network\n"); -=20 - loc_network_unref(*network); -@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database= * db, -=20 - *network =3D NULL; -=20 -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start =3D clock(); -+#endif -=20 - int r =3D __loc_database_lookup(db, address, network, &network_address, - db->network_nodes_v1, 0); -=20 -+#ifdef ENABLE_DEBUG - clock_t end =3D clock(); -=20 - // Log how fast this has been - DEBUG(db->ctx, "Executed network search in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif -=20 - return r; - } -@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_data= base* db, - off_t lo =3D 0; - off_t hi =3D db->countries_count - 1; -=20 -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start =3D clock(); -+#endif -=20 - while (lo <=3D hi) { - off_t i =3D (lo + hi) / 2; -@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_dat= abase* db, - int result =3D strcmp(code, cc); -=20 - if (result =3D=3D 0) { -+#ifdef ENABLE_DEBUG - clock_t end =3D clock(); -=20 - // Log how fast this has been - DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif -=20 - return 0; - } -@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_data= base* db, -=20 - // Enumerator -=20 -+static void loc_database_enumerator_free(struct loc_database_enumerator* en= umerator) { -+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -+ -+ // Release all references -+ loc_database_unref(enumerator->db); -+ loc_unref(enumerator->ctx); -+ -+ if (enumerator->string) -+ free(enumerator->string); -+ -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); -+ -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); -+ -+ // Free network search -+ free(enumerator->networks_visited); -+ -+ // Free subnet stack -+ if (enumerator->stack) -+ loc_network_list_unref(enumerator->stack); -+ -+ free(enumerator); -+} -+ - LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator**= enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode) { -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flag= s) { - struct loc_database_enumerator* e =3D calloc(1, sizeof(*e)); - if (!e) - return -ENOMEM; -@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_= database_enumerator** enum - e->mode =3D mode; - e->refcount =3D 1; -=20 -+ // Flatten output? -+ e->flatten =3D (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); -+ - // Initialise graph search -- //e->network_stack[++e->network_stack_depth] =3D 0; - e->network_stack_depth =3D 1; - e->networks_visited =3D calloc(db->network_nodes_count, sizeof(*e->network= s_visited)); -=20 -+ // Allocate stack -+ int r =3D loc_network_list_new(e->ctx, &e->stack); -+ if (r) { -+ loc_database_enumerator_free(e); -+ return r; -+ } -+ - DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); -=20 - *enumerator =3D e; -@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_databas= e_enumerator_ref(struct lo - return enumerator; - } -=20 --static void loc_database_enumerator_free(struct loc_database_enumerator* en= umerator) { -- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -- -- // Release all references -- loc_database_unref(enumerator->db); -- loc_unref(enumerator->ctx); -- -- if (enumerator->string) -- free(enumerator->string); -- -- // Free network search -- free(enumerator->networks_visited); -- -- free(enumerator); --} -- - LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(st= ruct loc_database_enumerator* enumerator) { - if (!enumerator) - return NULL; -@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(str= uct loc_database_enumerator - return 0; - } -=20 --LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database= _enumerator* enumerator, const char* country_code) { -- // Set empty country code -- if (!country_code || !*country_code) { -- *enumerator->country_code =3D '\0'; -- return 0; -- } -+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->countries) -+ return NULL; -=20 -- // Treat A1, A2, A3 as special country codes, -- // but perform search for flags instead -- if (strcmp(country_code, "A1") =3D=3D 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); -- } else if (strcmp(country_code, "A2") =3D=3D 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); -- } else if (strcmp(country_code, "A3") =3D=3D 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANYCAST); -- } -+ return loc_country_list_ref(enumerator->countries); -+} -=20 -- // Country codes must be two characters -- if (!loc_country_code_is_valid(country_code)) -- return -EINVAL; -+LOC_EXPORT int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* coun= tries) { -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); -=20 -- for (unsigned int i =3D 0; i < 3; i++) { -- enumerator->country_code[i] =3D country_code[i]; -- } -+ enumerator->countries =3D loc_country_list_ref(countries); -=20 - return 0; - } -=20 --LOC_EXPORT int loc_database_enumerator_set_asn( -- struct loc_database_enumerator* enumerator, unsigned int asn) { -- enumerator->asn =3D asn; -+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->asns) -+ return NULL; -+ -+ return loc_as_list_ref(enumerator->asns); -+} -+ -+LOC_EXPORT int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); -+ -+ enumerator->asns =3D loc_as_list_ref(asns); -=20 - return 0; - } -@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } -=20 --LOC_EXPORT int loc_database_enumerator_next_network( -- struct loc_database_enumerator* enumerator, struct loc_network** network)= { -- // Reset network -- *network =3D NULL; -+static int loc_database_enumerator_filter_network( -+ struct loc_database_enumerator* enumerator, struct loc_network* network) { -+ // Skip if the family does not match -+ if (enumerator->family && loc_network_address_family(network) !=3D enumera= tor->family) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matchin= g\n", network); -+ return 1; -+ } -=20 -- // Do not do anything if not in network mode -- if (enumerator->mode !=3D LOC_DB_ENUMERATE_NETWORKS) -- return 0; -+ // Skip if the country code does not match -+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries= )) { -+ const char* country_code =3D loc_network_get_country_code(network); -=20 -- int r; -+ if (!loc_country_list_contains_code(enumerator->countries, country_code))= { -+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not = matching\n", network); -+ return 1; -+ } -+ } -+ -+ // Skip if the ASN does not match -+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { -+ uint32_t asn =3D loc_network_get_asn(network); -+ -+ if (!loc_as_list_contains_number(enumerator->asns, asn)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\= n", network); -+ return 1; -+ } -+ } -+ -+ // Skip if flags do not match -+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flag= s)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching= \n", network); -+ return 1; -+ } -+ -+ // Do not filter -+ return 0; -+} -+ -+static int __loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network,= int filter) { -+ // Return top element from the stack -+ while (1) { -+ *network =3D loc_network_list_pop_first(enumerator->stack); -+ -+ // Stack is empty -+ if (!*network) -+ break; -+ -+ // Throw away any networks by filter -+ if (filter && loc_database_enumerator_filter_network(enumerator, *network= )) { -+ loc_network_unref(*network); -+ *network =3D NULL; -+ continue; -+ } -+ -+ // Return result -+ return 0; -+ } -=20 - DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", - enumerator->network_stack_depth); -@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( - enumerator->db->network_nodes_v1 + node->offset; -=20 - // Add edges to stack -- r =3D loc_database_enumerator_stack_push_node(enumerator, -+ int r =3D loc_database_enumerator_stack_push_node(enumerator, - be32toh(n->one), 1, node->depth + 1); -=20 - if (r) -@@ -1175,54 +1261,142 @@ LOC_EXPORT int loc_database_enumerator_next_network( - if (r) - return r; -=20 -- // Check if we are interested in this network -+ // Return all networks when the filter is disabled -+ if (!filter) -+ return 0; -=20 -- // Skip if the family does not match -- if (enumerator->family && loc_network_address_family(*network) !=3D enum= erator->family) { -+ // Check if we are interested in this network -+ if (loc_database_enumerator_filter_network(enumerator, *network)) { - loc_network_unref(*network); - *network =3D NULL; -=20 - continue; - } -=20 -- // Skip if the country code does not match -- if (*enumerator->country_code && -- !loc_network_match_country_code(*network, enumerator->country_code)) { -- loc_network_unref(*network); -- *network =3D NULL; -+ return 0; -+ } -+ } -=20 -- continue; -- } -+ // Reached the end of the search -+ return 0; -+} -=20 -- // Skip if the ASN does not match -- if (enumerator->asn && -- !loc_network_match_asn(*network, enumerator->asn)) { -- loc_network_unref(*network); -- *network =3D NULL; -+static int __loc_database_enumerator_next_network_flattened( -+ struct loc_database_enumerator* enumerator, struct loc_network** network)= { -+ // Fetch the next network -+ int r =3D __loc_database_enumerator_next_network(enumerator, network, 1); -+ if (r) -+ return r; -=20 -- continue; -- } -+ // End if we could not read another network -+ if (!*network) -+ return 0; -=20 -- // Skip if flags do not match -- if (enumerator->flags && -- !loc_network_match_flag(*network, enumerator->flags)) { -- loc_network_unref(*network); -- *network =3D NULL; -+ struct loc_network* subnet =3D NULL; -+ struct loc_network_list* subnets; -=20 -- continue; -+ // Create a list with all subnets -+ r =3D loc_network_list_new(enumerator->ctx, &subnets); -+ if (r) -+ return r; -+ -+ // Search all subnets from the database -+ while (1) { -+ // Fetch the next network in line -+ r =3D __loc_database_enumerator_next_network(enumerator, &subnet, 0); -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); -+ -+ return r; -+ } -+ -+ // End if we did not receive another subnet -+ if (!subnet) -+ break; -+ -+ // Collect all subnets in a list -+ if (loc_network_is_subnet(*network, subnet)) { -+ r =3D loc_network_list_push(subnets, subnet); -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); -+ -+ return r; - } -=20 -- return 0; -+ loc_network_unref(subnet); -+ continue; -+ } -+ -+ // If this is not a subnet, we push it back onto the stack and break -+ r =3D loc_network_list_push(enumerator->stack, subnet); -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); -+ -+ return r; - } -+ -+ loc_network_unref(subnet); -+ break; - } -=20 -- // Reached the end of the search -+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subn= ets)); -=20 -- // Mark all nodes as non-visited -- for (unsigned int i =3D 0; i < enumerator->db->network_nodes_count; i++) -- enumerator->networks_visited[i] =3D 0; -+ // We can abort here if the network has no subnets -+ if (loc_network_list_empty(subnets)) { -+ loc_network_list_unref(subnets); -=20 -- return 0; -+ return 0; -+ } -+ -+ // If the network has any subnets, we will break it into smaller parts -+ // without the subnets. -+ struct loc_network_list* excluded =3D loc_network_exclude_list(*network, s= ubnets); -+ if (!excluded) { -+ loc_network_list_unref(subnets); -+ return -1; -+ } -+ -+ // Merge subnets onto the stack -+ r =3D loc_network_list_merge(enumerator->stack, subnets); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); -+ -+ return r; -+ } -+ -+ // Push excluded list onto the stack -+ r =3D loc_network_list_merge(enumerator->stack, excluded); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); -+ -+ return r; -+ } -+ -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); -+ -+ // Drop the network and restart the whole process again to pick the next n= etwork -+ loc_network_unref(*network); -+ -+ return __loc_database_enumerator_next_network_flattened(enumerator, networ= k); -+} -+ -+LOC_EXPORT int loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network)= { -+ // Do not do anything if not in network mode -+ if (enumerator->mode !=3D LOC_DB_ENUMERATE_NETWORKS) -+ return 0; -+ -+ // Flatten output? -+ if (enumerator->flatten) -+ return __loc_database_enumerator_next_network_flattened(enumerator, netwo= rk); -+ -+ return __loc_database_enumerator_next_network(enumerator, network, 1); - } -=20 - LOC_EXPORT int loc_database_enumerator_next_country( -diff --git a/src/libloc.sym b/src/libloc.sym -index b8296eb..ee333f1 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -37,6 +37,18 @@ global: - loc_as_set_name; - loc_as_unref; -=20 -+ # AS List -+ loc_as_list_append; -+ loc_as_list_clear; -+ loc_as_list_contains; -+ loc_as_list_contains_number; -+ loc_as_list_empty; -+ loc_as_list_get; -+ loc_as_list_new; -+ loc_as_list_ref; -+ loc_as_list_size; -+ loc_as_list_unref; -+ - # Country - loc_country_cmp; - loc_country_code_is_valid; -@@ -49,6 +61,18 @@ global: - loc_country_set_name; - loc_country_unref; -=20 -+ # Country List -+ loc_country_list_append; -+ loc_country_list_clear; -+ loc_country_list_contains; -+ loc_country_list_contains_code; -+ loc_country_list_empty; -+ loc_country_list_get; -+ loc_country_list_new; -+ loc_country_list_ref; -+ loc_country_list_size; -+ loc_country_list_unref; -+ - # Database - loc_database_add_as; - loc_database_count_as; -@@ -66,13 +90,15 @@ global: - loc_database_verify; -=20 - # Database Enumerator -+ loc_database_enumerator_get_asns; -+ loc_database_enumerator_get_countries; - loc_database_enumerator_new; - loc_database_enumerator_next_as; - loc_database_enumerator_next_country; - loc_database_enumerator_next_network; - loc_database_enumerator_ref; -- loc_database_enumerator_set_asn; -- loc_database_enumerator_set_country_code; -+ loc_database_enumerator_set_asns; -+ loc_database_enumerator_set_countries; - loc_database_enumerator_set_family; - loc_database_enumerator_set_flag; - loc_database_enumerator_set_string; -@@ -80,24 +106,48 @@ global: -=20 - # Network - loc_network_address_family; -+ loc_network_cmp; -+ loc_network_exclude; -+ loc_network_exclude_list; - loc_network_format_first_address; - loc_network_format_last_address; - loc_network_get_asn; - loc_network_get_country_code; -+ loc_network_get_first_address; -+ loc_network_get_last_address; - loc_network_has_flag; -- loc_network_is_subnet_of; -+ loc_network_is_subnet; -+ loc_network_match_address; - loc_network_match_asn; - loc_network_match_country_code; - loc_network_match_flag; - loc_network_new; - loc_network_new_from_string; -+ loc_network_overlaps; -+ loc_network_prefix; - loc_network_ref; - loc_network_set_asn; - loc_network_set_country_code; - loc_network_set_flag; - loc_network_str; -+ loc_network_subnets; - loc_network_unref; -=20 -+ # Network List -+ loc_network_list_clear; -+ loc_network_list_contains; -+ loc_network_list_dump; -+ loc_network_list_empty; -+ loc_network_list_get; -+ loc_network_list_merge; -+ loc_network_list_new; -+ loc_network_list_pop; -+ loc_network_list_pop_first; -+ loc_network_list_push; -+ loc_network_list_ref; -+ loc_network_list_size; -+ loc_network_list_unref; -+ - # Writer - loc_writer_add_as; - loc_writer_add_country; -diff --git a/src/loc/as-list.h b/src/loc/as-list.h -new file mode 100644 -index 0000000..7b5c4e8 ---- /dev/null -+++ b/src/loc/as-list.h -@@ -0,0 +1,41 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#ifndef LIBLOC_AS_LIST_H -+#define LIBLOC_AS_LIST_H -+ -+#include -+#include -+ -+struct loc_as_list; -+ -+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); -+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); -+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); -+ -+size_t loc_as_list_size(struct loc_as_list* list); -+int loc_as_list_empty(struct loc_as_list* list); -+void loc_as_list_clear(struct loc_as_list* list); -+ -+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); -+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); -+ -+int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as); -+int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number); -+ -+#endif -diff --git a/src/loc/country-list.h b/src/loc/country-list.h -new file mode 100644 -index 0000000..a7f818a ---- /dev/null -+++ b/src/loc/country-list.h -@@ -0,0 +1,43 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#ifndef LIBLOC_COUNTRY_LIST_H -+#define LIBLOC_COUNTRY_LIST_H -+ -+#include -+ -+#include -+#include -+ -+struct loc_country_list; -+ -+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** lis= t); -+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list= ); -+struct loc_country_list* loc_country_list_unref(struct loc_country_list* li= st); -+ -+size_t loc_country_list_size(struct loc_country_list* list); -+int loc_country_list_empty(struct loc_country_list* list); -+void loc_country_list_clear(struct loc_country_list* list); -+ -+struct loc_country* loc_country_list_get(struct loc_country_list* list, siz= e_t index); -+int loc_country_list_append(struct loc_country_list* list, struct loc_count= ry* country); -+ -+int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country); -+int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code); -+ -+#endif -diff --git a/src/loc/database.h b/src/loc/database.h -index 43173dd..70801f0 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -25,6 +25,7 @@ - #include - #include - #include -+#include -=20 - struct loc_database; - int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, F= ILE* f); -@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode { - LOC_DB_ENUMERATE_COUNTRIES =3D 3, - }; -=20 -+enum loc_database_enumerator_flags { -+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN =3D (1 << 0), -+}; -+ - struct loc_database_enumerator; - int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode); -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags= ); - struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_data= base_enumerator* enumerator); - struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_da= tabase_enumerator* enumerator); -=20 - int loc_database_enumerator_set_string(struct loc_database_enumerator* enum= erator, const char* string); --int loc_database_enumerator_set_country_code(struct loc_database_enumerator= * enumerator, const char* country_code); --int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumera= tor, unsigned int asn); -+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_d= atabase_enumerator* enumerator); -+int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* count= ries); -+struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator); -+int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); - int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumer= ator, enum loc_network_flags flag); - int loc_database_enumerator_set_family(struct loc_database_enumerator* enum= erator, int family); - int loc_database_enumerator_next_as( -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -new file mode 100644 -index 0000000..bee21c4 ---- /dev/null -+++ b/src/loc/network-list.h -@@ -0,0 +1,37 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#ifndef LIBLOC_NETWORK_LIST_H -+#define LIBLOC_NETWORK_LIST_H -+ -+#include -+ -+struct loc_network_list; -+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** lis= t); -+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list= ); -+struct loc_network_list* loc_network_list_unref(struct loc_network_list* li= st); -+size_t loc_network_list_size(struct loc_network_list* list); -+int loc_network_list_empty(struct loc_network_list* list); -+void loc_network_list_clear(struct loc_network_list* list); -+void loc_network_list_dump(struct loc_network_list* list); -+struct loc_network* loc_network_list_get(struct loc_network_list* list, siz= e_t index); -+int loc_network_list_push(struct loc_network_list* list, struct loc_network= * network); -+struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+struct loc_network* loc_network_list_pop_first(struct loc_network_list* lis= t); -+int loc_network_list_contains(struct loc_network_list* list, struct loc_net= work* network); -+int loc_network_list_merge(struct loc_network_list* self, struct loc_networ= k_list* other); -+ -+#endif -diff --git a/src/loc/network.h b/src/loc/network.h -index 70c3803..af3dafd 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -21,6 +21,7 @@ -=20 - #include - #include -+#include -=20 - enum loc_network_flags { - LOC_NETWORK_FLAG_ANONYMOUS_PROXY =3D (1 << 0), // A1 -@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* n= etwork); - struct loc_network* loc_network_unref(struct loc_network* network); - char* loc_network_str(struct loc_network* network); - int loc_network_address_family(struct loc_network* network); -+unsigned int loc_network_prefix(struct loc_network* network); -=20 -+const struct in6_addr* loc_network_get_first_address(struct loc_network* ne= twork); - char* loc_network_format_first_address(struct loc_network* network); -+const struct in6_addr* loc_network_get_last_address(struct loc_network* net= work); - char* loc_network_format_last_address(struct loc_network* network); - int loc_network_match_address(struct loc_network* network, const struct in6= _addr* address); -=20 -@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uin= t32_t flag); - int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); -=20 --int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* = other); -+int loc_network_cmp(struct loc_network* self, struct loc_network* other); -+int loc_network_overlaps(struct loc_network* self, struct loc_network* othe= r); -+int loc_network_is_subnet(struct loc_network* self, struct loc_network* oth= er); -+int loc_network_subnets(struct loc_network* network, struct loc_network** s= ubnet1, struct loc_network** subnet2); -+struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other); -+struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list); -=20 - #ifdef LIBLOC_PRIVATE -=20 -diff --git a/src/network-list.c b/src/network-list.c -new file mode 100644 -index 0000000..698d3ab ---- /dev/null -+++ b/src/network-list.c -@@ -0,0 +1,299 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include -+#include -+#include -+ -+#include -+#include -+#include -+ -+struct loc_network_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_network** elements; -+ size_t elements_size; -+ -+ size_t size; -+}; -+ -+static int loc_network_list_grow(struct loc_network_list* list, size_t size= ) { -+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); -+ -+ struct loc_network** elements =3D reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; -+ -+ list->elements =3D elements; -+ list->elements_size +=3D size; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -+ struct loc_network_list** list) { -+ struct loc_network_list* l =3D calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx =3D loc_ref(ctx); -+ l->refcount =3D 1; -+ -+ DEBUG(l->ctx, "Network list allocated at %p\n", l); -+ *list =3D l; -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network= _list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_network_list_free(struct loc_network_list* list) { -+ DEBUG(list->ctx, "Releasing network list at %p\n", list); -+ -+ // Remove all content -+ loc_network_list_clear(list); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_netwo= rk_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_network_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -+ return list->size =3D=3D 0; -+} -+ -+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ if (!list->elements) -+ return; -+ -+ for (unsigned int i =3D 0; i < list->size; i++) -+ loc_network_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements =3D NULL; -+ list->elements_size =3D 0; -+ -+ list->size =3D 0; -+} -+ -+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -+ struct loc_network* network; -+ char* s; -+ -+ for (unsigned int i =3D 0; i < list->size; i++) { -+ network =3D list->elements[i]; -+ -+ s =3D loc_network_str(network); -+ -+ INFO(list->ctx, "%4d: %s\n", i, s); -+ free(s); -+ } -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list= * list, size_t index) { -+ // Check index -+ if (index >=3D list->size) -+ return NULL; -+ -+ return loc_network_ref(list->elements[index]); -+} -+ -+static off_t loc_network_list_find(struct loc_network_list* list, -+ struct loc_network* network, int* found) { -+ // Insert at the beginning for an empty list -+ if (loc_network_list_empty(list)) -+ return 0; -+ -+ off_t lo =3D 0; -+ off_t hi =3D list->size - 1; -+ int result; -+ -+ // Since we are working on an ordered list, there is often a good chance t= hat -+ // the network we are looking for is at the end or has to go to the end. -+ if (hi >=3D 0) { -+ result =3D loc_network_cmp(network, list->elements[hi]); -+ -+ // Match, so we are done -+ if (result =3D=3D 0) { -+ *found =3D 1; -+ -+ return hi; -+ -+ // This needs to be added after the last one -+ } else if (result > 0) { -+ *found =3D 0; -+ -+ return hi + 1; -+ } -+ } -+ -+#ifdef ENABLE_DEBUG -+ // Save start time -+ clock_t start =3D clock(); -+#endif -+ -+ off_t i =3D 0; -+ -+ while (lo <=3D hi) { -+ i =3D (lo + hi) / 2; -+ -+ // Check if this is a match -+ result =3D loc_network_cmp(network, list->elements[i]); -+ -+ if (result =3D=3D 0) { -+ *found =3D 1; -+ -+#ifdef ENABLE_DEBUG -+ clock_t end =3D clock(); -+ -+ // Log how fast this has been -+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif -+ -+ return i; -+ } -+ -+ if (result > 0) { -+ lo =3D i + 1; -+ i++; -+ } else { -+ hi =3D i - 1; -+ } -+ } -+ -+ *found =3D 0; -+ -+#ifdef ENABLE_DEBUG -+ clock_t end =3D clock(); -+ -+ // Log how fast this has been -+ DEBUG(list->ctx, "Did not find network in %.4fms (last i =3D %jd)\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif -+ -+ return i; -+} -+ -+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct = loc_network* network) { -+ int found =3D 0; -+ -+ off_t index =3D loc_network_list_find(list, network, &found); -+ -+ // The network has been found on the list. Nothing to do. -+ if (found) -+ return 0; -+ -+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n", -+ list, network, (intmax_t)index); -+ -+ // Check if we have space left -+ if (list->size >=3D list->elements_size) { -+ int r =3D loc_network_list_grow(list, 64); -+ if (r) -+ return r; -+ } -+ -+ // The list is now larger -+ list->size++; -+ -+ // Move all elements out of the way -+ for (unsigned int i =3D list->size - 1; i > index; i--) -+ list->elements[i] =3D list->elements[i - 1]; -+ -+ // Add the new element at the right place -+ list->elements[index] =3D loc_network_ref(network); -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list= * list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network =3D list->elements[--list->size]; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_networ= k_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network =3D list->elements[0]; -+ -+ // Move all elements to the top of the stack -+ for (unsigned int i =3D 0; i < list->size - 1; i++) { -+ list->elements[i] =3D list->elements[i+1]; -+ } -+ -+ // The list is shorter now -+ --list->size; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ -+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, str= uct loc_network* network) { -+ int found =3D 0; -+ -+ loc_network_list_find(list, network, &found); -+ -+ return found; -+} -+ -+LOC_EXPORT int loc_network_list_merge( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; -+ -+ for (unsigned int i =3D 0; i < other->size; i++) { -+ r =3D loc_network_list_push(self, other->elements[i]); -+ if (r) -+ return r; -+ } -+ -+ return 0; -+} -diff --git a/src/network.c b/src/network.c -index 366caa2..a6b679c 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -29,6 +29,7 @@ - #include - #include - #include -+#include - #include -=20 - struct loc_network { -@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6= _addr* address, const s - return a; - } -=20 -+static struct in6_addr address_increment(const struct in6_addr* address) { -+ struct in6_addr a =3D *address; -+ -+ for (int octet =3D 15; octet >=3D 0; octet--) { -+ if (a.s6_addr[octet] < 255) { -+ a.s6_addr[octet]++; -+ break; -+ } else { -+ a.s6_addr[octet] =3D 0; -+ } -+ } -+ -+ return a; -+} -+ - LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** ne= twork, - struct in6_addr* address, unsigned int prefix) { - // Address cannot be unspecified -@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, str= uct loc_network** network - LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_= network** network, - const char* address_string) { - struct in6_addr first_address; -- unsigned int prefix =3D 0; - char* prefix_string; -- int r =3D 1; -+ unsigned int prefix =3D 128; -+ int r =3D -EINVAL; -+ -+ DEBUG(ctx, "Attempting to parse network %s\n", address_string); -=20 - // Make a copy of the string to work on it - char* buffer =3D strdup(address_string); -@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_= ctx* ctx, struct loc_netwo - // Split address and prefix - address_string =3D strsep(&prefix_string, "/"); -=20 -- // Did we find a prefix? -+ DEBUG(ctx, " Split into address =3D %s, prefix =3D %s\n", address_string,= prefix_string); -+ -+ // Parse the address -+ r =3D loc_parse_address(ctx, address_string, &first_address); -+ if (r) { -+ DEBUG(ctx, "The address could not be parsed\n"); -+ goto FAIL; -+ } -+ -+ // If a prefix was given, we will try to parse it - if (prefix_string) { - // Convert prefix to integer - prefix =3D strtol(prefix_string, NULL, 10); -=20 -- if (prefix) { -- // Parse the address -- r =3D loc_parse_address(ctx, address_string, &first_address); -- -- // Map the prefix to IPv6 if needed -- if (IN6_IS_ADDR_V4MAPPED(&first_address)) -- prefix +=3D 96; -+ if (!prefix) { -+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); -+ goto FAIL; - } -+ -+ // Map the prefix to IPv6 if needed -+ if (IN6_IS_ADDR_V4MAPPED(&first_address)) -+ prefix +=3D 96; - } -=20 -+FAIL: - // Free temporary buffer - free(buffer); -=20 -- if (r =3D=3D 0) { -- r =3D loc_network_new(ctx, network, &first_address, prefix); -- } -+ // Exit if the parsing was unsuccessful -+ if (r) -+ return r; -=20 -- return r; -+ // Create a new network -+ return loc_network_new(ctx, network, &first_address, prefix); - } -=20 - LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network)= { -@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_ne= twork* network) { - return network->family; - } -=20 -+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) { -+ switch (network->family) { -+ case AF_INET6: -+ return network->prefix; -+ -+ case AF_INET: -+ return network->prefix - 96; -+ } -+ -+ return 0; -+} -+ - static char* loc_network_format_address(struct loc_network* network, const = struct in6_addr* address) { - const size_t length =3D INET6_ADDRSTRLEN; -=20 -@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_net= work* network, const struc - return string; - } -=20 -+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_= network* network) { -+ return &network->first_address; -+} -+ - LOC_EXPORT char* loc_network_format_first_address(struct loc_network* netwo= rk) { - return loc_network_format_address(network, &network->first_address); - } -=20 -+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_n= etwork* network) { -+ return &network->last_address; -+} -+ - LOC_EXPORT char* loc_network_format_last_address(struct loc_network* networ= k) { - return loc_network_format_address(network, &network->last_address); - } -@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struc= t loc_network* network) { - LOC_EXPORT int loc_network_match_address(struct loc_network* network, const= struct in6_addr* address) { - // Address must be larger than the start address - if (in6_addr_cmp(&network->first_address, address) > 0) -- return 1; -+ return 0; -=20 - // Address must be smaller than the last address - if (in6_addr_cmp(&network->last_address, address) < 0) -- return 1; -+ return 0; -=20 - // The address is inside this network -- return 0; -+ return 1; - } -=20 - LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* net= work) { -@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_netw= ork* network, uint32_t flag - return loc_network_has_flag(network, flag); - } -=20 --LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct lo= c_network* other) { -+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network= * other) { -+ // Compare address -+ int r =3D in6_addr_cmp(&self->first_address, &other->first_address); -+ if (r) -+ return r; -+ -+ // Compare prefix -+ if (self->prefix > other->prefix) -+ return 1; -+ else if (self->prefix < other->prefix) -+ return -1; -+ -+ // Both networks are equal -+ return 0; -+} -+ -+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_ne= twork* other) { -+ // Either of the start addresses must be in the other subnet -+ if (loc_network_match_address(self, &other->first_address)) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->first_address)) -+ return 1; -+ -+ // Or either of the end addresses is in the other subnet -+ if (loc_network_match_address(self, &other->last_address)) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->last_address)) -+ return 1; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_n= etwork* other) { -+ // The prefix must be smaller (this avoids the more complex comparisons la= ter) -+ if (self->prefix > other->prefix) -+ return 0; -+ - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) - return 0; -=20 - // If the end address of the other network is greater than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0) - return 0; -=20 - return 1; - } -=20 -+LOC_EXPORT int loc_network_subnets(struct loc_network* network, -+ struct loc_network** subnet1, struct loc_network** subnet2) { -+ int r; -+ *subnet1 =3D NULL; -+ *subnet2 =3D NULL; -+ -+ // New prefix length -+ unsigned int prefix =3D network->prefix + 1; -+ -+ // Check if the new prefix is valid -+ if (valid_prefix(&network->first_address, prefix)) -+ return -1; -+ -+ // Create the first half of the network -+ r =3D loc_network_new(network->ctx, subnet1, &network->first_address, pref= ix); -+ if (r) -+ return r; -+ -+ // The next subnet starts after the first one -+ struct in6_addr first_address =3D address_increment(&(*subnet1)->last_addr= ess); -+ -+ // Create the second half of the network -+ r =3D loc_network_new(network->ctx, subnet2, &first_address, prefix); -+ if (r) -+ return r; -+ -+ // Copy country code -+ const char* country_code =3D loc_network_get_country_code(network); -+ if (country_code) { -+ loc_network_set_country_code(*subnet1, country_code); -+ loc_network_set_country_code(*subnet2, country_code); -+ } -+ -+ // Copy ASN -+ uint32_t asn =3D loc_network_get_asn(network); -+ if (asn) { -+ loc_network_set_asn(*subnet1, asn); -+ loc_network_set_asn(*subnet2, asn); -+ } -+ -+ // Copy flags -+ loc_network_set_flag(*subnet1, network->flags); -+ loc_network_set_flag(*subnet2, network->flags); -+ -+ return 0; -+} -+ -+static int __loc_network_exclude(struct loc_network* network, -+ struct loc_network* other, struct loc_network_list* list) { -+ struct loc_network* subnet1 =3D NULL; -+ struct loc_network* subnet2 =3D NULL; -+ -+ int r =3D loc_network_subnets(network, &subnet1, &subnet2); -+ if (r) -+ goto ERROR; -+ -+ if (loc_network_cmp(other, subnet1) =3D=3D 0) { -+ r =3D loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_cmp(other, subnet2) =3D=3D 0) { -+ r =3D loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_is_subnet(subnet1, other)) { -+ r =3D loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ r =3D __loc_network_exclude(subnet1, other, list); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_is_subnet(subnet2, other)) { -+ r =3D loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ r =3D __loc_network_exclude(subnet2, other, list); -+ if (r) -+ goto ERROR; -+ -+ } else { -+ ERROR(network->ctx, "We should never get here\n"); -+ r =3D 1; -+ goto ERROR; -+ } -+ -+ERROR: -+ if (subnet1) -+ loc_network_unref(subnet1); -+ -+ if (subnet2) -+ loc_network_unref(subnet2); -+ -+ return r; -+} -+ -+static int __loc_network_exclude_to_list(struct loc_network* self, -+ struct loc_network* other, struct loc_network_list* list) { -+ // Other must be a subnet of self -+ if (!loc_network_is_subnet(self, other)) { -+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, se= lf); -+ -+ // Exit silently -+ return 0; -+ } -+ -+ // We cannot perform this operation if both networks equal -+ if (loc_network_cmp(self, other) =3D=3D 0) { -+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); -+ -+ // Exit silently -+ return 0; -+ } -+ -+ return __loc_network_exclude(self, other, list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other) { -+ struct loc_network_list* list; -+ -+#ifdef ENABLE_DEBUG -+ char* n1 =3D loc_network_str(self); -+ char* n2 =3D loc_network_str(other); -+ -+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); -+ -+ free(n1); -+ free(n2); -+#endif -+ -+ // Create a new list with the result -+ int r =3D loc_network_list_new(self->ctx, &list); -+ if (r) { -+ ERROR(self->ctx, "Could not create network list: %d\n", r); -+ -+ return NULL; -+ } -+ -+ r =3D __loc_network_exclude_to_list(self, other, list); -+ if (r) { -+ loc_network_list_unref(list); -+ -+ return NULL; -+ } -+ -+ // Return the result -+ return list; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list) { -+ struct loc_network_list* to_check; -+ -+ // Create a new list with all networks to look at -+ int r =3D loc_network_list_new(network->ctx, &to_check); -+ if (r) -+ return NULL; -+ -+ struct loc_network* subnet =3D NULL; -+ struct loc_network_list* subnets =3D NULL; -+ -+ for (unsigned int i =3D 0; i < loc_network_list_size(list); i++) { -+ subnet =3D loc_network_list_get(list, i); -+ -+ // Find all excluded networks -+ if (!loc_network_list_contains(to_check, subnet)) { -+ r =3D __loc_network_exclude_to_list(network, subnet, to_check); -+ if (r) { -+ loc_network_list_unref(to_check); -+ loc_network_unref(subnet); -+ -+ return NULL; -+ } -+ } -+ -+ // Cleanup -+ loc_network_unref(subnet); -+ } -+ -+ r =3D loc_network_list_new(network->ctx, &subnets); -+ if (r) { -+ loc_network_list_unref(to_check); -+ return NULL; -+ } -+ -+ off_t smallest_subnet =3D 0; -+ -+ while (!loc_network_list_empty(to_check)) { -+ struct loc_network* subnet_to_check =3D loc_network_list_pop_first(to_che= ck); -+ -+ // Check whether the subnet to check is part of the input list -+ if (loc_network_list_contains(list, subnet_to_check)) { -+ loc_network_unref(subnet_to_check); -+ continue; -+ } -+ -+ // Marks whether this subnet passed all checks -+ int passed =3D 1; -+ -+ for (unsigned int i =3D smallest_subnet; i < loc_network_list_size(list);= i++) { -+ subnet =3D loc_network_list_get(list, i); -+ -+ // Drop this subnet if is a subnet of another subnet -+ if (loc_network_is_subnet(subnet, subnet_to_check)) { -+ passed =3D 0; -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // Break it down if it overlaps -+ if (loc_network_overlaps(subnet, subnet_to_check)) { -+ passed =3D 0; -+ -+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); -+ -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // If the subnet is strictly greater, we do not need to continue the sea= rch -+ r =3D loc_network_cmp(subnet, subnet_to_check); -+ if (r > 0) { -+ loc_network_unref(subnet); -+ break; -+ -+ // If it is strictly smaller, we can continue the search from here next -+ // time because all networks that are to be checked can only be larger -+ // than this one. -+ } else if (r < 0) { -+ smallest_subnet =3D i; -+ } -+ -+ loc_network_unref(subnet); -+ } -+ -+ if (passed) { -+ r =3D loc_network_list_push(subnets, subnet_to_check); -+ } -+ -+ loc_network_unref(subnet_to_check); -+ } -+ -+ loc_network_list_unref(to_check); -+ -+ return subnets; -+} -+ - LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, stru= ct loc_database_network_v1* dbobj) { - // Add country code - loc_country_code_copy(dbobj->country_code, network->country_code); -@@ -474,7 +813,7 @@ struct loc_network_tree_node { - struct loc_network* network; - }; -=20 --LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network= _tree** tree) { -+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tre= e) { - struct loc_network_tree* t =3D calloc(1, sizeof(*t)); - if (!t) - return -ENOMEM; -@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx,= struct loc_network_tree - return 0; - } -=20 --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct l= oc_network_tree* tree) { -+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_= tree* tree) { - return loc_network_tree_node_ref(tree->root); - } -=20 -@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, = struct loc_network_tree_ - return 0; - } -=20 --LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, -+int loc_network_tree_walk(struct loc_network_tree* tree, - int(*filter_callback)(struct loc_network* network, void* data), - int(*callback)(struct loc_network* network, void* data), void* data) { - return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, cal= lback, data); -@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tre= e* tree) { - free(tree); - } -=20 --LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_netwo= rk_tree* tree) { -+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tr= ee) { - if (--tree->refcount > 0) - return tree; -=20 -@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network*= network, void* data) { - return 0; - } -=20 --LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { -+int loc_network_tree_dump(struct loc_network_tree* tree) { - DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); -=20 - return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); - } -=20 --LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, = struct loc_network* network) { -+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_= network* network) { - DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); -=20 - struct loc_network_tree_node* node =3D loc_network_tree_get_path(tree, -@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* = network, void* data) { - return 0; - } -=20 --LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* = tree) { -+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { - size_t counter =3D 0; -=20 - int r =3D loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &cou= nter); -@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct l= oc_network_tree_node* node) - return counter; - } -=20 --LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tre= e) { -+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { - return __loc_network_tree_count_nodes(tree->root); - } -=20 --LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_ne= twork_tree_node** node) { -+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_= node** node) { - struct loc_network_tree_node* n =3D calloc(1, sizeof(*n)); - if (!n) - return -ENOMEM; -@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx= * ctx, struct loc_network - return 0; - } -=20 --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct l= oc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_= tree_node* node) { - if (node) - node->refcount++; -=20 -@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_netwo= rk_tree_node* node) { - free(node); - } -=20 --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct= loc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_networ= k_tree_node* node) { - if (!node) - return NULL; -=20 -@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tr= ee_node_unref(struct loc_ - return NULL; - } -=20 --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct l= oc_network_tree_node* node, unsigned int index) { -+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_= tree_node* node, unsigned int index) { - if (index =3D=3D 0) - node =3D node->zero; - else -@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_= tree_node_get(struct loc_ne - return loc_network_tree_node_ref(node); - } -=20 --LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* = node) { -+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { - return (!!node->network); - } -=20 --LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc= _network_tree_node* node) { -+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tr= ee_node* node) { - return loc_network_ref(node->network); - } -diff --git a/src/perl/Location.xs b/src/perl/Location.xs -index dcf3f0d..b7676d2 100644 ---- a/src/perl/Location.xs -+++ b/src/perl/Location.xs -@@ -125,7 +125,7 @@ database_countries(db) - PPCODE: - // Create Database enumerator - struct loc_database_enumerator* enumerator; -- int err =3D loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE= _COUNTRIES); -+ int err =3D loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE= _COUNTRIES, 0); -=20 - if (err) { - croak("Could not create a database enumerator\n"); -diff --git a/src/python/database.c b/src/python/database.c -index 1013a58..0aa03cc 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -17,6 +17,8 @@ - #include -=20 - #include -+#include -+#include - #include -=20 - #include "locationmodule.h" -@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject*= type, struct loc_database - return (PyObject*)self; - } -=20 --static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_databa= se_enumerator_mode what) { -+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_databa= se_enumerator_mode what, int flags) { - struct loc_database_enumerator* enumerator; -=20 -- int r =3D loc_database_enumerator_new(&enumerator, self->db, what); -+ int r =3D loc_database_enumerator_new(&enumerator, self->db, what, flags); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* se= lf, enum loc_database_en - } -=20 - static PyObject* Database_ases(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); - } -=20 - static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { -@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self= , PyObject* args) { -=20 - struct loc_database_enumerator* enumerator; -=20 -- int r =3D loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMER= ATE_ASES); -+ int r =3D loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMER= ATE_ASES, 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* s= elf, PyObject* args) { - } -=20 - static PyObject* Database_networks(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); -+} -+ -+static PyObject* Database_networks_flattened(DatabaseObject *self) { -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMER= ATOR_FLAGS_FLATTEN); - } -=20 - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* a= rgs, PyObject* kwargs) { -- char* kwlist[] =3D { "country_code", "asn", "flags", "family", NULL }; -- const char* country_code =3D NULL; -- unsigned int asn =3D 0; -+ char* kwlist[] =3D { "country_codes", "asns", "flags", "family", "flatten"= , NULL }; -+ PyObject* country_codes =3D NULL; -+ PyObject* asn_list =3D NULL; - int flags =3D 0; - int family =3D 0; -+ int flatten =3D 0; -=20 -- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_c= ode, &asn, &flags, &family)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, -+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, = &flatten)) - return NULL; -=20 - struct loc_database_enumerator* enumerator; -- int r =3D loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMER= ATE_NETWORKS); -+ int r =3D loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMER= ATE_NETWORKS, -+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; - } -=20 - // Set country code we are searching for -- if (country_code) { -- r =3D loc_database_enumerator_set_country_code(enumerator, country_code); -+ if (country_codes) { -+ struct loc_country_list* countries; -+ r =3D loc_country_list_new(loc_ctx, &countries); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create country list"); -+ return NULL; -+ } -+ -+ for (int i =3D 0; i < PyList_Size(country_codes); i++) { -+ PyObject* item =3D PyList_GetItem(country_codes, i); -+ -+ if (!PyUnicode_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); -+ loc_country_list_unref(countries); -+ return NULL; -+ } -+ -+ const char* country_code =3D PyUnicode_AsUTF8(item); -+ -+ struct loc_country* country; -+ r =3D loc_country_new(loc_ctx, &country, country_code); -+ if (r) { -+ if (r =3D=3D -EINVAL) { -+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_cod= e); -+ } else { -+ PyErr_SetString(PyExc_SystemError, "Could not create country"); -+ } -+ -+ loc_country_list_unref(countries); -+ return NULL; -+ } -+ -+ // Append it to the list -+ r =3D loc_country_list_append(countries, country); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append country to the lis= t"); -+ -+ loc_country_list_unref(countries); -+ loc_country_unref(country); -+ return NULL; -+ } -+ -+ loc_country_unref(country); -+ } -=20 -+ r =3D loc_database_enumerator_set_countries(enumerator, countries); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_country_list_unref(countries); - return NULL; - } -+ -+ loc_country_list_unref(countries); - } -=20 - // Set the ASN we are searching for -- if (asn) { -- r =3D loc_database_enumerator_set_asn(enumerator, asn); -+ if (asn_list) { -+ struct loc_as_list* asns; -+ r =3D loc_as_list_new(loc_ctx, &asns); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); -+ return NULL; -+ } -+ -+ for (int i =3D 0; i < PyList_Size(asn_list); i++) { -+ PyObject* item =3D PyList_GetItem(asn_list, i); -+ -+ if (!PyLong_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); -=20 -+ loc_as_list_unref(asns); -+ return NULL; -+ } -+ -+ unsigned long number =3D PyLong_AsLong(item); -+ -+ struct loc_as* as; -+ r =3D loc_as_new(loc_ctx, &as, number); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS"); -+ -+ loc_as_list_unref(asns); -+ loc_as_unref(as); -+ return NULL; -+ } -+ -+ r =3D loc_as_list_append(asns, as); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); -+ -+ loc_as_list_unref(asns); -+ loc_as_unref(as); -+ return NULL; -+ } -+ -+ loc_as_unref(as); -+ } -+ -+ r =3D loc_database_enumerator_set_asns(enumerator, asns); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_as_list_unref(asns); - return NULL; - } -+ -+ loc_as_list_unref(asns); - } -=20 - // Set the flags we are searching for -@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject= * self, PyObject* args, - } -=20 - static PyObject* Database_countries(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); - } -=20 - static struct PyMethodDef Database_methods[] =3D { -@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] =3D { - NULL, - NULL, - }, -+ { -+ "networks_flattened", -+ (getter)Database_networks_flattened, -+ NULL, -+ NULL, -+ NULL, -+ }, - { - "vendor", - (getter)Database_get_vendor, -diff --git a/src/python/downloader.py b/src/python/downloader.py -index 87bbb68..05f7872 100644 ---- a/src/python/downloader.py -+++ b/src/python/downloader.py -@@ -119,8 +119,8 @@ class Downloader(object): -=20 - headers =3D {} - if timestamp: -- headers["If-Modified-Since"] =3D timestamp.strftime( -- "%a, %d %b %Y %H:%M:%S GMT", -+ headers["If-Modified-Since"] =3D time.strftime( -+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp), - ) -=20 - t =3D tempfile.NamedTemporaryFile(dir=3Dtmpdir, delete=3DFalse) -@@ -195,7 +195,7 @@ class Downloader(object): - db =3D Database(f.name) -=20 - # Database is not recent -- if timestamp and db.created_at < timestamp.timestamp(): -+ if timestamp and db.created_at < timestamp: - return False -=20 - log.info("Downloaded new database from %s" % (time.strftime( -diff --git a/src/python/export.py b/src/python/export.py -index d15c6f0..f0eae26 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -29,7 +29,7 @@ import _location - log =3D logging.getLogger("location.export") - log.propagate =3D 1 -=20 --flags =3D { -+FLAGS =3D { - _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", - _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", - _location.NETWORK_FLAG_ANYCAST : "A3", -@@ -39,11 +39,8 @@ class OutputWriter(object): - suffix =3D "networks" - mode =3D "w" -=20 -- def __init__(self, f, prefix=3DNone, flatten=3DTrue): -- self.f, self.prefix, self.flatten =3D f, prefix, flatten -- -- # The previously written network -- self._last_network =3D None -+ def __init__(self, f, prefix=3DNone): -+ self.f, self.prefix =3D f, prefix -=20 - # Immediately write the header - self._write_header() -@@ -60,18 +57,6 @@ class OutputWriter(object): - def __repr__(self): - return "<%s f=3D%s>" % (self.__class__.__name__, self.f) -=20 -- def _flatten(self, network): -- """ -- Checks if the given network needs to be written to file, -- or if it is a subnet of the previously written network. -- """ -- if self._last_network and network.is_subnet_of(self._last_network): -- return True -- -- # Remember this network for the next call -- self._last_network =3D network -- return False -- - def _write_header(self): - """ - The header of the file -@@ -84,16 +69,8 @@ class OutputWriter(object): - """ - pass -=20 -- def _write_network(self, network): -- self.f.write("%s\n" % network) -- - def write(self, network): -- if self.flatten and self._flatten(network): -- log.debug("Skipping writing network %s" % network) -- return -- -- # Write the network to file -- self._write_network(network) -+ self.f.write("%s\n" % network) -=20 - def finish(self): - """ -@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter): - def _write_header(self): - self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\= n" % self.prefix) -=20 -- def _write_network(self, network): -+ def write(self, network): - self.f.write("add %s %s\n" % (self.prefix, network)) -=20 -=20 -@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter): - def _write_footer(self): - self.f.write("}\n") -=20 -- def _write_network(self, network): -+ def write(self, network): - self.f.write(" %s,\n" % network) -=20 -=20 -@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter): - suffix =3D "iv" - mode =3D "wb" -=20 -- def _write_network(self, network): -- for address in (network.first_address, network.last_address): -- # Convert this into a string of bits -- bytes =3D socket.inet_pton( -- network.family, address, -- ) -- -- self.f.write(bytes) -+ def write(self, network): -+ self.f.write(network._first_address) -+ self.f.write(network._last_address) -=20 -=20 - formats =3D { -@@ -185,8 +157,14 @@ class Exporter(object): -=20 - writers[asn] =3D self.writer.open(filename, prefix=3D"AS%s" % asn) -=20 -+ # Filter countries from special country codes -+ country_codes =3D [ -+ country_code for country_code in countries if not country_code in FLAGS= .values() -+ ] -+ - # Get all networks that match the family -- networks =3D self.db.search_networks(family=3Dfamily) -+ networks =3D self.db.search_networks(family=3Dfamily, -+ country_codes=3Dcountry_codes, asns=3Dasns, flatten=3DTrue) -=20 - # Walk through all networks - for network in networks: -@@ -203,10 +181,10 @@ class Exporter(object): - pass -=20 - # Handle flags -- for flag in flags: -+ for flag in FLAGS: - if network.has_flag(flag): - # Fetch the "fake" country code -- country =3D flags[flag] -+ country =3D FLAGS[flag] -=20 - try: - writers[country].write(network) -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..5f46bc3 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -64,7 +64,7 @@ EXTENDED_SOURCES =3D ( - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", -=20 - # Latin America and Caribbean Network Information Centre -- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", -+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", -=20 - # R=C3=A9seaux IP Europ=C3=A9ens - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest= ", -diff --git a/src/python/location-importer.in b/src/python/location-importer.= in -index 1467923..2dec89e 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -152,6 +152,7 @@ class CLI(object): - last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); - CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcemen= ts(network); - CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family= (network)); -+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING = GIST(network inet_ops); -=20 - -- autnums - CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); -@@ -165,6 +166,7 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(fami= ly(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(netwo= rk inet_ops); -=20 - -- overrides -@@ -188,6 +190,8 @@ class CLI(object): - ); - CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network - ON network_overrides(network); -+ CREATE INDEX IF NOT EXISTS network_overrides_search -+ ON network_overrides USING GIST(network inet_ops); - """) -=20 - return db -@@ -234,32 +238,24 @@ class CLI(object): -=20 - # Select all known networks - rows =3D self.db.query(""" -- -- Get a (sorted) list of all known networks -- WITH known_networks AS ( -- SELECT network FROM announcements -- UNION -- SELECT network FROM networks -- ORDER BY network -- ) -- - -- Return a list of those networks enriched with all - -- other information that we store in the database - SELECT -- DISTINCT ON (known_networks.network) -- known_networks.network AS network, -- announcements.autnum AS autnum, -+ DISTINCT ON (network) -+ network, -+ autnum, -=20 - -- Country - COALESCE( - ( - SELECT country FROM network_overrides overrides -- WHERE announcements.network <<=3D overrides.network -+ WHERE networks.network <<=3D overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT country FROM autnum_overrides overrides -- WHERE announcements.autnum =3D overrides.number -+ WHERE networks.autnum =3D overrides.number - ), - networks.country - ) AS country, -@@ -268,50 +264,67 @@ class CLI(object): - COALESCE( - ( - SELECT is_anonymous_proxy FROM network_overrides overrides -- WHERE announcements.network <<=3D overrides.network -+ WHERE networks.network <<=3D overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anonymous_proxy FROM autnum_overrides overrides -- WHERE announcements.autnum =3D overrides.number -+ WHERE networks.autnum =3D overrides.number - ), - FALSE - ) AS is_anonymous_proxy, - COALESCE( - ( - SELECT is_satellite_provider FROM network_overrides overrides -- WHERE announcements.network <<=3D overrides.network -+ WHERE networks.network <<=3D overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_satellite_provider FROM autnum_overrides overrides -- WHERE announcements.autnum =3D overrides.number -+ WHERE networks.autnum =3D overrides.number - ), - FALSE - ) AS is_satellite_provider, - COALESCE( - ( - SELECT is_anycast FROM network_overrides overrides -- WHERE announcements.network <<=3D overrides.network -+ WHERE networks.network <<=3D overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anycast FROM autnum_overrides overrides -- WHERE announcements.autnum =3D overrides.number -+ WHERE networks.autnum =3D overrides.number - ), - FALSE -- ) AS is_anycast, -- -- -- Must be part of returned values for ORDER BY clause -- masklen(announcements.network) AS sort_a, -- masklen(networks.network) AS sort_b -- FROM known_networks -- LEFT JOIN announcements ON known_networks.network <<=3D announcements.n= etwork -- LEFT JOIN networks ON known_networks.network <<=3D networks.network -- ORDER BY known_networks.network, sort_a DESC, sort_b DESC -+ ) AS is_anycast -+ FROM ( -+ SELECT -+ known_networks.network AS network, -+ announcements.autnum AS autnum, -+ networks.country AS country, -+ -+ -- Must be part of returned values for ORDER BY clause -+ masklen(announcements.network) AS sort_a, -+ masklen(networks.network) AS sort_b -+ FROM ( -+ SELECT network FROM announcements -+ UNION ALL -+ SELECT network FROM networks -+ UNION ALL -+ SELECT network FROM network_overrides -+ ) known_networks -+ LEFT JOIN -+ announcements ON known_networks.network <<=3D announcements.network -+ LEFT JOIN -+ networks ON known_networks.network <<=3D networks.network -+ ORDER BY -+ known_networks.network, -+ sort_a DESC, -+ sort_b DESC -+ ) networks - """) -=20 - for row in rows: -@@ -363,6 +376,16 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); -+ -+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT= NULL) -+ ON COMMIT DROP; -+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), m= asklen(network)); -+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -+ """) -+ -+ # Remove all previously imported content -+ self.db.execute(""" -+ TRUNCATE TABLE networks; - """) -=20 - for source in location.importer.WHOIS_SOURCES: -@@ -370,31 +393,72 @@ class CLI(object): - for block in f: - self._parse_block(block) -=20 -+ # Process all parsed networks from every RIR we happen to have access to, -+ # insert the largest network chunks into the networks table immediately.= .. -+ families =3D self.db.query("SELECT DISTINCT family(network) AS family FR= OM _rirdata ORDER BY family(network)") -+ -+ for family in (row.family for row in families): -+ smallest =3D self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _= rirdata WHERE family(network) =3D %s", family) -+ -+ self.db.execute("INSERT INTO networks(network, country) \ -+ SELECT network, country FROM _rirdata WHERE masklen(network) =3D %s AN= D family(network) =3D %s", smallest.prefix, family) -+ -+ # ... determine any other prefixes for this network family, ... -+ prefixes =3D self.db.query("SELECT DISTINCT masklen(network) AS prefix = FROM _rirdata \ -+ WHERE family(network) =3D %s ORDER BY masklen(network) ASC OFFSET 1", = family) -+ -+ # ... and insert networks with this prefix in case they provide additio= nal -+ # information (i. e. subnet of a larger chunk with a different country) -+ for prefix in (row.prefix for row in prefixes): -+ self.db.execute(""" -+ WITH candidates AS ( -+ SELECT -+ _rirdata.network, -+ _rirdata.country -+ FROM -+ _rirdata -+ WHERE -+ family(_rirdata.network) =3D %s -+ AND -+ masklen(_rirdata.network) =3D %s -+ ), -+ filtered AS ( -+ SELECT -+ DISTINCT ON (c.network) -+ c.network, -+ c.country, -+ masklen(networks.network), -+ networks.country AS parent_country -+ FROM -+ candidates c -+ LEFT JOIN -+ networks -+ ON -+ c.network << networks.network -+ ORDER BY -+ c.network, -+ masklen(networks.network) DESC NULLS LAST -+ ) -+ INSERT INTO -+ networks(network, country) -+ SELECT -+ network, -+ country -+ FROM -+ filtered -+ WHERE -+ parent_country IS NULL -+ OR -+ country <> parent_country -+ ON CONFLICT DO NOTHING""", -+ family, prefix, -+ ) -+ - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums - JOIN _organizations ON _autnums.organization =3D _organizations.handle -- ON CONFLICT (number) DO UPDATE SET name =3D excluded.name -- """) -- -- self.db.execute(""" -- --- Purge any redundant entries -- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP -- AS -- SELECT network FROM networks candidates -- WHERE EXISTS ( -- SELECT FROM networks -- WHERE -- networks.network << candidates.network -- AND -- networks.country =3D candidates.country -- ); -- -- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); -- -- DELETE FROM networks WHERE EXISTS ( -- SELECT FROM _garbage WHERE networks.network =3D _garbage.network -- ); -+ ON CONFLICT (number) DO UPDATE SET name =3D excluded.name; - """) -=20 - # Download all extended sources -@@ -405,6 +469,69 @@ class CLI(object): - for line in f: - self._parse_line(line) -=20 -+ def _check_parsed_network(self, network): -+ """ -+ Assistive function to detect and subsequently sort out parsed -+ networks from RIR data (both Whois and so-called "extended sources"), -+ which are or have... -+ -+ (a) not globally routable (RFC 1918 space, et al.) -+ (b) covering a too large chunk of the IP address space (prefix length -+ is < 7 for IPv4 networks, and < 10 for IPv6) -+ (c) "0.0.0.0" or "::" as a network address -+ (d) are too small for being publicly announced (we have decided not to -+ process them at the moment, as they significantly enlarge our -+ database without providing very helpful additional information) -+ -+ This unfortunately is necessary due to brain-dead clutter across -+ various RIR databases, causing mismatches and eventually disruptions. -+ -+ We will return False in case a network is not suitable for adding -+ it to our database, and True otherwise. -+ """ -+ -+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isin= stance(network, ipaddress.IPv6Network)): -+ return False -+ -+ if not network.is_global: -+ log.warning("Skipping non-globally routable network: %s" % network) -+ return False -+ -+ if network.version =3D=3D 4: -+ if network.prefixlen < 7: -+ log.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 24: -+ log.debug("Skipping network too small to be publicly announced: %s" % n= etwork) -+ return False -+ -+ if str(network.network_address) =3D=3D "0.0.0.0": -+ log.warning("Skipping network based on 0.0.0.0: %s" % network) -+ return False -+ -+ elif network.version =3D=3D 6: -+ if network.prefixlen < 10: -+ log.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 48: -+ log.debug("Skipping network too small to be publicly announced: %s" % n= etwork) -+ return False -+ -+ if str(network.network_address) =3D=3D "::": -+ log.warning("Skipping network based on '::': %s" % network) -+ return False -+ -+ else: -+ # This should not happen... -+ log.warning("Skipping network of unknown family, this should not happen:= %s" % network) -+ return False -+ -+ # In case we have made it here, the network is considered to -+ # be suitable for libloc consumption... -+ return True -+ - def _parse_block(self, block): - # Get first line to find out what type of block this is - line =3D block[0] -@@ -433,7 +560,7 @@ class CLI(object): - autnum["asn"] =3D m.group(2) -=20 - elif key =3D=3D "org": -- autnum[key] =3D val -+ autnum[key] =3D val.upper() -=20 - # Skip empty objects - if not autnum: -@@ -447,15 +574,22 @@ class CLI(object): - ) -=20 - def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -+ log.debug("Parsing inetnum block:") -=20 - inetnum =3D {} - for line in block: -- logging.debug(line) -+ log.debug(line) -=20 - # Split line - key, val =3D split_line(line) -=20 -+ # Filter any inetnum records which are only referring to IP space -+ # not managed by that specific RIR... -+ if key =3D=3D "netname": -+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-= NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): -+ log.warning("Skipping record indicating historic/orphaned data: %s" % = val.strip()) -+ return -+ - if key =3D=3D "inetnum": - start_address, delim, end_address =3D val.partition("-") -=20 -@@ -467,7 +601,7 @@ class CLI(object): - start_address =3D ipaddress.ip_address(start_address) - end_address =3D ipaddress.ip_address(end_address) - except ValueError: -- logging.warning("Could not parse line: %s" % line) -+ log.warning("Could not parse line: %s" % line) - return -=20 - # Set prefix to default -@@ -484,23 +618,24 @@ class CLI(object): - inetnum[key] =3D val -=20 - elif key =3D=3D "country": -- if val =3D=3D "UNITED STATES": -- val =3D "US" -- - inetnum[key] =3D val.upper() -=20 - # Skip empty objects -- if not inetnum: -+ if not inetnum or not "country" in inetnum: -+ return -+ -+ # Skip objects with bogus country code 'ZZ' -+ if inetnum.get("country") =3D=3D "ZZ": -+ log.warning("Skipping network with bogus country 'ZZ': %s" % \ -+ (inetnum.get("inet6num") or inetnum.get("inetnum"))) - return -=20 - network =3D ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("= inetnum"), strict=3DFalse) -=20 -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -+ if not self._check_parsed_network(network): - return -=20 -- self.db.execute("INSERT INTO networks(network, country) \ -+ self.db.execute("INSERT INTO _rirdata(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country =3D excluded.= country", - "%s" % network, inetnum.get("country"), - ) -@@ -511,7 +646,9 @@ class CLI(object): - # Split line - key, val =3D split_line(line) -=20 -- if key in ("organisation", "org-name"): -+ if key =3D=3D "organisation": -+ org[key] =3D val.upper() -+ elif key =3D=3D "org-name": - org[key] =3D val -=20 - # Skip empty objects -@@ -581,6 +718,9 @@ class CLI(object): - log.warning("Invalid IP address: %s" % address) - return -=20 -+ if not self._check_parsed_network(network): -+ return -+ - self.db.execute("INSERT INTO networks(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO \ - UPDATE SET country =3D excluded.country", -diff --git a/src/python/location.in b/src/python/location.in -index 44ad726..b30beae 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -253,6 +253,7 @@ class CLI(object): - network =3D db.lookup(address) - except ValueError: - print(_("Invalid IP address: %s") % address, file=3Dsys.stderr) -+ return 2 -=20 - args =3D { - "address" : address, -@@ -398,10 +399,7 @@ class CLI(object): -=20 - def handle_update(self, db, ns): - if ns.cron and db: -- now =3D datetime.datetime.utcnow() -- -- # Parse the database timestamp -- t =3D datetime.datetime.utcfromtimestamp(db.created_at) -+ now =3D time.time() -=20 - if ns.cron =3D=3D "daily": - delta =3D datetime.timedelta(days=3D1) -@@ -410,22 +408,20 @@ class CLI(object): - elif ns.cron =3D=3D "monthly": - delta =3D datetime.timedelta(days=3D30) -=20 -+ delta =3D delta.total_seconds() -+ - # Check if the database has recently been updated -- if t >=3D (now - delta): -+ if db.created_at >=3D (now - delta): - log.info( -- _("The database has been updated recently (%s)") % \ -- format_timedelta(now - t), -+ _("The database has been updated recently"), - ) - return 3 -=20 - # Fetch the timestamp we need from DNS - t =3D location.discover_latest_version() -=20 -- # Parse timestamp into datetime format -- timestamp =3D datetime.datetime.utcfromtimestamp(t) if t else None -- - # Check the version of the local database -- if db and timestamp and db.created_at >=3D timestamp.timestamp(): -+ if db and t and db.created_at >=3D t: - log.info("Already on the latest version") - return -=20 -@@ -437,7 +433,7 @@ class CLI(object): -=20 - # Try downloading a new database - try: -- t =3D d.download(public_key=3Dns.public_key, timestamp=3Dtimestamp, tmpd= ir=3Dtmpdir) -+ t =3D d.download(public_key=3Dns.public_key, timestamp=3Dt, tmpdir=3Dtmp= dir) -=20 - # If no file could be downloaded, log a message - except FileNotFoundError as e: -@@ -453,13 +449,7 @@ class CLI(object): -=20 - return 0 -=20 -- def handle_verify(self, ns): -- try: -- db =3D location.Database(ns.database) -- except FileNotFoundError as e: -- log.error("%s: %s" % (ns.database, e)) -- return 127 -- -+ def handle_verify(self, db, ns): - # Verify the database - with open(ns.public_key, "r") as f: - if not db.verify(f): -diff --git a/src/python/network.c b/src/python/network.c -index 5496d1e..5b1369d 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -17,13 +17,33 @@ - #include -=20 - #include -+#include -=20 - #include - #include -+#include -=20 - #include "locationmodule.h" - #include "network.h" -=20 -+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { -+ PyObject* list =3D PyList_New(0); -+ if (!networks) -+ return list; -+ -+ while (!loc_network_list_empty(networks)) { -+ struct loc_network* network =3D loc_network_list_pop(networks); -+ -+ PyObject* n =3D new_network(&NetworkType, network); -+ PyList_Append(list, n); -+ -+ loc_network_unref(network); -+ Py_DECREF(n); -+ } -+ -+ return list; -+} -+ - PyObject* new_network(PyTypeObject* type, struct loc_network* network) { - NetworkObject* self =3D (NetworkObject*)type->tp_alloc(type, 0); - if (self) { -@@ -114,10 +134,18 @@ static int Network_set_asn(NetworkObject* self, PyObje= ct* value) { - long int asn =3D PyLong_AsLong(value); -=20 - // Check if the ASN is within the valid range -- if (asn <=3D 0 || asn > UINT32_MAX) { -+ if (asn <=3D 0) { -+ PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn); -+ return -1; -+ } -+ -+#if (__WORDSIZE > 32) -+ // Check whether the input was longer than 32 bit -+ if (asn > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn); - return -1; - } -+#endif -=20 - int r =3D loc_network_set_asn(self->network, asn); - if (r) -@@ -154,13 +182,28 @@ static PyObject* Network_set_flag(NetworkObject* self,= PyObject* args) { - Py_RETURN_NONE; - } -=20 -+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { -+ NetworkObject* other =3D NULL; -+ -+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) -+ return NULL; -+ -+ struct loc_network_list* list =3D loc_network_exclude(self->network, other= ->network); -+ -+ // Convert to Python objects -+ PyObject* obj =3D PyList_FromNetworkList(list); -+ loc_network_list_unref(list); -+ -+ return obj; -+} -+ - static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { - NetworkObject* other =3D NULL; -=20 - if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) - return NULL; -=20 -- if (loc_network_is_subnet_of(self->network, other->network)) -+ if (loc_network_is_subnet(other->network, self->network)) - Py_RETURN_TRUE; -=20 - Py_RETURN_FALSE; -@@ -181,6 +224,26 @@ static PyObject* Network_get_first_address(NetworkObjec= t* self) { - return obj; - } -=20 -+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) { -+ struct in_addr address4; -+ -+ // Convert IPv4 addresses to struct in_addr -+ if (IN6_IS_ADDR_V4MAPPED(address6)) { -+ address4.s_addr =3D address6->s6_addr32[3]; -+ -+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4)= ); -+ } -+ -+ // Return IPv6 addresses as they are -+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6)); -+} -+ -+static PyObject* Network_get__first_address(NetworkObject* self) { -+ const struct in6_addr* address =3D loc_network_get_first_address(self->net= work); -+ -+ return PyBytes_FromAddress(address); -+} -+ - static PyObject* Network_get_last_address(NetworkObject* self) { - char* address =3D loc_network_format_last_address(self->network); -=20 -@@ -190,7 +253,19 @@ static PyObject* Network_get_last_address(NetworkObject= * self) { - return obj; - } -=20 -+static PyObject* Network_get__last_address(NetworkObject* self) { -+ const struct in6_addr* address =3D loc_network_get_last_address(self->netw= ork); -+ -+ return PyBytes_FromAddress(address); -+} -+ - static struct PyMethodDef Network_methods[] =3D { -+ { -+ "exclude", -+ (PyCFunction)Network_exclude, -+ METH_VARARGS, -+ NULL, -+ }, - { - "has_flag", - (PyCFunction)Network_has_flag, -@@ -241,6 +316,13 @@ static struct PyGetSetDef Network_getsetters[] =3D { - NULL, - NULL, - }, -+ { -+ "_first_address", -+ (getter)Network_get__first_address, -+ NULL, -+ NULL, -+ NULL, -+ }, - { - "last_address", - (getter)Network_get_last_address, -@@ -248,6 +330,13 @@ static struct PyGetSetDef Network_getsetters[] =3D { - NULL, - NULL, - }, -+ { -+ "_last_address", -+ (getter)Network_get__last_address, -+ NULL, -+ NULL, -+ NULL, -+ }, - { NULL }, - }; -=20 -diff --git a/src/test-as.c b/src/test-as.c -index 839a04c..2d61675 100644 ---- a/src/test-as.c -+++ b/src/test-as.c -@@ -95,7 +95,7 @@ int main(int argc, char** argv) { - // Enumerator -=20 - struct loc_database_enumerator* enumerator; -- err =3D loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES= ); -+ err =3D loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES= , 0); - if (err) { - fprintf(stderr, "Could not create a database enumerator\n"); - exit(EXIT_FAILURE); -diff --git a/src/test-database.c b/src/test-database.c -index b4a75c4..da4b11c 100644 ---- a/src/test-database.c -+++ b/src/test-database.c -@@ -38,6 +38,14 @@ const char* DESCRIPTION =3D - "Maecenas ut venenatis nunc."; - const char* LICENSE =3D "CC"; -=20 -+const char* networks[] =3D { -+ "2001:db8::/32", -+ "2001:db8:1000::/48", -+ "2001:db8:2000::/48", -+ "2001:db8:2020::/48", -+ NULL, -+}; -+ - static int attempt_to_open(struct loc_ctx* ctx, char* path) { - FILE* f =3D fopen(path, "r"); - if (!f) -@@ -139,6 +147,24 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } -=20 -+ struct loc_network* network =3D NULL; -+ -+ // Add some networks -+ const char** n =3D networks; -+ while (*n) { -+ err =3D loc_writer_add_network(writer, &network, *n); -+ if (err) { -+ fprintf(stderr, "Could not add network %s\n", *n); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Set a country -+ loc_network_set_country_code(network, "XX"); -+ -+ // Next one -+ n++; -+ } -+ - FILE* f =3D tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); -@@ -170,6 +196,33 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } -=20 -+ // Enumerator -+ struct loc_database_enumerator* enumerator; -+ err =3D loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETW= ORKS, 0); -+ if (err) { -+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Walk through all networks -+ while (1) { -+ err =3D loc_database_enumerator_next_network(enumerator, &network); -+ if (err) { -+ fprintf(stderr, "Error fetching the next network: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!network) -+ break; -+ -+ char* s =3D loc_network_str(network); -+ printf("Got network: %s\n", s); -+ free(s); -+ } -+ -+ // Free the enumerator -+ loc_database_enumerator_unref(enumerator); -+ - // Close the database - loc_database_unref(db); - loc_unref(ctx); -diff --git a/src/test-network-list.c b/src/test-network-list.c -new file mode 100644 -index 0000000..6f32ff7 ---- /dev/null -+++ b/src/test-network-list.c -@@ -0,0 +1,183 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team -+ -+ This program is free software; you can redistribute it and/or modify -+ it under the terms of the GNU General Public License as published by -+ the Free Software Foundation; either version 2 of the License, or -+ (at your option) any later version. -+ -+ This program is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ GNU General Public License for more details. -+*/ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+ -+int main(int argc, char** argv) { -+ int err; -+ -+ struct loc_ctx* ctx; -+ err =3D loc_new(&ctx); -+ if (err < 0) -+ exit(EXIT_FAILURE); -+ -+ // Enable debug logging -+ loc_set_log_priority(ctx, LOG_DEBUG); -+ -+ // Create a network -+ struct loc_network* network1; -+ err =3D loc_network_new_from_string(ctx, &network1, "2001:db8::/32"); -+ if (err) { -+ fprintf(stderr, "Could not create the network1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet1; -+ err =3D loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet2; -+ err =3D loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet2\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet3; -+ err =3D loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet3\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet4; -+ err =3D loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet4\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet5; -+ err =3D loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet5\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet6; -+ err =3D loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet6\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Make a list with both subnets -+ struct loc_network_list* subnets; -+ err =3D loc_network_list_new(ctx, &subnets); -+ if (err) { -+ fprintf(stderr, "Could not create subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ size_t size =3D loc_network_list_size(subnets); -+ if (size > 0) { -+ fprintf(stderr, "The list is not empty: %zu\n", size); -+ exit(EXIT_FAILURE); -+ } -+ -+ err =3D loc_network_list_push(subnets, subnet1); -+ if (err) { -+ fprintf(stderr, "Could not add subnet1 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (loc_network_list_empty(subnets)) { -+ fprintf(stderr, "The subnets list reports that it is empty\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err =3D loc_network_list_push(subnets, subnet2); -+ if (err) { -+ fprintf(stderr, "Could not add subnet2 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Add the fourth one next -+ err =3D loc_network_list_push(subnets, subnet4); -+ if (err) { -+ fprintf(stderr, "Could not add subnet4 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Add the third one -+ err =3D loc_network_list_push(subnets, subnet3); -+ if (err) { -+ fprintf(stderr, "Could not add subnet3 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Add more subnets -+ err =3D loc_network_list_push(subnets, subnet5); -+ if (err) { -+ fprintf(stderr, "Could not add subnet5 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err =3D loc_network_list_push(subnets, subnet6); -+ if (err) { -+ fprintf(stderr, "Could not add subnet6 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(subnets); -+ -+ size =3D loc_network_list_size(subnets); -+ if (size !=3D 6) { -+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", siz= e); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Exclude subnet1 from network1 -+ struct loc_network_list* excluded =3D loc_network_exclude(network1, subnet= 1); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for = subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ // Exclude all subnets from network1 -+ excluded =3D loc_network_exclude_list(network1, subnets); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for = subnets\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ if (excluded) -+ loc_network_list_unref(excluded); -+ -+ loc_network_list_unref(subnets); -+ loc_network_unref(network1); -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ loc_unref(ctx); -+ -+ return EXIT_SUCCESS; -+} -diff --git a/src/test-network.c b/src/test-network.c -index d38f13d..dde13f1 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -14,6 +14,7 @@ - GNU General Public License for more details. - */ -=20 -+#include - #include - #include - #include -@@ -37,12 +38,21 @@ int main(int argc, char** argv) { - // Enable debug logging - loc_set_log_priority(ctx, LOG_DEBUG); -=20 -+#if 0 - struct loc_network_tree* tree; - err =3D loc_network_tree_new(ctx, &tree); - if (err) { - fprintf(stderr, "Could not create the network tree\n"); - exit(EXIT_FAILURE); - } -+#endif -+ -+ struct in6_addr address; -+ err =3D inet_pton(AF_INET6, "2001:db8::1", &address); -+ if (err !=3D 1) { -+ fprintf(stderr, "Could not parse IP address\n"); -+ exit(EXIT_FAILURE); -+ } -=20 - // Create a network - struct loc_network* network1; -@@ -58,12 +68,14 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } -=20 -+#if 0 - // Adding network to the tree - err =3D loc_network_tree_add_network(tree, network1); - if (err) { - fprintf(stderr, "Could not add network to the tree\n"); - exit(EXIT_FAILURE); - } -+#endif -=20 - // Check if the first and last addresses are correct - char* string =3D loc_network_format_first_address(network1); -@@ -88,6 +100,12 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } -=20 -+ err =3D loc_network_match_address(network1, &address); -+ if (!err) { -+ fprintf(stderr, "Network1 does not match address\n"); -+ exit(EXIT_FAILURE); -+ } -+ - struct loc_network* network2; - err =3D loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48"); - if (err) { -@@ -101,6 +119,7 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } -=20 -+#if 0 - // Adding network to the tree - err =3D loc_network_tree_add_network(tree, network2); - if (err) { -@@ -117,20 +136,84 @@ int main(int argc, char** argv) { -=20 - size_t nodes =3D loc_network_tree_count_nodes(tree); - printf("The tree has %zu nodes\n", nodes); -+#endif -+ -+ // Check equals function -+ err =3D loc_network_cmp(network1, network1); -+ if (err) { -+ fprintf(stderr, "Network is not equal with itself\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err =3D loc_network_cmp(network1, network2); -+ if (!err) { -+ fprintf(stderr, "Networks equal unexpectedly\n"); -+ exit(EXIT_FAILURE); -+ } -=20 - // Check subnet function -- err =3D loc_network_is_subnet_of(network1, network2); -- if (err !=3D 0) { -+ err =3D loc_network_is_subnet(network1, network2); -+ if (!err) { - fprintf(stderr, "Subnet check 1 failed: %d\n", err); - exit(EXIT_FAILURE); - } -=20 -- err =3D loc_network_is_subnet_of(network2, network1); -- if (err !=3D 1) { -+ err =3D loc_network_is_subnet(network2, network1); -+ if (err) { - fprintf(stderr, "Subnet check 2 failed: %d\n", err); - exit(EXIT_FAILURE); - } -=20 -+ // Make subnets -+ struct loc_network* subnet1 =3D NULL; -+ struct loc_network* subnet2 =3D NULL; -+ -+ err =3D loc_network_subnets(network1, &subnet1, &subnet2); -+ if (err || !subnet1 || !subnet2) { -+ fprintf(stderr, "Could not find subnets of network: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ char* s =3D loc_network_str(subnet1); -+ printf("Received subnet1 =3D %s\n", s); -+ free(s); -+ -+ s =3D loc_network_str(subnet2); -+ printf("Received subnet2 =3D %s\n", s); -+ free(s); -+ -+ if (!loc_network_is_subnet(network1, subnet1)) { -+ fprintf(stderr, "Subnet1 is not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!loc_network_is_subnet(network1, subnet2)) { -+ fprintf(stderr, "Subnet2 is not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!loc_network_overlaps(network1, subnet1)) { -+ fprintf(stderr, "Network1 does not seem to contain subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!loc_network_overlaps(network1, subnet2)) { -+ fprintf(stderr, "Network1 does not seem to contain subnet2\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ -+ struct loc_network_list* excluded =3D loc_network_exclude(network1, networ= k2); -+ if (!excluded) { -+ fprintf(stderr, "Could not create excluded list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ loc_network_list_unref(excluded); -+ - // Create a database - struct loc_writer* writer; - err =3D loc_writer_new(ctx, &writer, NULL, NULL); -@@ -160,6 +243,28 @@ int main(int argc, char** argv) { - // Set ASN - loc_network_set_asn(network4, 1024); -=20 -+ // Try adding an invalid network -+ struct loc_network* network; -+ err =3D loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); -+ if (err !=3D -EINVAL) { -+ fprintf(stderr, "It was possible to add an invalid network (err =3D %d)\n= ", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Try adding a single address -+ err =3D loc_writer_add_network(writer, &network, "2001:db8::"); -+ if (err) { -+ fprintf(stderr, "It was impossible to add an single IP address (err =3D %= d)\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Try adding localhost -+ err =3D loc_writer_add_network(writer, &network, "::1/128"); -+ if (err !=3D -EINVAL) { -+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f =3D tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); -@@ -177,7 +282,10 @@ int main(int argc, char** argv) { - loc_network_unref(network2); - loc_network_unref(network3); - loc_network_unref(network4); -+ -+#if 0 - loc_network_tree_unref(tree); -+#endif -=20 - // And open it again from disk - struct loc_database* db; -diff --git a/src/writer.c b/src/writer.c -index 5939cff..c61a6df 100644 ---- a/src/writer.c -+++ b/src/writer.c -@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) { - EVP_PKEY_free(writer->private_key2); -=20 - // Unref all AS -- for (unsigned int i =3D 0; i < writer->as_count; i++) { -- loc_as_unref(writer->as[i]); -+ if (writer->as) { -+ for (unsigned int i =3D 0; i < writer->as_count; i++) { -+ loc_as_unref(writer->as[i]); -+ } -+ free(writer->as); -+ } -+ -+ // Unref all countries -+ if (writer->countries) { -+ for (unsigned int i =3D 0; i < writer->countries_count; i++) { -+ loc_country_unref(writer->countries[i]); -+ } -+ free(writer->countries); - } -=20 - // Release network tree -@@ -601,7 +612,7 @@ static int loc_writer_create_signature(struct loc_writer= * writer, - goto END; - } -=20 -- DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n", *len= gth); -+ DEBUG(writer->ctx, "Successfully generated signature of %zu bytes\n", *len= gth); - r =3D 0; -=20 - // Dump signature diff --git a/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch = b/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch new file mode 100644 index 000000000..ea29885a8 --- /dev/null +++ b/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch @@ -0,0 +1,27 @@ +From b178117bac33b4b1e7ce341a6f2eec493cca13f8 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Mon, 21 Dec 2020 16:25:46 +0000 +Subject: [PATCH] location: Fix list-networks-by-as + +Fixes: #12554 +Signed-off-by: Michael Tremer +--- + src/python/location.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/python/location.in b/src/python/location.in +index b30beae..ad2ccf5 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -493,7 +493,7 @@ class CLI(object): + f =3D writer(sys.stdout, prefix=3D"AS%s" % asn) +=20 + # Print all matching networks +- for n in db.search_networks(asn=3Dasn, family=3Dns.family): ++ for n in db.search_networks(asns=3D[asn], family=3Dns.family): + f.write(n) +=20 + f.finish() +--=20 +2.26.2 + --=20 2.26.2 --===============2053765717606856132==--