[PATCH] libloc: update to 0.9.5 and backport fix for #12554

This patch updates libloc to 0.9.5, deletes the upstream patchset from version 0.9.4, and includes a latest upstream patch to backport a fix for #12554.

Cc: Michael Tremer michael.tremer@ipfire.org Signed-off-by: Peter Müller peter.mueller@ipfire.org --- lfs/libloc | 14 +- src/patches/libloc-0.9.4-upstream.patch | 3850 ----------------- ...9.5-location-Fix-list-networks-by-as.patch | 27 + 3 files changed, 34 insertions(+), 3857 deletions(-) delete mode 100644 src/patches/libloc-0.9.4-upstream.patch create mode 100644 src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch

diff --git a/lfs/libloc b/lfs/libloc index 010ddf1cb..279840535 100644 --- a/lfs/libloc +++ b/lfs/libloc @@ -24,8 +24,8 @@

include Config

-VER = 0.9.4 -DB_DATE = 2020-09-21 +VER = 0.9.5 +DB_DATE = 2020-12-23

THISAPP = libloc-$(VER) DL_FILE = $(THISAPP).tar.gz @@ -40,11 +40,11 @@ TARGET = $(DIR_INFO)/$(THISAPP) objects = $(DL_FILE) \ location-$(DB_DATE).db.xz

-$(DL_FILE) = https://source.ipfire.org/releases/libloc//$(DL_FILE) +$(DL_FILE) = https://source.ipfire.org/releases/libloc/$(DL_FILE) location-$(DB_DATE).db.xz = https://location.ipfire.org/databases/1/archive/location-$(DB_DATE).db.xz

-$(DL_FILE)_MD5 = 82770e9eba20f636c96e6fa42ff234b5 -location-$(DB_DATE).db.xz_MD5 = fa3069bf31170629d638317e283913c0 +$(DL_FILE)_MD5 = 41d8dc3fb4e498db958b7696cadd61f5 +location-$(DB_DATE).db.xz_MD5 = 3a71931555623fc3665f280c8e5c292a

install : $(TARGET)

@@ -78,8 +78,8 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) @$(PREBUILD) @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE)

- # Import changes from upstream - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.4-upstream.patch + # Import recent patches from upstream + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch

# Add patch for i586 to disable strong stack protector. ifeq "$(BUILD_ARCH)" "i586" diff --git a/src/patches/libloc-0.9.4-upstream.patch b/src/patches/libloc-0.9.4-upstream.patch deleted file mode 100644 index a6ec1066b..000000000 --- a/src/patches/libloc-0.9.4-upstream.patch +++ /dev/null @@ -1,3850 +0,0 @@ -diff --git a/Makefile.am b/Makefile.am -index a0431a6..dc594f8 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -91,11 +91,14 @@ EXTRA_DIST += \ - pkginclude_HEADERS = \ - src/loc/libloc.h \ - src/loc/as.h \ -+ src/loc/as-list.h \ - src/loc/compat.h \ - src/loc/country.h \ -+ src/loc/country-list.h \ - src/loc/database.h \ - src/loc/format.h \ - src/loc/network.h \ -+ src/loc/network-list.h \ - src/loc/private.h \ - src/loc/stringpool.h \ - src/loc/resolv.h \ -@@ -107,9 +110,12 @@ lib_LTLIBRARIES = \ - src_libloc_la_SOURCES = \ - src/libloc.c \ - src/as.c \ -+ src/as-list.c \ - src/country.c \ -+ src/country-list.c \ - src/database.c \ - src/network.c \ -+ src/network-list.c \ - src/resolv.c \ - src/stringpool.c \ - src/writer.c -@@ -312,6 +318,7 @@ check_PROGRAMS = \ - src/test-database \ - src/test-as \ - src/test-network \ -+ src/test-network-list \ - src/test-country \ - src/test-signature - -@@ -351,6 +358,15 @@ src_test_network_CFLAGS = \ - src_test_network_LDADD = \ - src/libloc.la - -+src_test_network_list_SOURCES = \ -+ src/test-network-list.c -+ -+src_test_network_list_CFLAGS = \ -+ $(TESTS_CFLAGS) -+ -+src_test_network_list_LDADD = \ -+ src/libloc.la -+ - src_test_stringpool_SOURCES = \ - src/test-stringpool.c - -@@ -390,7 +406,7 @@ MANPAGES_XML = $(patsubst %.txt,%.xml,$(MANPAGES_TXT)) - .PHONY: man - man: $(MANPAGES) $(MANPAGES_HTML) - --if ENABLE_MANPAGES -+if ENABLE_MAN_PAGES - man_MANS = \ - $(MANPAGES) - endif -diff --git a/configure.ac b/configure.ac -index 2364dfd..9eb9012 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1,6 +1,6 @@ - AC_PREREQ(2.60) - AC_INIT([libloc], -- [0.9.4], -+ [0.9.5], - [location@lists.ipfire.org], - [libloc], - [https://location.ipfire.org/]) -@@ -43,16 +43,16 @@ AC_PROG_MKDIR_P - - # - man ------------------------------------------------------------------------ - --have_manpages=no --AC_ARG_ENABLE(manpages, AS_HELP_STRING([--disable-man-pages], -+have_man_pages=no -+AC_ARG_ENABLE(man_pages, AS_HELP_STRING([--disable-man-pages], - [do not install man pages])) --AS_IF([test "x$enable_manpages" != xno], [have_manpages=yes]) --AM_CONDITIONAL(ENABLE_MANPAGES, [test "x$have_manpages" = "xyes"]) -+AS_IF([test "x$enable_man_pages" != xno], [have_man_pages=yes]) -+AM_CONDITIONAL(ENABLE_MAN_PAGES, [test "x$have_man_pages" = "xyes"]) - - AC_PATH_PROG([XSLTPROC], [xsltproc]) - - AC_CHECK_PROGS(ASCIIDOC, [asciidoc]) --if test "${have_manpages}" = "yes" && test -z "${ASCIIDOC}"; then -+if test "${have_man_pages}" = "yes" && test -z "${ASCIIDOC}"; then - AC_MSG_ERROR([Required program 'asciidoc' not found]) - fi - # - debug ---------------------------------------------------------------------- -diff --git a/src/.gitignore b/src/.gitignore -index caf80b5..3ccbdb8 100644 ---- a/src/.gitignore -+++ b/src/.gitignore -@@ -10,5 +10,6 @@ test-libloc - test-database - test-country - test-network -+test-network-list - test-signature - test-stringpool -diff --git a/src/as-list.c b/src/as-list.c -new file mode 100644 -index 0000000..5acbb8a ---- /dev/null -+++ b/src/as-list.c -@@ -0,0 +1,161 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team info@ipfire.org -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include <errno.h> -+#include <stdlib.h> -+ -+#include <loc/as.h> -+#include <loc/as-list.h> -+#include <loc/private.h> -+ -+struct loc_as_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_as** elements; -+ size_t elements_size; -+ -+ size_t size; -+}; -+ -+static int loc_as_list_grow(struct loc_as_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); -+ -+ struct loc_as** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; -+ -+ list->elements = elements; -+ list->elements_size += size; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, -+ struct loc_as_list** list) { -+ struct loc_as_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; -+ -+ DEBUG(l->ctx, "AS list allocated at %p\n", l); -+ *list = l; -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_as_list_free(struct loc_as_list* list) { -+ DEBUG(list->ctx, "Releasing AS list at %p\n", list); -+ -+ loc_as_list_clear(list); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_as_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) { -+ return list->size == 0; -+} -+ -+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) { -+ if (!list->elements) -+ return; -+ -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_as_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements = NULL; -+ list->elements_size = 0; -+ -+ list->size = 0; -+} -+ -+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_as_ref(list->elements[index]); -+} -+ -+LOC_EXPORT int loc_as_list_append( -+ struct loc_as_list* list, struct loc_as* as) { -+ if (loc_as_list_contains(list, as)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size >= list->elements_size) { -+ int r = loc_as_list_grow(list, 64); -+ if (r) -+ return r; -+ } -+ -+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as); -+ -+ list->elements[list->size++] = loc_as_ref(as); -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_as_cmp(as, list->elements[i]) == 0) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number) { -+ struct loc_as* as; -+ -+ int r = loc_as_new(list->ctx, &as, number); -+ if (r) -+ return -1; -+ -+ r = loc_as_list_contains(list, as); -+ loc_as_unref(as); -+ -+ return r; -+} -diff --git a/src/as.c b/src/as.c -index e1fbb01..757bf3d 100644 ---- a/src/as.c -+++ b/src/as.c -@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) { - } - - LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) { -- as->name = strdup(name); -+ if (as->name) -+ free(as->name); -+ -+ if (name) -+ as->name = strdup(name); -+ else -+ as->name = NULL; - - return 0; - } -@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) { - if (!string) - return 1; - -+ // Cannot match anything when name is not set -+ if (!as->name) -+ return 1; -+ - // Search if string is in name - if (strcasestr(as->name, string) != NULL) - return 1; -diff --git a/src/country-list.c b/src/country-list.c -new file mode 100644 -index 0000000..cc36740 ---- /dev/null -+++ b/src/country-list.c -@@ -0,0 +1,161 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team info@ipfire.org -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include <errno.h> -+#include <stdlib.h> -+ -+#include <loc/country.h> -+#include <loc/country-list.h> -+#include <loc/private.h> -+ -+struct loc_country_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_country** elements; -+ size_t elements_size; -+ -+ size_t size; -+}; -+ -+static int loc_country_list_grow(struct loc_country_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); -+ -+ struct loc_country** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; -+ -+ list->elements = elements; -+ list->elements_size += size; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx, -+ struct loc_country_list** list) { -+ struct loc_country_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; -+ -+ DEBUG(l->ctx, "Country list allocated at %p\n", l); -+ *list = l; -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_country_list_free(struct loc_country_list* list) { -+ DEBUG(list->ctx, "Releasing country list at %p\n", list); -+ -+ loc_country_list_clear(list); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_country_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) { -+ return list->size == 0; -+} -+ -+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) { -+ if (!list->elements) -+ return; -+ -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_country_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements = NULL; -+ list->elements_size = 0; -+ -+ list->size = 0; -+} -+ -+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_country_ref(list->elements[index]); -+} -+ -+LOC_EXPORT int loc_country_list_append( -+ struct loc_country_list* list, struct loc_country* country) { -+ if (loc_country_list_contains(list, country)) -+ return 0; -+ -+ // Check if we have space left -+ if (list->size >= list->elements_size) { -+ int r = loc_country_list_grow(list, 64); -+ if (r) -+ return r; -+ } -+ -+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country); -+ -+ list->elements[list->size++] = loc_country_ref(country); -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country) { -+ for (unsigned int i = 0; i < list->size; i++) { -+ if (loc_country_cmp(country, list->elements[i]) == 0) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code) { -+ struct loc_country* country; -+ -+ int r = loc_country_new(list->ctx, &country, code); -+ if (r) -+ return -1; -+ -+ r = loc_country_list_contains(list, country); -+ loc_country_unref(country); -+ -+ return r; -+} -diff --git a/src/country.c b/src/country.c -index 2ba93e6..7aac0db 100644 ---- a/src/country.c -+++ b/src/country.c -@@ -34,6 +34,9 @@ struct loc_country { - }; - - LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) { -+ if (!loc_country_code_is_valid(country_code)) -+ return -EINVAL; -+ - struct loc_country* c = calloc(1, sizeof(*c)); - if (!c) - return -ENOMEM; -diff --git a/src/database.c b/src/database.c -index fa1dad0..4b8bf1d 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -38,8 +38,10 @@ - - #include <loc/libloc.h> - #include <loc/as.h> -+#include <loc/as-list.h> - #include <loc/compat.h> - #include <loc/country.h> -+#include <loc/country-list.h> - #include <loc/database.h> - #include <loc/format.h> - #include <loc/network.h> -@@ -99,11 +101,14 @@ struct loc_database_enumerator { - - // Search string - char* string; -- char country_code[3]; -- uint32_t asn; -+ struct loc_country_list* countries; -+ struct loc_as_list* asns; - enum loc_network_flags flags; - int family; - -+ // Flatten output? -+ int flatten; -+ - // Index of the AS we are looking at - unsigned int as_index; - -@@ -115,6 +120,9 @@ struct loc_database_enumerator { - struct loc_node_stack network_stack[MAX_STACK_DEPTH]; - int network_stack_depth; - unsigned int* networks_visited; -+ -+ // For subnet search -+ struct loc_network_list* stack; - }; - - static int loc_database_read_magic(struct loc_database* db) { -@@ -242,11 +250,11 @@ static int loc_database_read_signature(struct loc_database* db, - char** dst, char* src, size_t length) { - // Check for a plausible signature length - if (length > LOC_SIGNATURE_MAX_LENGTH) { -- ERROR(db->ctx, "Signature too long: %ld\n", length); -+ ERROR(db->ctx, "Signature too long: %zu\n", length); - return -EINVAL; - } - -- DEBUG(db->ctx, "Reading signature of %ld bytes\n", length); -+ DEBUG(db->ctx, "Reading signature of %zu bytes\n", length); - - // Allocate space - *dst = malloc(length); -@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { - } - - clock_t end = clock(); -- DEBUG(db->ctx, "Signature checked in %.4fms\n", -+ INFO(db->ctx, "Signature checked in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); - - CLEANUP: -@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, - off_t lo = 0; - off_t hi = db->as_count - 1; - -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif - - while (lo <= hi) { - off_t i = (lo + hi) / 2; -@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, - // Check if this is a match - uint32_t as_number = loc_as_get_number(*as); - if (as_number == number) { -+#ifdef ENABLE_DEBUG - clock_t end = clock(); - - // Log how fast this has been - DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif - - return 0; - } -@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ - return -1; - } - -+#ifdef ENABLE_DEBUG - if (r == 0) { - char* string = loc_network_str(*network); - DEBUG(db->ctx, "Got network %s\n", string); - free(string); - } -+#endif - - return r; - } -@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru - } - - // Check if the given IP address is inside the network -- r = loc_network_match_address(*network, address); -- if (r) { -+ if (!loc_network_match_address(*network, address)) { - DEBUG(db->ctx, "Searched address is not part of the network\n"); - - loc_network_unref(*network); -@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db, - - *network = NULL; - -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif - - int r = __loc_database_lookup(db, address, network, &network_address, - db->network_nodes_v1, 0); - -+#ifdef ENABLE_DEBUG - clock_t end = clock(); - - // Log how fast this has been - DEBUG(db->ctx, "Executed network search in %.4fms\n", - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif - - return r; - } -@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - off_t lo = 0; - off_t hi = db->countries_count - 1; - -+#ifdef ENABLE_DEBUG - // Save start time - clock_t start = clock(); -+#endif - - while (lo <= hi) { - off_t i = (lo + hi) / 2; -@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - int result = strcmp(code, cc); - - if (result == 0) { -+#ifdef ENABLE_DEBUG - clock_t end = clock(); - - // Log how fast this has been - DEBUG(db->ctx, "Found country %s in %.4fms\n", cc, - (double)(end - start) / CLOCKS_PER_SEC * 1000); -+#endif - - return 0; - } -@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, - - // Enumerator - -+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { -+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -+ -+ // Release all references -+ loc_database_unref(enumerator->db); -+ loc_unref(enumerator->ctx); -+ -+ if (enumerator->string) -+ free(enumerator->string); -+ -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); -+ -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); -+ -+ // Free network search -+ free(enumerator->networks_visited); -+ -+ // Free subnet stack -+ if (enumerator->stack) -+ loc_network_list_unref(enumerator->stack); -+ -+ free(enumerator); -+} -+ - LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode) { -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { - struct loc_database_enumerator* e = calloc(1, sizeof(*e)); - if (!e) - return -ENOMEM; -@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum - e->mode = mode; - e->refcount = 1; - -+ // Flatten output? -+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN); -+ - // Initialise graph search -- //e->network_stack[++e->network_stack_depth] = 0; - e->network_stack_depth = 1; - e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); - -+ // Allocate stack -+ int r = loc_network_list_new(e->ctx, &e->stack); -+ if (r) { -+ loc_database_enumerator_free(e); -+ return r; -+ } -+ - DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e); - - *enumerator = e; -@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo - return enumerator; - } - --static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) { -- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator); -- -- // Release all references -- loc_database_unref(enumerator->db); -- loc_unref(enumerator->ctx); -- -- if (enumerator->string) -- free(enumerator->string); -- -- // Free network search -- free(enumerator->networks_visited); -- -- free(enumerator); --} -- - LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) { - if (!enumerator) - return NULL; -@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator - return 0; - } - --LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) { -- // Set empty country code -- if (!country_code || !*country_code) { -- *enumerator->country_code = '\0'; -- return 0; -- } -+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->countries) -+ return NULL; - -- // Treat A1, A2, A3 as special country codes, -- // but perform search for flags instead -- if (strcmp(country_code, "A1") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANONYMOUS_PROXY); -- } else if (strcmp(country_code, "A2") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_SATELLITE_PROVIDER); -- } else if (strcmp(country_code, "A3") == 0) { -- return loc_database_enumerator_set_flag(enumerator, -- LOC_NETWORK_FLAG_ANYCAST); -- } -+ return loc_country_list_ref(enumerator->countries); -+} - -- // Country codes must be two characters -- if (!loc_country_code_is_valid(country_code)) -- return -EINVAL; -+LOC_EXPORT int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) { -+ if (enumerator->countries) -+ loc_country_list_unref(enumerator->countries); - -- for (unsigned int i = 0; i < 3; i++) { -- enumerator->country_code[i] = country_code[i]; -- } -+ enumerator->countries = loc_country_list_ref(countries); - - return 0; - } - --LOC_EXPORT int loc_database_enumerator_set_asn( -- struct loc_database_enumerator* enumerator, unsigned int asn) { -- enumerator->asn = asn; -+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator) { -+ if (!enumerator->asns) -+ return NULL; -+ -+ return loc_as_list_ref(enumerator->asns); -+} -+ -+LOC_EXPORT int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) { -+ if (enumerator->asns) -+ loc_as_list_unref(enumerator->asns); -+ -+ enumerator->asns = loc_as_list_ref(asns); - - return 0; - } -@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node( - return 0; - } - --LOC_EXPORT int loc_database_enumerator_next_network( -- struct loc_database_enumerator* enumerator, struct loc_network** network) { -- // Reset network -- *network = NULL; -+static int loc_database_enumerator_filter_network( -+ struct loc_database_enumerator* enumerator, struct loc_network* network) { -+ // Skip if the family does not match -+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network); -+ return 1; -+ } - -- // Do not do anything if not in network mode -- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) -- return 0; -+ // Skip if the country code does not match -+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) { -+ const char* country_code = loc_network_get_country_code(network); - -- int r; -+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network); -+ return 1; -+ } -+ } -+ -+ // Skip if the ASN does not match -+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) { -+ uint32_t asn = loc_network_get_asn(network); -+ -+ if (!loc_as_list_contains_number(enumerator->asns, asn)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network); -+ return 1; -+ } -+ } -+ -+ // Skip if flags do not match -+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) { -+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network); -+ return 1; -+ } -+ -+ // Do not filter -+ return 0; -+} -+ -+static int __loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) { -+ // Return top element from the stack -+ while (1) { -+ *network = loc_network_list_pop_first(enumerator->stack); -+ -+ // Stack is empty -+ if (!*network) -+ break; -+ -+ // Throw away any networks by filter -+ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) { -+ loc_network_unref(*network); -+ *network = NULL; -+ continue; -+ } -+ -+ // Return result -+ return 0; -+ } - - DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n", - enumerator->network_stack_depth); -@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network( - enumerator->db->network_nodes_v1 + node->offset; - - // Add edges to stack -- r = loc_database_enumerator_stack_push_node(enumerator, -+ int r = loc_database_enumerator_stack_push_node(enumerator, - be32toh(n->one), 1, node->depth + 1); - - if (r) -@@ -1175,54 +1261,142 @@ LOC_EXPORT int loc_database_enumerator_next_network( - if (r) - return r; - -- // Check if we are interested in this network -+ // Return all networks when the filter is disabled -+ if (!filter) -+ return 0; - -- // Skip if the family does not match -- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) { -+ // Check if we are interested in this network -+ if (loc_database_enumerator_filter_network(enumerator, *network)) { - loc_network_unref(*network); - *network = NULL; - - continue; - } - -- // Skip if the country code does not match -- if (*enumerator->country_code && -- !loc_network_match_country_code(*network, enumerator->country_code)) { -- loc_network_unref(*network); -- *network = NULL; -+ return 0; -+ } -+ } - -- continue; -- } -+ // Reached the end of the search -+ return 0; -+} - -- // Skip if the ASN does not match -- if (enumerator->asn && -- !loc_network_match_asn(*network, enumerator->asn)) { -- loc_network_unref(*network); -- *network = NULL; -+static int __loc_database_enumerator_next_network_flattened( -+ struct loc_database_enumerator* enumerator, struct loc_network** network) { -+ // Fetch the next network -+ int r = __loc_database_enumerator_next_network(enumerator, network, 1); -+ if (r) -+ return r; - -- continue; -- } -+ // End if we could not read another network -+ if (!*network) -+ return 0; - -- // Skip if flags do not match -- if (enumerator->flags && -- !loc_network_match_flag(*network, enumerator->flags)) { -- loc_network_unref(*network); -- *network = NULL; -+ struct loc_network* subnet = NULL; -+ struct loc_network_list* subnets; - -- continue; -+ // Create a list with all subnets -+ r = loc_network_list_new(enumerator->ctx, &subnets); -+ if (r) -+ return r; -+ -+ // Search all subnets from the database -+ while (1) { -+ // Fetch the next network in line -+ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0); -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); -+ -+ return r; -+ } -+ -+ // End if we did not receive another subnet -+ if (!subnet) -+ break; -+ -+ // Collect all subnets in a list -+ if (loc_network_is_subnet(*network, subnet)) { -+ r = loc_network_list_push(subnets, subnet); -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); -+ -+ return r; - } - -- return 0; -+ loc_network_unref(subnet); -+ continue; -+ } -+ -+ // If this is not a subnet, we push it back onto the stack and break -+ r = loc_network_list_push(enumerator->stack, subnet); -+ if (r) { -+ loc_network_unref(subnet); -+ loc_network_list_unref(subnets); -+ -+ return r; - } -+ -+ loc_network_unref(subnet); -+ break; - } - -- // Reached the end of the search -+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets)); - -- // Mark all nodes as non-visited -- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++) -- enumerator->networks_visited[i] = 0; -+ // We can abort here if the network has no subnets -+ if (loc_network_list_empty(subnets)) { -+ loc_network_list_unref(subnets); - -- return 0; -+ return 0; -+ } -+ -+ // If the network has any subnets, we will break it into smaller parts -+ // without the subnets. -+ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets); -+ if (!excluded) { -+ loc_network_list_unref(subnets); -+ return -1; -+ } -+ -+ // Merge subnets onto the stack -+ r = loc_network_list_merge(enumerator->stack, subnets); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); -+ -+ return r; -+ } -+ -+ // Push excluded list onto the stack -+ r = loc_network_list_merge(enumerator->stack, excluded); -+ if (r) { -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); -+ -+ return r; -+ } -+ -+ loc_network_list_unref(subnets); -+ loc_network_list_unref(excluded); -+ -+ // Drop the network and restart the whole process again to pick the next network -+ loc_network_unref(*network); -+ -+ return __loc_database_enumerator_next_network_flattened(enumerator, network); -+} -+ -+LOC_EXPORT int loc_database_enumerator_next_network( -+ struct loc_database_enumerator* enumerator, struct loc_network** network) { -+ // Do not do anything if not in network mode -+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS) -+ return 0; -+ -+ // Flatten output? -+ if (enumerator->flatten) -+ return __loc_database_enumerator_next_network_flattened(enumerator, network); -+ -+ return __loc_database_enumerator_next_network(enumerator, network, 1); - } - - LOC_EXPORT int loc_database_enumerator_next_country( -diff --git a/src/libloc.sym b/src/libloc.sym -index b8296eb..ee333f1 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -37,6 +37,18 @@ global: - loc_as_set_name; - loc_as_unref; - -+ # AS List -+ loc_as_list_append; -+ loc_as_list_clear; -+ loc_as_list_contains; -+ loc_as_list_contains_number; -+ loc_as_list_empty; -+ loc_as_list_get; -+ loc_as_list_new; -+ loc_as_list_ref; -+ loc_as_list_size; -+ loc_as_list_unref; -+ - # Country - loc_country_cmp; - loc_country_code_is_valid; -@@ -49,6 +61,18 @@ global: - loc_country_set_name; - loc_country_unref; - -+ # Country List -+ loc_country_list_append; -+ loc_country_list_clear; -+ loc_country_list_contains; -+ loc_country_list_contains_code; -+ loc_country_list_empty; -+ loc_country_list_get; -+ loc_country_list_new; -+ loc_country_list_ref; -+ loc_country_list_size; -+ loc_country_list_unref; -+ - # Database - loc_database_add_as; - loc_database_count_as; -@@ -66,13 +90,15 @@ global: - loc_database_verify; - - # Database Enumerator -+ loc_database_enumerator_get_asns; -+ loc_database_enumerator_get_countries; - loc_database_enumerator_new; - loc_database_enumerator_next_as; - loc_database_enumerator_next_country; - loc_database_enumerator_next_network; - loc_database_enumerator_ref; -- loc_database_enumerator_set_asn; -- loc_database_enumerator_set_country_code; -+ loc_database_enumerator_set_asns; -+ loc_database_enumerator_set_countries; - loc_database_enumerator_set_family; - loc_database_enumerator_set_flag; - loc_database_enumerator_set_string; -@@ -80,24 +106,48 @@ global: - - # Network - loc_network_address_family; -+ loc_network_cmp; -+ loc_network_exclude; -+ loc_network_exclude_list; - loc_network_format_first_address; - loc_network_format_last_address; - loc_network_get_asn; - loc_network_get_country_code; -+ loc_network_get_first_address; -+ loc_network_get_last_address; - loc_network_has_flag; -- loc_network_is_subnet_of; -+ loc_network_is_subnet; -+ loc_network_match_address; - loc_network_match_asn; - loc_network_match_country_code; - loc_network_match_flag; - loc_network_new; - loc_network_new_from_string; -+ loc_network_overlaps; -+ loc_network_prefix; - loc_network_ref; - loc_network_set_asn; - loc_network_set_country_code; - loc_network_set_flag; - loc_network_str; -+ loc_network_subnets; - loc_network_unref; - -+ # Network List -+ loc_network_list_clear; -+ loc_network_list_contains; -+ loc_network_list_dump; -+ loc_network_list_empty; -+ loc_network_list_get; -+ loc_network_list_merge; -+ loc_network_list_new; -+ loc_network_list_pop; -+ loc_network_list_pop_first; -+ loc_network_list_push; -+ loc_network_list_ref; -+ loc_network_list_size; -+ loc_network_list_unref; -+ - # Writer - loc_writer_add_as; - loc_writer_add_country; -diff --git a/src/loc/as-list.h b/src/loc/as-list.h -new file mode 100644 -index 0000000..7b5c4e8 ---- /dev/null -+++ b/src/loc/as-list.h -@@ -0,0 +1,41 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team info@ipfire.org -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#ifndef LIBLOC_AS_LIST_H -+#define LIBLOC_AS_LIST_H -+ -+#include <loc/as.h> -+#include <loc/libloc.h> -+ -+struct loc_as_list; -+ -+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list); -+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list); -+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list); -+ -+size_t loc_as_list_size(struct loc_as_list* list); -+int loc_as_list_empty(struct loc_as_list* list); -+void loc_as_list_clear(struct loc_as_list* list); -+ -+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index); -+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as); -+ -+int loc_as_list_contains( -+ struct loc_as_list* list, struct loc_as* as); -+int loc_as_list_contains_number( -+ struct loc_as_list* list, uint32_t number); -+ -+#endif -diff --git a/src/loc/country-list.h b/src/loc/country-list.h -new file mode 100644 -index 0000000..a7f818a ---- /dev/null -+++ b/src/loc/country-list.h -@@ -0,0 +1,43 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team info@ipfire.org -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#ifndef LIBLOC_COUNTRY_LIST_H -+#define LIBLOC_COUNTRY_LIST_H -+ -+#include <stdlib.h> -+ -+#include <loc/libloc.h> -+#include <loc/country.h> -+ -+struct loc_country_list; -+ -+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list); -+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list); -+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list); -+ -+size_t loc_country_list_size(struct loc_country_list* list); -+int loc_country_list_empty(struct loc_country_list* list); -+void loc_country_list_clear(struct loc_country_list* list); -+ -+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index); -+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country); -+ -+int loc_country_list_contains( -+ struct loc_country_list* list, struct loc_country* country); -+int loc_country_list_contains_code( -+ struct loc_country_list* list, const char* code); -+ -+#endif -diff --git a/src/loc/database.h b/src/loc/database.h -index 43173dd..70801f0 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -25,6 +25,7 @@ - #include <loc/network.h> - #include <loc/as.h> - #include <loc/country.h> -+#include <loc/country-list.h> - - struct loc_database; - int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f); -@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode { - LOC_DB_ENUMERATE_COUNTRIES = 3, - }; - -+enum loc_database_enumerator_flags { -+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0), -+}; -+ - struct loc_database_enumerator; - int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, -- struct loc_database* db, enum loc_database_enumerator_mode mode); -+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags); - struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator); - struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator); - - int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string); --int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code); --int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn); -+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator); -+int loc_database_enumerator_set_countries( -+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries); -+struct loc_as_list* loc_database_enumerator_get_asns( -+ struct loc_database_enumerator* enumerator); -+int loc_database_enumerator_set_asns( -+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns); - int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag); - int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family); - int loc_database_enumerator_next_as( -diff --git a/src/loc/network-list.h b/src/loc/network-list.h -new file mode 100644 -index 0000000..bee21c4 ---- /dev/null -+++ b/src/loc/network-list.h -@@ -0,0 +1,37 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team info@ipfire.org -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#ifndef LIBLOC_NETWORK_LIST_H -+#define LIBLOC_NETWORK_LIST_H -+ -+#include <loc/network.h> -+ -+struct loc_network_list; -+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list); -+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list); -+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list); -+size_t loc_network_list_size(struct loc_network_list* list); -+int loc_network_list_empty(struct loc_network_list* list); -+void loc_network_list_clear(struct loc_network_list* list); -+void loc_network_list_dump(struct loc_network_list* list); -+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index); -+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network); -+struct loc_network* loc_network_list_pop(struct loc_network_list* list); -+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list); -+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network); -+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other); -+ -+#endif -diff --git a/src/loc/network.h b/src/loc/network.h -index 70c3803..af3dafd 100644 ---- a/src/loc/network.h -+++ b/src/loc/network.h -@@ -21,6 +21,7 @@ - - #include <loc/libloc.h> - #include <loc/format.h> -+#include <loc/network-list.h> - - enum loc_network_flags { - LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 -@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network); - struct loc_network* loc_network_unref(struct loc_network* network); - char* loc_network_str(struct loc_network* network); - int loc_network_address_family(struct loc_network* network); -+unsigned int loc_network_prefix(struct loc_network* network); - -+const struct in6_addr* loc_network_get_first_address(struct loc_network* network); - char* loc_network_format_first_address(struct loc_network* network); -+const struct in6_addr* loc_network_get_last_address(struct loc_network* network); - char* loc_network_format_last_address(struct loc_network* network); - int loc_network_match_address(struct loc_network* network, const struct in6_addr* address); - -@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag); - int loc_network_set_flag(struct loc_network* network, uint32_t flag); - int loc_network_match_flag(struct loc_network* network, uint32_t flag); - --int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other); -+int loc_network_cmp(struct loc_network* self, struct loc_network* other); -+int loc_network_overlaps(struct loc_network* self, struct loc_network* other); -+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other); -+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2); -+struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other); -+struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list); - - #ifdef LIBLOC_PRIVATE - -diff --git a/src/network-list.c b/src/network-list.c -new file mode 100644 -index 0000000..698d3ab ---- /dev/null -+++ b/src/network-list.c -@@ -0,0 +1,299 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2020 IPFire Development Team info@ipfire.org -+ -+ This library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ This library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+*/ -+ -+#include <errno.h> -+#include <stdlib.h> -+#include <time.h> -+ -+#include <loc/libloc.h> -+#include <loc/network.h> -+#include <loc/private.h> -+ -+struct loc_network_list { -+ struct loc_ctx* ctx; -+ int refcount; -+ -+ struct loc_network** elements; -+ size_t elements_size; -+ -+ size_t size; -+}; -+ -+static int loc_network_list_grow(struct loc_network_list* list, size_t size) { -+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n", -+ list, size, list->elements_size + size); -+ -+ struct loc_network** elements = reallocarray(list->elements, -+ list->elements_size + size, sizeof(*list->elements)); -+ if (!elements) -+ return -errno; -+ -+ list->elements = elements; -+ list->elements_size += size; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx, -+ struct loc_network_list** list) { -+ struct loc_network_list* l = calloc(1, sizeof(*l)); -+ if (!l) -+ return -ENOMEM; -+ -+ l->ctx = loc_ref(ctx); -+ l->refcount = 1; -+ -+ DEBUG(l->ctx, "Network list allocated at %p\n", l); -+ *list = l; -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) { -+ list->refcount++; -+ -+ return list; -+} -+ -+static void loc_network_list_free(struct loc_network_list* list) { -+ DEBUG(list->ctx, "Releasing network list at %p\n", list); -+ -+ // Remove all content -+ loc_network_list_clear(list); -+ -+ loc_unref(list->ctx); -+ free(list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { -+ if (!list) -+ return NULL; -+ -+ if (--list->refcount > 0) -+ return list; -+ -+ loc_network_list_free(list); -+ return NULL; -+} -+ -+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) { -+ return list->size; -+} -+ -+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) { -+ return list->size == 0; -+} -+ -+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) { -+ if (!list->elements) -+ return; -+ -+ for (unsigned int i = 0; i < list->size; i++) -+ loc_network_unref(list->elements[i]); -+ -+ free(list->elements); -+ list->elements = NULL; -+ list->elements_size = 0; -+ -+ list->size = 0; -+} -+ -+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) { -+ struct loc_network* network; -+ char* s; -+ -+ for (unsigned int i = 0; i < list->size; i++) { -+ network = list->elements[i]; -+ -+ s = loc_network_str(network); -+ -+ INFO(list->ctx, "%4d: %s\n", i, s); -+ free(s); -+ } -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) { -+ // Check index -+ if (index >= list->size) -+ return NULL; -+ -+ return loc_network_ref(list->elements[index]); -+} -+ -+static off_t loc_network_list_find(struct loc_network_list* list, -+ struct loc_network* network, int* found) { -+ // Insert at the beginning for an empty list -+ if (loc_network_list_empty(list)) -+ return 0; -+ -+ off_t lo = 0; -+ off_t hi = list->size - 1; -+ int result; -+ -+ // Since we are working on an ordered list, there is often a good chance that -+ // the network we are looking for is at the end or has to go to the end. -+ if (hi >= 0) { -+ result = loc_network_cmp(network, list->elements[hi]); -+ -+ // Match, so we are done -+ if (result == 0) { -+ *found = 1; -+ -+ return hi; -+ -+ // This needs to be added after the last one -+ } else if (result > 0) { -+ *found = 0; -+ -+ return hi + 1; -+ } -+ } -+ -+#ifdef ENABLE_DEBUG -+ // Save start time -+ clock_t start = clock(); -+#endif -+ -+ off_t i = 0; -+ -+ while (lo <= hi) { -+ i = (lo + hi) / 2; -+ -+ // Check if this is a match -+ result = loc_network_cmp(network, list->elements[i]); -+ -+ if (result == 0) { -+ *found = 1; -+ -+#ifdef ENABLE_DEBUG -+ clock_t end = clock(); -+ -+ // Log how fast this has been -+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif -+ -+ return i; -+ } -+ -+ if (result > 0) { -+ lo = i + 1; -+ i++; -+ } else { -+ hi = i - 1; -+ } -+ } -+ -+ *found = 0; -+ -+#ifdef ENABLE_DEBUG -+ clock_t end = clock(); -+ -+ // Log how fast this has been -+ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n", -+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i); -+#endif -+ -+ return i; -+} -+ -+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) { -+ int found = 0; -+ -+ off_t index = loc_network_list_find(list, network, &found); -+ -+ // The network has been found on the list. Nothing to do. -+ if (found) -+ return 0; -+ -+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n", -+ list, network, (intmax_t)index); -+ -+ // Check if we have space left -+ if (list->size >= list->elements_size) { -+ int r = loc_network_list_grow(list, 64); -+ if (r) -+ return r; -+ } -+ -+ // The list is now larger -+ list->size++; -+ -+ // Move all elements out of the way -+ for (unsigned int i = list->size - 1; i > index; i--) -+ list->elements[i] = list->elements[i - 1]; -+ -+ // Add the new element at the right place -+ list->elements[index] = loc_network_ref(network); -+ -+ return 0; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->elements[--list->size]; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ -+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) { -+ // Return nothing when empty -+ if (loc_network_list_empty(list)) { -+ DEBUG(list->ctx, "%p: Popped empty stack\n", list); -+ return NULL; -+ } -+ -+ struct loc_network* network = list->elements[0]; -+ -+ // Move all elements to the top of the stack -+ for (unsigned int i = 0; i < list->size - 1; i++) { -+ list->elements[i] = list->elements[i+1]; -+ } -+ -+ // The list is shorter now -+ --list->size; -+ -+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network); -+ -+ return network; -+} -+ -+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) { -+ int found = 0; -+ -+ loc_network_list_find(list, network, &found); -+ -+ return found; -+} -+ -+LOC_EXPORT int loc_network_list_merge( -+ struct loc_network_list* self, struct loc_network_list* other) { -+ int r; -+ -+ for (unsigned int i = 0; i < other->size; i++) { -+ r = loc_network_list_push(self, other->elements[i]); -+ if (r) -+ return r; -+ } -+ -+ return 0; -+} -diff --git a/src/network.c b/src/network.c -index 366caa2..a6b679c 100644 ---- a/src/network.c -+++ b/src/network.c -@@ -29,6 +29,7 @@ - #include <loc/compat.h> - #include <loc/country.h> - #include <loc/network.h> -+#include <loc/network-list.h> - #include <loc/private.h> - - struct loc_network { -@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s - return a; - } - -+static struct in6_addr address_increment(const struct in6_addr* address) { -+ struct in6_addr a = *address; -+ -+ for (int octet = 15; octet >= 0; octet--) { -+ if (a.s6_addr[octet] < 255) { -+ a.s6_addr[octet]++; -+ break; -+ } else { -+ a.s6_addr[octet] = 0; -+ } -+ } -+ -+ return a; -+} -+ - LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network, - struct in6_addr* address, unsigned int prefix) { - // Address cannot be unspecified -@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network - LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network, - const char* address_string) { - struct in6_addr first_address; -- unsigned int prefix = 0; - char* prefix_string; -- int r = 1; -+ unsigned int prefix = 128; -+ int r = -EINVAL; -+ -+ DEBUG(ctx, "Attempting to parse network %s\n", address_string); - - // Make a copy of the string to work on it - char* buffer = strdup(address_string); -@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo - // Split address and prefix - address_string = strsep(&prefix_string, "/"); - -- // Did we find a prefix? -+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string); -+ -+ // Parse the address -+ r = loc_parse_address(ctx, address_string, &first_address); -+ if (r) { -+ DEBUG(ctx, "The address could not be parsed\n"); -+ goto FAIL; -+ } -+ -+ // If a prefix was given, we will try to parse it - if (prefix_string) { - // Convert prefix to integer - prefix = strtol(prefix_string, NULL, 10); - -- if (prefix) { -- // Parse the address -- r = loc_parse_address(ctx, address_string, &first_address); -- -- // Map the prefix to IPv6 if needed -- if (IN6_IS_ADDR_V4MAPPED(&first_address)) -- prefix += 96; -+ if (!prefix) { -+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string); -+ goto FAIL; - } -+ -+ // Map the prefix to IPv6 if needed -+ if (IN6_IS_ADDR_V4MAPPED(&first_address)) -+ prefix += 96; - } - -+FAIL: - // Free temporary buffer - free(buffer); - -- if (r == 0) { -- r = loc_network_new(ctx, network, &first_address, prefix); -- } -+ // Exit if the parsing was unsuccessful -+ if (r) -+ return r; - -- return r; -+ // Create a new network -+ return loc_network_new(ctx, network, &first_address, prefix); - } - - LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) { -@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) { - return network->family; - } - -+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) { -+ switch (network->family) { -+ case AF_INET6: -+ return network->prefix; -+ -+ case AF_INET: -+ return network->prefix - 96; -+ } -+ -+ return 0; -+} -+ - static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) { - const size_t length = INET6_ADDRSTRLEN; - -@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc - return string; - } - -+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) { -+ return &network->first_address; -+} -+ - LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) { - return loc_network_format_address(network, &network->first_address); - } - -+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) { -+ return &network->last_address; -+} -+ - LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { - return loc_network_format_address(network, &network->last_address); - } -@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) { - LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) { - // Address must be larger than the start address - if (in6_addr_cmp(&network->first_address, address) > 0) -- return 1; -+ return 0; - - // Address must be smaller than the last address - if (in6_addr_cmp(&network->last_address, address) < 0) -- return 1; -+ return 0; - - // The address is inside this network -- return 0; -+ return 1; - } - - LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) { -@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag - return loc_network_has_flag(network, flag); - } - --LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) { -+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) { -+ // Compare address -+ int r = in6_addr_cmp(&self->first_address, &other->first_address); -+ if (r) -+ return r; -+ -+ // Compare prefix -+ if (self->prefix > other->prefix) -+ return 1; -+ else if (self->prefix < other->prefix) -+ return -1; -+ -+ // Both networks are equal -+ return 0; -+} -+ -+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) { -+ // Either of the start addresses must be in the other subnet -+ if (loc_network_match_address(self, &other->first_address)) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->first_address)) -+ return 1; -+ -+ // Or either of the end addresses is in the other subnet -+ if (loc_network_match_address(self, &other->last_address)) -+ return 1; -+ -+ if (loc_network_match_address(other, &self->last_address)) -+ return 1; -+ -+ return 0; -+} -+ -+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) { -+ // The prefix must be smaller (this avoids the more complex comparisons later) -+ if (self->prefix > other->prefix) -+ return 0; -+ - // If the start address of the other network is smaller than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0) -+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0) - return 0; - - // If the end address of the other network is greater than this network, - // it cannot be a subnet. -- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0) -+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0) - return 0; - - return 1; - } - -+LOC_EXPORT int loc_network_subnets(struct loc_network* network, -+ struct loc_network** subnet1, struct loc_network** subnet2) { -+ int r; -+ *subnet1 = NULL; -+ *subnet2 = NULL; -+ -+ // New prefix length -+ unsigned int prefix = network->prefix + 1; -+ -+ // Check if the new prefix is valid -+ if (valid_prefix(&network->first_address, prefix)) -+ return -1; -+ -+ // Create the first half of the network -+ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix); -+ if (r) -+ return r; -+ -+ // The next subnet starts after the first one -+ struct in6_addr first_address = address_increment(&(*subnet1)->last_address); -+ -+ // Create the second half of the network -+ r = loc_network_new(network->ctx, subnet2, &first_address, prefix); -+ if (r) -+ return r; -+ -+ // Copy country code -+ const char* country_code = loc_network_get_country_code(network); -+ if (country_code) { -+ loc_network_set_country_code(*subnet1, country_code); -+ loc_network_set_country_code(*subnet2, country_code); -+ } -+ -+ // Copy ASN -+ uint32_t asn = loc_network_get_asn(network); -+ if (asn) { -+ loc_network_set_asn(*subnet1, asn); -+ loc_network_set_asn(*subnet2, asn); -+ } -+ -+ // Copy flags -+ loc_network_set_flag(*subnet1, network->flags); -+ loc_network_set_flag(*subnet2, network->flags); -+ -+ return 0; -+} -+ -+static int __loc_network_exclude(struct loc_network* network, -+ struct loc_network* other, struct loc_network_list* list) { -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; -+ -+ int r = loc_network_subnets(network, &subnet1, &subnet2); -+ if (r) -+ goto ERROR; -+ -+ if (loc_network_cmp(other, subnet1) == 0) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_cmp(other, subnet2) == 0) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_is_subnet(subnet1, other)) { -+ r = loc_network_list_push(list, subnet2); -+ if (r) -+ goto ERROR; -+ -+ r = __loc_network_exclude(subnet1, other, list); -+ if (r) -+ goto ERROR; -+ -+ } else if (loc_network_is_subnet(subnet2, other)) { -+ r = loc_network_list_push(list, subnet1); -+ if (r) -+ goto ERROR; -+ -+ r = __loc_network_exclude(subnet2, other, list); -+ if (r) -+ goto ERROR; -+ -+ } else { -+ ERROR(network->ctx, "We should never get here\n"); -+ r = 1; -+ goto ERROR; -+ } -+ -+ERROR: -+ if (subnet1) -+ loc_network_unref(subnet1); -+ -+ if (subnet2) -+ loc_network_unref(subnet2); -+ -+ return r; -+} -+ -+static int __loc_network_exclude_to_list(struct loc_network* self, -+ struct loc_network* other, struct loc_network_list* list) { -+ // Other must be a subnet of self -+ if (!loc_network_is_subnet(self, other)) { -+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self); -+ -+ // Exit silently -+ return 0; -+ } -+ -+ // We cannot perform this operation if both networks equal -+ if (loc_network_cmp(self, other) == 0) { -+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other); -+ -+ // Exit silently -+ return 0; -+ } -+ -+ return __loc_network_exclude(self, other, list); -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude( -+ struct loc_network* self, struct loc_network* other) { -+ struct loc_network_list* list; -+ -+#ifdef ENABLE_DEBUG -+ char* n1 = loc_network_str(self); -+ char* n2 = loc_network_str(other); -+ -+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2); -+ -+ free(n1); -+ free(n2); -+#endif -+ -+ // Create a new list with the result -+ int r = loc_network_list_new(self->ctx, &list); -+ if (r) { -+ ERROR(self->ctx, "Could not create network list: %d\n", r); -+ -+ return NULL; -+ } -+ -+ r = __loc_network_exclude_to_list(self, other, list); -+ if (r) { -+ loc_network_list_unref(list); -+ -+ return NULL; -+ } -+ -+ // Return the result -+ return list; -+} -+ -+LOC_EXPORT struct loc_network_list* loc_network_exclude_list( -+ struct loc_network* network, struct loc_network_list* list) { -+ struct loc_network_list* to_check; -+ -+ // Create a new list with all networks to look at -+ int r = loc_network_list_new(network->ctx, &to_check); -+ if (r) -+ return NULL; -+ -+ struct loc_network* subnet = NULL; -+ struct loc_network_list* subnets = NULL; -+ -+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) { -+ subnet = loc_network_list_get(list, i); -+ -+ // Find all excluded networks -+ if (!loc_network_list_contains(to_check, subnet)) { -+ r = __loc_network_exclude_to_list(network, subnet, to_check); -+ if (r) { -+ loc_network_list_unref(to_check); -+ loc_network_unref(subnet); -+ -+ return NULL; -+ } -+ } -+ -+ // Cleanup -+ loc_network_unref(subnet); -+ } -+ -+ r = loc_network_list_new(network->ctx, &subnets); -+ if (r) { -+ loc_network_list_unref(to_check); -+ return NULL; -+ } -+ -+ off_t smallest_subnet = 0; -+ -+ while (!loc_network_list_empty(to_check)) { -+ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check); -+ -+ // Check whether the subnet to check is part of the input list -+ if (loc_network_list_contains(list, subnet_to_check)) { -+ loc_network_unref(subnet_to_check); -+ continue; -+ } -+ -+ // Marks whether this subnet passed all checks -+ int passed = 1; -+ -+ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) { -+ subnet = loc_network_list_get(list, i); -+ -+ // Drop this subnet if is a subnet of another subnet -+ if (loc_network_is_subnet(subnet, subnet_to_check)) { -+ passed = 0; -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // Break it down if it overlaps -+ if (loc_network_overlaps(subnet, subnet_to_check)) { -+ passed = 0; -+ -+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check); -+ -+ loc_network_unref(subnet); -+ break; -+ } -+ -+ // If the subnet is strictly greater, we do not need to continue the search -+ r = loc_network_cmp(subnet, subnet_to_check); -+ if (r > 0) { -+ loc_network_unref(subnet); -+ break; -+ -+ // If it is strictly smaller, we can continue the search from here next -+ // time because all networks that are to be checked can only be larger -+ // than this one. -+ } else if (r < 0) { -+ smallest_subnet = i; -+ } -+ -+ loc_network_unref(subnet); -+ } -+ -+ if (passed) { -+ r = loc_network_list_push(subnets, subnet_to_check); -+ } -+ -+ loc_network_unref(subnet_to_check); -+ } -+ -+ loc_network_list_unref(to_check); -+ -+ return subnets; -+} -+ - LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) { - // Add country code - loc_country_code_copy(dbobj->country_code, network->country_code); -@@ -474,7 +813,7 @@ struct loc_network_tree_node { - struct loc_network* network; - }; - --LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { -+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { - struct loc_network_tree* t = calloc(1, sizeof(*t)); - if (!t) - return -ENOMEM; -@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree - return 0; - } - --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { -+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) { - return loc_network_tree_node_ref(tree->root); - } - -@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_ - return 0; - } - --LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree, -+int loc_network_tree_walk(struct loc_network_tree* tree, - int(*filter_callback)(struct loc_network* network, void* data), - int(*callback)(struct loc_network* network, void* data), void* data) { - return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data); -@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) { - free(tree); - } - --LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { -+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) { - if (--tree->refcount > 0) - return tree; - -@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) { - return 0; - } - --LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) { -+int loc_network_tree_dump(struct loc_network_tree* tree) { - DEBUG(tree->ctx, "Dumping network tree at %p\n", tree); - - return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL); - } - --LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { -+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) { - DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree); - - struct loc_network_tree_node* node = loc_network_tree_get_path(tree, -@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) { - return 0; - } - --LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { -+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) { - size_t counter = 0; - - int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter); -@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node) - return counter; - } - --LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { -+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) { - return __loc_network_tree_count_nodes(tree->root); - } - --LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { -+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) { - struct loc_network_tree_node* n = calloc(1, sizeof(*n)); - if (!n) - return -ENOMEM; -@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network - return 0; - } - --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) { - if (node) - node->refcount++; - -@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { - free(node); - } - --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { -+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { - if (!node) - return NULL; - -@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_ - return NULL; - } - --LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { -+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) { - if (index == 0) - node = node->zero; - else -@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne - return loc_network_tree_node_ref(node); - } - --LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { -+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) { - return (!!node->network); - } - --LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { -+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) { - return loc_network_ref(node->network); - } -diff --git a/src/perl/Location.xs b/src/perl/Location.xs -index dcf3f0d..b7676d2 100644 ---- a/src/perl/Location.xs -+++ b/src/perl/Location.xs -@@ -125,7 +125,7 @@ database_countries(db) - PPCODE: - // Create Database enumerator - struct loc_database_enumerator* enumerator; -- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES); -+ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0); - - if (err) { - croak("Could not create a database enumerator\n"); -diff --git a/src/python/database.c b/src/python/database.c -index 1013a58..0aa03cc 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -17,6 +17,8 @@ - #include <Python.h> - - #include <loc/libloc.h> -+#include <loc/as.h> -+#include <loc/as-list.h> - #include <loc/database.h> - - #include "locationmodule.h" -@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database - return (PyObject*)self; - } - --static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) { -+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) { - struct loc_database_enumerator* enumerator; - -- int r = loc_database_enumerator_new(&enumerator, self->db, what); -+ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en - } - - static PyObject* Database_ases(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0); - } - - static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { -@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { - - struct loc_database_enumerator* enumerator; - -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; -@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) { - } - - static PyObject* Database_networks(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0); -+} -+ -+static PyObject* Database_networks_flattened(DatabaseObject *self) { -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN); - } - - static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) { -- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL }; -- const char* country_code = NULL; -- unsigned int asn = 0; -+ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL }; -+ PyObject* country_codes = NULL; -+ PyObject* asn_list = NULL; - int flags = 0; - int family = 0; -+ int flatten = 0; - -- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family)) -+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist, -+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten)) - return NULL; - - struct loc_database_enumerator* enumerator; -- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS); -+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, -+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); - return NULL; - } - - // Set country code we are searching for -- if (country_code) { -- r = loc_database_enumerator_set_country_code(enumerator, country_code); -+ if (country_codes) { -+ struct loc_country_list* countries; -+ r = loc_country_list_new(loc_ctx, &countries); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create country list"); -+ return NULL; -+ } -+ -+ for (int i = 0; i < PyList_Size(country_codes); i++) { -+ PyObject* item = PyList_GetItem(country_codes, i); -+ -+ if (!PyUnicode_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings"); -+ loc_country_list_unref(countries); -+ return NULL; -+ } -+ -+ const char* country_code = PyUnicode_AsUTF8(item); -+ -+ struct loc_country* country; -+ r = loc_country_new(loc_ctx, &country, country_code); -+ if (r) { -+ if (r == -EINVAL) { -+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code); -+ } else { -+ PyErr_SetString(PyExc_SystemError, "Could not create country"); -+ } -+ -+ loc_country_list_unref(countries); -+ return NULL; -+ } -+ -+ // Append it to the list -+ r = loc_country_list_append(countries, country); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append country to the list"); -+ -+ loc_country_list_unref(countries); -+ loc_country_unref(country); -+ return NULL; -+ } -+ -+ loc_country_unref(country); -+ } - -+ r = loc_database_enumerator_set_countries(enumerator, countries); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_country_list_unref(countries); - return NULL; - } -+ -+ loc_country_list_unref(countries); - } - - // Set the ASN we are searching for -- if (asn) { -- r = loc_database_enumerator_set_asn(enumerator, asn); -+ if (asn_list) { -+ struct loc_as_list* asns; -+ r = loc_as_list_new(loc_ctx, &asns); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS list"); -+ return NULL; -+ } -+ -+ for (int i = 0; i < PyList_Size(asn_list); i++) { -+ PyObject* item = PyList_GetItem(asn_list, i); -+ -+ if (!PyLong_Check(item)) { -+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers"); - -+ loc_as_list_unref(asns); -+ return NULL; -+ } -+ -+ unsigned long number = PyLong_AsLong(item); -+ -+ struct loc_as* as; -+ r = loc_as_new(loc_ctx, &as, number); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not create AS"); -+ -+ loc_as_list_unref(asns); -+ loc_as_unref(as); -+ return NULL; -+ } -+ -+ r = loc_as_list_append(asns, as); -+ if (r) { -+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); -+ -+ loc_as_list_unref(asns); -+ loc_as_unref(as); -+ return NULL; -+ } -+ -+ loc_as_unref(as); -+ } -+ -+ r = loc_database_enumerator_set_asns(enumerator, asns); - if (r) { - PyErr_SetFromErrno(PyExc_SystemError); -+ -+ loc_as_list_unref(asns); - return NULL; - } -+ -+ loc_as_list_unref(asns); - } - - // Set the flags we are searching for -@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - } - - static PyObject* Database_countries(DatabaseObject* self) { -- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0); - } - - static struct PyMethodDef Database_methods[] = { -@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "networks_flattened", -+ (getter)Database_networks_flattened, -+ NULL, -+ NULL, -+ NULL, -+ }, - { - "vendor", - (getter)Database_get_vendor, -diff --git a/src/python/downloader.py b/src/python/downloader.py -index 87bbb68..05f7872 100644 ---- a/src/python/downloader.py -+++ b/src/python/downloader.py -@@ -119,8 +119,8 @@ class Downloader(object): - - headers = {} - if timestamp: -- headers["If-Modified-Since"] = timestamp.strftime( -- "%a, %d %b %Y %H:%M:%S GMT", -+ headers["If-Modified-Since"] = time.strftime( -+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp), - ) - - t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) -@@ -195,7 +195,7 @@ class Downloader(object): - db = Database(f.name) - - # Database is not recent -- if timestamp and db.created_at < timestamp.timestamp(): -+ if timestamp and db.created_at < timestamp: - return False - - log.info("Downloaded new database from %s" % (time.strftime( -diff --git a/src/python/export.py b/src/python/export.py -index d15c6f0..f0eae26 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -29,7 +29,7 @@ import _location - log = logging.getLogger("location.export") - log.propagate = 1 - --flags = { -+FLAGS = { - _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", - _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", - _location.NETWORK_FLAG_ANYCAST : "A3", -@@ -39,11 +39,8 @@ class OutputWriter(object): - suffix = "networks" - mode = "w" - -- def __init__(self, f, prefix=None, flatten=True): -- self.f, self.prefix, self.flatten = f, prefix, flatten -- -- # The previously written network -- self._last_network = None -+ def __init__(self, f, prefix=None): -+ self.f, self.prefix = f, prefix - - # Immediately write the header - self._write_header() -@@ -60,18 +57,6 @@ class OutputWriter(object): - def __repr__(self): - return "<%s f=%s>" % (self.__class__.__name__, self.f) - -- def _flatten(self, network): -- """ -- Checks if the given network needs to be written to file, -- or if it is a subnet of the previously written network. -- """ -- if self._last_network and network.is_subnet_of(self._last_network): -- return True -- -- # Remember this network for the next call -- self._last_network = network -- return False -- - def _write_header(self): - """ - The header of the file -@@ -84,16 +69,8 @@ class OutputWriter(object): - """ - pass - -- def _write_network(self, network): -- self.f.write("%s\n" % network) -- - def write(self, network): -- if self.flatten and self._flatten(network): -- log.debug("Skipping writing network %s" % network) -- return -- -- # Write the network to file -- self._write_network(network) -+ self.f.write("%s\n" % network) - - def finish(self): - """ -@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter): - def _write_header(self): - self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix) - -- def _write_network(self, network): -+ def write(self, network): - self.f.write("add %s %s\n" % (self.prefix, network)) - - -@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter): - def _write_footer(self): - self.f.write("}\n") - -- def _write_network(self, network): -+ def write(self, network): - self.f.write(" %s,\n" % network) - - -@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter): - suffix = "iv" - mode = "wb" - -- def _write_network(self, network): -- for address in (network.first_address, network.last_address): -- # Convert this into a string of bits -- bytes = socket.inet_pton( -- network.family, address, -- ) -- -- self.f.write(bytes) -+ def write(self, network): -+ self.f.write(network._first_address) -+ self.f.write(network._last_address) - - - formats = { -@@ -185,8 +157,14 @@ class Exporter(object): - - writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) - -+ # Filter countries from special country codes -+ country_codes = [ -+ country_code for country_code in countries if not country_code in FLAGS.values() -+ ] -+ - # Get all networks that match the family -- networks = self.db.search_networks(family=family) -+ networks = self.db.search_networks(family=family, -+ country_codes=country_codes, asns=asns, flatten=True) - - # Walk through all networks - for network in networks: -@@ -203,10 +181,10 @@ class Exporter(object): - pass - - # Handle flags -- for flag in flags: -+ for flag in FLAGS: - if network.has_flag(flag): - # Fetch the "fake" country code -- country = flags[flag] -+ country = FLAGS[flag] - - try: - writers[country].write(network) -diff --git a/src/python/importer.py b/src/python/importer.py -index f19db4b..5f46bc3 100644 ---- a/src/python/importer.py -+++ b/src/python/importer.py -@@ -64,7 +64,7 @@ EXTENDED_SOURCES = ( - "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest", - - # Latin America and Caribbean Network Information Centre -- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", -+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest", - - # Réseaux IP Européens - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest", -diff --git a/src/python/location-importer.in b/src/python/location-importer.in -index 1467923..2dec89e 100644 ---- a/src/python/location-importer.in -+++ b/src/python/location-importer.in -@@ -152,6 +152,7 @@ class CLI(object): - last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); - CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); - CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); -+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); - - -- autnums - CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); -@@ -165,6 +166,7 @@ class CLI(object): - -- networks - CREATE TABLE IF NOT EXISTS networks(network inet, country text); - CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); -+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); - CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); - - -- overrides -@@ -188,6 +190,8 @@ class CLI(object): - ); - CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network - ON network_overrides(network); -+ CREATE INDEX IF NOT EXISTS network_overrides_search -+ ON network_overrides USING GIST(network inet_ops); - """) - - return db -@@ -234,32 +238,24 @@ class CLI(object): - - # Select all known networks - rows = self.db.query(""" -- -- Get a (sorted) list of all known networks -- WITH known_networks AS ( -- SELECT network FROM announcements -- UNION -- SELECT network FROM networks -- ORDER BY network -- ) -- - -- Return a list of those networks enriched with all - -- other information that we store in the database - SELECT -- DISTINCT ON (known_networks.network) -- known_networks.network AS network, -- announcements.autnum AS autnum, -+ DISTINCT ON (network) -+ network, -+ autnum, - - -- Country - COALESCE( - ( - SELECT country FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT country FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - networks.country - ) AS country, -@@ -268,50 +264,67 @@ class CLI(object): - COALESCE( - ( - SELECT is_anonymous_proxy FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anonymous_proxy FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE - ) AS is_anonymous_proxy, - COALESCE( - ( - SELECT is_satellite_provider FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_satellite_provider FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE - ) AS is_satellite_provider, - COALESCE( - ( - SELECT is_anycast FROM network_overrides overrides -- WHERE announcements.network <<= overrides.network -+ WHERE networks.network <<= overrides.network - ORDER BY masklen(overrides.network) DESC - LIMIT 1 - ), - ( - SELECT is_anycast FROM autnum_overrides overrides -- WHERE announcements.autnum = overrides.number -+ WHERE networks.autnum = overrides.number - ), - FALSE -- ) AS is_anycast, -- -- -- Must be part of returned values for ORDER BY clause -- masklen(announcements.network) AS sort_a, -- masklen(networks.network) AS sort_b -- FROM known_networks -- LEFT JOIN announcements ON known_networks.network <<= announcements.network -- LEFT JOIN networks ON known_networks.network <<= networks.network -- ORDER BY known_networks.network, sort_a DESC, sort_b DESC -+ ) AS is_anycast -+ FROM ( -+ SELECT -+ known_networks.network AS network, -+ announcements.autnum AS autnum, -+ networks.country AS country, -+ -+ -- Must be part of returned values for ORDER BY clause -+ masklen(announcements.network) AS sort_a, -+ masklen(networks.network) AS sort_b -+ FROM ( -+ SELECT network FROM announcements -+ UNION ALL -+ SELECT network FROM networks -+ UNION ALL -+ SELECT network FROM network_overrides -+ ) known_networks -+ LEFT JOIN -+ announcements ON known_networks.network <<= announcements.network -+ LEFT JOIN -+ networks ON known_networks.network <<= networks.network -+ ORDER BY -+ known_networks.network, -+ sort_a DESC, -+ sort_b DESC -+ ) networks - """) - - for row in rows: -@@ -363,6 +376,16 @@ class CLI(object): - CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); -+ -+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) -+ ON COMMIT DROP; -+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); -+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); -+ """) -+ -+ # Remove all previously imported content -+ self.db.execute(""" -+ TRUNCATE TABLE networks; - """) - - for source in location.importer.WHOIS_SOURCES: -@@ -370,31 +393,72 @@ class CLI(object): - for block in f: - self._parse_block(block) - -+ # Process all parsed networks from every RIR we happen to have access to, -+ # insert the largest network chunks into the networks table immediately... -+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") -+ -+ for family in (row.family for row in families): -+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) -+ -+ self.db.execute("INSERT INTO networks(network, country) \ -+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) -+ -+ # ... determine any other prefixes for this network family, ... -+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ -+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) -+ -+ # ... and insert networks with this prefix in case they provide additional -+ # information (i. e. subnet of a larger chunk with a different country) -+ for prefix in (row.prefix for row in prefixes): -+ self.db.execute(""" -+ WITH candidates AS ( -+ SELECT -+ _rirdata.network, -+ _rirdata.country -+ FROM -+ _rirdata -+ WHERE -+ family(_rirdata.network) = %s -+ AND -+ masklen(_rirdata.network) = %s -+ ), -+ filtered AS ( -+ SELECT -+ DISTINCT ON (c.network) -+ c.network, -+ c.country, -+ masklen(networks.network), -+ networks.country AS parent_country -+ FROM -+ candidates c -+ LEFT JOIN -+ networks -+ ON -+ c.network << networks.network -+ ORDER BY -+ c.network, -+ masklen(networks.network) DESC NULLS LAST -+ ) -+ INSERT INTO -+ networks(network, country) -+ SELECT -+ network, -+ country -+ FROM -+ filtered -+ WHERE -+ parent_country IS NULL -+ OR -+ country <> parent_country -+ ON CONFLICT DO NOTHING""", -+ family, prefix, -+ ) -+ - self.db.execute(""" - INSERT INTO autnums(number, name) - SELECT _autnums.number, _organizations.name FROM _autnums - JOIN _organizations ON _autnums.organization = _organizations.handle -- ON CONFLICT (number) DO UPDATE SET name = excluded.name -- """) -- -- self.db.execute(""" -- --- Purge any redundant entries -- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP -- AS -- SELECT network FROM networks candidates -- WHERE EXISTS ( -- SELECT FROM networks -- WHERE -- networks.network << candidates.network -- AND -- networks.country = candidates.country -- ); -- -- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network); -- -- DELETE FROM networks WHERE EXISTS ( -- SELECT FROM _garbage WHERE networks.network = _garbage.network -- ); -+ ON CONFLICT (number) DO UPDATE SET name = excluded.name; - """) - - # Download all extended sources -@@ -405,6 +469,69 @@ class CLI(object): - for line in f: - self._parse_line(line) - -+ def _check_parsed_network(self, network): -+ """ -+ Assistive function to detect and subsequently sort out parsed -+ networks from RIR data (both Whois and so-called "extended sources"), -+ which are or have... -+ -+ (a) not globally routable (RFC 1918 space, et al.) -+ (b) covering a too large chunk of the IP address space (prefix length -+ is < 7 for IPv4 networks, and < 10 for IPv6) -+ (c) "0.0.0.0" or "::" as a network address -+ (d) are too small for being publicly announced (we have decided not to -+ process them at the moment, as they significantly enlarge our -+ database without providing very helpful additional information) -+ -+ This unfortunately is necessary due to brain-dead clutter across -+ various RIR databases, causing mismatches and eventually disruptions. -+ -+ We will return False in case a network is not suitable for adding -+ it to our database, and True otherwise. -+ """ -+ -+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): -+ return False -+ -+ if not network.is_global: -+ log.warning("Skipping non-globally routable network: %s" % network) -+ return False -+ -+ if network.version == 4: -+ if network.prefixlen < 7: -+ log.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 24: -+ log.debug("Skipping network too small to be publicly announced: %s" % network) -+ return False -+ -+ if str(network.network_address) == "0.0.0.0": -+ log.warning("Skipping network based on 0.0.0.0: %s" % network) -+ return False -+ -+ elif network.version == 6: -+ if network.prefixlen < 10: -+ log.warning("Skipping too big IP chunk: %s" % network) -+ return False -+ -+ if network.prefixlen > 48: -+ log.debug("Skipping network too small to be publicly announced: %s" % network) -+ return False -+ -+ if str(network.network_address) == "::": -+ log.warning("Skipping network based on '::': %s" % network) -+ return False -+ -+ else: -+ # This should not happen... -+ log.warning("Skipping network of unknown family, this should not happen: %s" % network) -+ return False -+ -+ # In case we have made it here, the network is considered to -+ # be suitable for libloc consumption... -+ return True -+ - def _parse_block(self, block): - # Get first line to find out what type of block this is - line = block[0] -@@ -433,7 +560,7 @@ class CLI(object): - autnum["asn"] = m.group(2) - - elif key == "org": -- autnum[key] = val -+ autnum[key] = val.upper() - - # Skip empty objects - if not autnum: -@@ -447,15 +574,22 @@ class CLI(object): - ) - - def _parse_inetnum_block(self, block): -- logging.debug("Parsing inetnum block:") -+ log.debug("Parsing inetnum block:") - - inetnum = {} - for line in block: -- logging.debug(line) -+ log.debug(line) - - # Split line - key, val = split_line(line) - -+ # Filter any inetnum records which are only referring to IP space -+ # not managed by that specific RIR... -+ if key == "netname": -+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): -+ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) -+ return -+ - if key == "inetnum": - start_address, delim, end_address = val.partition("-") - -@@ -467,7 +601,7 @@ class CLI(object): - start_address = ipaddress.ip_address(start_address) - end_address = ipaddress.ip_address(end_address) - except ValueError: -- logging.warning("Could not parse line: %s" % line) -+ log.warning("Could not parse line: %s" % line) - return - - # Set prefix to default -@@ -484,23 +618,24 @@ class CLI(object): - inetnum[key] = val - - elif key == "country": -- if val == "UNITED STATES": -- val = "US" -- - inetnum[key] = val.upper() - - # Skip empty objects -- if not inetnum: -+ if not inetnum or not "country" in inetnum: -+ return -+ -+ # Skip objects with bogus country code 'ZZ' -+ if inetnum.get("country") == "ZZ": -+ log.warning("Skipping network with bogus country 'ZZ': %s" % \ -+ (inetnum.get("inet6num") or inetnum.get("inetnum"))) - return - - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) - -- # Bail out in case we have processed a non-public IP network -- if network.is_private: -- logging.warning("Skipping non-globally routable network: %s" % network) -+ if not self._check_parsed_network(network): - return - -- self.db.execute("INSERT INTO networks(network, country) \ -+ self.db.execute("INSERT INTO _rirdata(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % network, inetnum.get("country"), - ) -@@ -511,7 +646,9 @@ class CLI(object): - # Split line - key, val = split_line(line) - -- if key in ("organisation", "org-name"): -+ if key == "organisation": -+ org[key] = val.upper() -+ elif key == "org-name": - org[key] = val - - # Skip empty objects -@@ -581,6 +718,9 @@ class CLI(object): - log.warning("Invalid IP address: %s" % address) - return - -+ if not self._check_parsed_network(network): -+ return -+ - self.db.execute("INSERT INTO networks(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO \ - UPDATE SET country = excluded.country", -diff --git a/src/python/location.in b/src/python/location.in -index 44ad726..b30beae 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -253,6 +253,7 @@ class CLI(object): - network = db.lookup(address) - except ValueError: - print(_("Invalid IP address: %s") % address, file=sys.stderr) -+ return 2 - - args = { - "address" : address, -@@ -398,10 +399,7 @@ class CLI(object): - - def handle_update(self, db, ns): - if ns.cron and db: -- now = datetime.datetime.utcnow() -- -- # Parse the database timestamp -- t = datetime.datetime.utcfromtimestamp(db.created_at) -+ now = time.time() - - if ns.cron == "daily": - delta = datetime.timedelta(days=1) -@@ -410,22 +408,20 @@ class CLI(object): - elif ns.cron == "monthly": - delta = datetime.timedelta(days=30) - -+ delta = delta.total_seconds() -+ - # Check if the database has recently been updated -- if t >= (now - delta): -+ if db.created_at >= (now - delta): - log.info( -- _("The database has been updated recently (%s)") % \ -- format_timedelta(now - t), -+ _("The database has been updated recently"), - ) - return 3 - - # Fetch the timestamp we need from DNS - t = location.discover_latest_version() - -- # Parse timestamp into datetime format -- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None -- - # Check the version of the local database -- if db and timestamp and db.created_at >= timestamp.timestamp(): -+ if db and t and db.created_at >= t: - log.info("Already on the latest version") - return - -@@ -437,7 +433,7 @@ class CLI(object): - - # Try downloading a new database - try: -- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) -+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir) - - # If no file could be downloaded, log a message - except FileNotFoundError as e: -@@ -453,13 +449,7 @@ class CLI(object): - - return 0 - -- def handle_verify(self, ns): -- try: -- db = location.Database(ns.database) -- except FileNotFoundError as e: -- log.error("%s: %s" % (ns.database, e)) -- return 127 -- -+ def handle_verify(self, db, ns): - # Verify the database - with open(ns.public_key, "r") as f: - if not db.verify(f): -diff --git a/src/python/network.c b/src/python/network.c -index 5496d1e..5b1369d 100644 ---- a/src/python/network.c -+++ b/src/python/network.c -@@ -17,13 +17,33 @@ - #include <Python.h> - - #include <errno.h> -+#include <limits.h> - - #include <loc/libloc.h> - #include <loc/network.h> -+#include <loc/network-list.h> - - #include "locationmodule.h" - #include "network.h" - -+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) { -+ PyObject* list = PyList_New(0); -+ if (!networks) -+ return list; -+ -+ while (!loc_network_list_empty(networks)) { -+ struct loc_network* network = loc_network_list_pop(networks); -+ -+ PyObject* n = new_network(&NetworkType, network); -+ PyList_Append(list, n); -+ -+ loc_network_unref(network); -+ Py_DECREF(n); -+ } -+ -+ return list; -+} -+ - PyObject* new_network(PyTypeObject* type, struct loc_network* network) { - NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0); - if (self) { -@@ -114,10 +134,18 @@ static int Network_set_asn(NetworkObject* self, PyObject* value) { - long int asn = PyLong_AsLong(value); - - // Check if the ASN is within the valid range -- if (asn <= 0 || asn > UINT32_MAX) { -+ if (asn <= 0) { -+ PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn); -+ return -1; -+ } -+ -+#if (__WORDSIZE > 32) -+ // Check whether the input was longer than 32 bit -+ if (asn > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn); - return -1; - } -+#endif - - int r = loc_network_set_asn(self->network, asn); - if (r) -@@ -154,13 +182,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) { - Py_RETURN_NONE; - } - -+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) { -+ NetworkObject* other = NULL; -+ -+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) -+ return NULL; -+ -+ struct loc_network_list* list = loc_network_exclude(self->network, other->network); -+ -+ // Convert to Python objects -+ PyObject* obj = PyList_FromNetworkList(list); -+ loc_network_list_unref(list); -+ -+ return obj; -+} -+ - static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) { - NetworkObject* other = NULL; - - if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other)) - return NULL; - -- if (loc_network_is_subnet_of(self->network, other->network)) -+ if (loc_network_is_subnet(other->network, self->network)) - Py_RETURN_TRUE; - - Py_RETURN_FALSE; -@@ -181,6 +224,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) { - return obj; - } - -+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) { -+ struct in_addr address4; -+ -+ // Convert IPv4 addresses to struct in_addr -+ if (IN6_IS_ADDR_V4MAPPED(address6)) { -+ address4.s_addr = address6->s6_addr32[3]; -+ -+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4)); -+ } -+ -+ // Return IPv6 addresses as they are -+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6)); -+} -+ -+static PyObject* Network_get__first_address(NetworkObject* self) { -+ const struct in6_addr* address = loc_network_get_first_address(self->network); -+ -+ return PyBytes_FromAddress(address); -+} -+ - static PyObject* Network_get_last_address(NetworkObject* self) { - char* address = loc_network_format_last_address(self->network); - -@@ -190,7 +253,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) { - return obj; - } - -+static PyObject* Network_get__last_address(NetworkObject* self) { -+ const struct in6_addr* address = loc_network_get_last_address(self->network); -+ -+ return PyBytes_FromAddress(address); -+} -+ - static struct PyMethodDef Network_methods[] = { -+ { -+ "exclude", -+ (PyCFunction)Network_exclude, -+ METH_VARARGS, -+ NULL, -+ }, - { - "has_flag", - (PyCFunction)Network_has_flag, -@@ -241,6 +316,13 @@ static struct PyGetSetDef Network_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "_first_address", -+ (getter)Network_get__first_address, -+ NULL, -+ NULL, -+ NULL, -+ }, - { - "last_address", - (getter)Network_get_last_address, -@@ -248,6 +330,13 @@ static struct PyGetSetDef Network_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "_last_address", -+ (getter)Network_get__last_address, -+ NULL, -+ NULL, -+ NULL, -+ }, - { NULL }, - }; - -diff --git a/src/test-as.c b/src/test-as.c -index 839a04c..2d61675 100644 ---- a/src/test-as.c -+++ b/src/test-as.c -@@ -95,7 +95,7 @@ int main(int argc, char** argv) { - // Enumerator - - struct loc_database_enumerator* enumerator; -- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES); -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0); - if (err) { - fprintf(stderr, "Could not create a database enumerator\n"); - exit(EXIT_FAILURE); -diff --git a/src/test-database.c b/src/test-database.c -index b4a75c4..da4b11c 100644 ---- a/src/test-database.c -+++ b/src/test-database.c -@@ -38,6 +38,14 @@ const char* DESCRIPTION = - "Maecenas ut venenatis nunc."; - const char* LICENSE = "CC"; - -+const char* networks[] = { -+ "2001:db8::/32", -+ "2001:db8:1000::/48", -+ "2001:db8:2000::/48", -+ "2001:db8:2020::/48", -+ NULL, -+}; -+ - static int attempt_to_open(struct loc_ctx* ctx, char* path) { - FILE* f = fopen(path, "r"); - if (!f) -@@ -139,6 +147,24 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ struct loc_network* network = NULL; -+ -+ // Add some networks -+ const char** n = networks; -+ while (*n) { -+ err = loc_writer_add_network(writer, &network, *n); -+ if (err) { -+ fprintf(stderr, "Could not add network %s\n", *n); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Set a country -+ loc_network_set_country_code(network, "XX"); -+ -+ // Next one -+ n++; -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); -@@ -170,6 +196,33 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ // Enumerator -+ struct loc_database_enumerator* enumerator; -+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0); -+ if (err) { -+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Walk through all networks -+ while (1) { -+ err = loc_database_enumerator_next_network(enumerator, &network); -+ if (err) { -+ fprintf(stderr, "Error fetching the next network: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!network) -+ break; -+ -+ char* s = loc_network_str(network); -+ printf("Got network: %s\n", s); -+ free(s); -+ } -+ -+ // Free the enumerator -+ loc_database_enumerator_unref(enumerator); -+ - // Close the database - loc_database_unref(db); - loc_unref(ctx); -diff --git a/src/test-network-list.c b/src/test-network-list.c -new file mode 100644 -index 0000000..6f32ff7 ---- /dev/null -+++ b/src/test-network-list.c -@@ -0,0 +1,183 @@ -+/* -+ libloc - A library to determine the location of someone on the Internet -+ -+ Copyright (C) 2017 IPFire Development Team info@ipfire.org -+ -+ This program is free software; you can redistribute it and/or modify -+ it under the terms of the GNU General Public License as published by -+ the Free Software Foundation; either version 2 of the License, or -+ (at your option) any later version. -+ -+ This program is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ GNU General Public License for more details. -+*/ -+ -+#include <errno.h> -+#include <stdio.h> -+#include <stddef.h> -+#include <stdlib.h> -+#include <string.h> -+#include <syslog.h> -+ -+#include <loc/libloc.h> -+#include <loc/network.h> -+#include <loc/network-list.h> -+ -+int main(int argc, char** argv) { -+ int err; -+ -+ struct loc_ctx* ctx; -+ err = loc_new(&ctx); -+ if (err < 0) -+ exit(EXIT_FAILURE); -+ -+ // Enable debug logging -+ loc_set_log_priority(ctx, LOG_DEBUG); -+ -+ // Create a network -+ struct loc_network* network1; -+ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32"); -+ if (err) { -+ fprintf(stderr, "Could not create the network1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet1; -+ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet2; -+ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet2\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet3; -+ err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet3\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet4; -+ err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet4\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet5; -+ err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet5\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ struct loc_network* subnet6; -+ err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48"); -+ if (err) { -+ fprintf(stderr, "Could not create the subnet6\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Make a list with both subnets -+ struct loc_network_list* subnets; -+ err = loc_network_list_new(ctx, &subnets); -+ if (err) { -+ fprintf(stderr, "Could not create subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ size_t size = loc_network_list_size(subnets); -+ if (size > 0) { -+ fprintf(stderr, "The list is not empty: %zu\n", size); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_list_push(subnets, subnet1); -+ if (err) { -+ fprintf(stderr, "Could not add subnet1 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (loc_network_list_empty(subnets)) { -+ fprintf(stderr, "The subnets list reports that it is empty\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_list_push(subnets, subnet2); -+ if (err) { -+ fprintf(stderr, "Could not add subnet2 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Add the fourth one next -+ err = loc_network_list_push(subnets, subnet4); -+ if (err) { -+ fprintf(stderr, "Could not add subnet4 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Add the third one -+ err = loc_network_list_push(subnets, subnet3); -+ if (err) { -+ fprintf(stderr, "Could not add subnet3 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Add more subnets -+ err = loc_network_list_push(subnets, subnet5); -+ if (err) { -+ fprintf(stderr, "Could not add subnet5 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_list_push(subnets, subnet6); -+ if (err) { -+ fprintf(stderr, "Could not add subnet6 to subnets list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(subnets); -+ -+ size = loc_network_list_size(subnets); -+ if (size != 6) { -+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Exclude subnet1 from network1 -+ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ // Exclude all subnets from network1 -+ excluded = loc_network_exclude_list(network1, subnets); -+ if (!excluded) { -+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ -+ if (excluded) -+ loc_network_list_unref(excluded); -+ -+ loc_network_list_unref(subnets); -+ loc_network_unref(network1); -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ loc_unref(ctx); -+ -+ return EXIT_SUCCESS; -+} -diff --git a/src/test-network.c b/src/test-network.c -index d38f13d..dde13f1 100644 ---- a/src/test-network.c -+++ b/src/test-network.c -@@ -14,6 +14,7 @@ - GNU General Public License for more details. - */ - -+#include <arpa/inet.h> - #include <errno.h> - #include <stdio.h> - #include <stddef.h> -@@ -37,12 +38,21 @@ int main(int argc, char** argv) { - // Enable debug logging - loc_set_log_priority(ctx, LOG_DEBUG); - -+#if 0 - struct loc_network_tree* tree; - err = loc_network_tree_new(ctx, &tree); - if (err) { - fprintf(stderr, "Could not create the network tree\n"); - exit(EXIT_FAILURE); - } -+#endif -+ -+ struct in6_addr address; -+ err = inet_pton(AF_INET6, "2001:db8::1", &address); -+ if (err != 1) { -+ fprintf(stderr, "Could not parse IP address\n"); -+ exit(EXIT_FAILURE); -+ } - - // Create a network - struct loc_network* network1; -@@ -58,12 +68,14 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+#if 0 - // Adding network to the tree - err = loc_network_tree_add_network(tree, network1); - if (err) { - fprintf(stderr, "Could not add network to the tree\n"); - exit(EXIT_FAILURE); - } -+#endif - - // Check if the first and last addresses are correct - char* string = loc_network_format_first_address(network1); -@@ -88,6 +100,12 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+ err = loc_network_match_address(network1, &address); -+ if (!err) { -+ fprintf(stderr, "Network1 does not match address\n"); -+ exit(EXIT_FAILURE); -+ } -+ - struct loc_network* network2; - err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48"); - if (err) { -@@ -101,6 +119,7 @@ int main(int argc, char** argv) { - exit(EXIT_FAILURE); - } - -+#if 0 - // Adding network to the tree - err = loc_network_tree_add_network(tree, network2); - if (err) { -@@ -117,20 +136,84 @@ int main(int argc, char** argv) { - - size_t nodes = loc_network_tree_count_nodes(tree); - printf("The tree has %zu nodes\n", nodes); -+#endif -+ -+ // Check equals function -+ err = loc_network_cmp(network1, network1); -+ if (err) { -+ fprintf(stderr, "Network is not equal with itself\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ err = loc_network_cmp(network1, network2); -+ if (!err) { -+ fprintf(stderr, "Networks equal unexpectedly\n"); -+ exit(EXIT_FAILURE); -+ } - - // Check subnet function -- err = loc_network_is_subnet_of(network1, network2); -- if (err != 0) { -+ err = loc_network_is_subnet(network1, network2); -+ if (!err) { - fprintf(stderr, "Subnet check 1 failed: %d\n", err); - exit(EXIT_FAILURE); - } - -- err = loc_network_is_subnet_of(network2, network1); -- if (err != 1) { -+ err = loc_network_is_subnet(network2, network1); -+ if (err) { - fprintf(stderr, "Subnet check 2 failed: %d\n", err); - exit(EXIT_FAILURE); - } - -+ // Make subnets -+ struct loc_network* subnet1 = NULL; -+ struct loc_network* subnet2 = NULL; -+ -+ err = loc_network_subnets(network1, &subnet1, &subnet2); -+ if (err || !subnet1 || !subnet2) { -+ fprintf(stderr, "Could not find subnets of network: %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ char* s = loc_network_str(subnet1); -+ printf("Received subnet1 = %s\n", s); -+ free(s); -+ -+ s = loc_network_str(subnet2); -+ printf("Received subnet2 = %s\n", s); -+ free(s); -+ -+ if (!loc_network_is_subnet(network1, subnet1)) { -+ fprintf(stderr, "Subnet1 is not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!loc_network_is_subnet(network1, subnet2)) { -+ fprintf(stderr, "Subnet2 is not a subnet\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!loc_network_overlaps(network1, subnet1)) { -+ fprintf(stderr, "Network1 does not seem to contain subnet1\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (!loc_network_overlaps(network1, subnet2)) { -+ fprintf(stderr, "Network1 does not seem to contain subnet2\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_unref(subnet1); -+ loc_network_unref(subnet2); -+ -+ struct loc_network_list* excluded = loc_network_exclude(network1, network2); -+ if (!excluded) { -+ fprintf(stderr, "Could not create excluded list\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ loc_network_list_dump(excluded); -+ loc_network_list_unref(excluded); -+ - // Create a database - struct loc_writer* writer; - err = loc_writer_new(ctx, &writer, NULL, NULL); -@@ -160,6 +243,28 @@ int main(int argc, char** argv) { - // Set ASN - loc_network_set_asn(network4, 1024); - -+ // Try adding an invalid network -+ struct loc_network* network; -+ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Try adding a single address -+ err = loc_writer_add_network(writer, &network, "2001:db8::"); -+ if (err) { -+ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err); -+ exit(EXIT_FAILURE); -+ } -+ -+ // Try adding localhost -+ err = loc_writer_add_network(writer, &network, "::1/128"); -+ if (err != -EINVAL) { -+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err); -+ exit(EXIT_FAILURE); -+ } -+ - FILE* f = tmpfile(); - if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); -@@ -177,7 +282,10 @@ int main(int argc, char** argv) { - loc_network_unref(network2); - loc_network_unref(network3); - loc_network_unref(network4); -+ -+#if 0 - loc_network_tree_unref(tree); -+#endif - - // And open it again from disk - struct loc_database* db; -diff --git a/src/writer.c b/src/writer.c -index 5939cff..c61a6df 100644 ---- a/src/writer.c -+++ b/src/writer.c -@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) { - EVP_PKEY_free(writer->private_key2); - - // Unref all AS -- for (unsigned int i = 0; i < writer->as_count; i++) { -- loc_as_unref(writer->as[i]); -+ if (writer->as) { -+ for (unsigned int i = 0; i < writer->as_count; i++) { -+ loc_as_unref(writer->as[i]); -+ } -+ free(writer->as); -+ } -+ -+ // Unref all countries -+ if (writer->countries) { -+ for (unsigned int i = 0; i < writer->countries_count; i++) { -+ loc_country_unref(writer->countries[i]); -+ } -+ free(writer->countries); - } - - // Release network tree -@@ -601,7 +612,7 @@ static int loc_writer_create_signature(struct loc_writer* writer, - goto END; - } - -- DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n", *length); -+ DEBUG(writer->ctx, "Successfully generated signature of %zu bytes\n", *length); - r = 0; - - // Dump signature diff --git a/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch b/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch new file mode 100644 index 000000000..ea29885a8 --- /dev/null +++ b/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch @@ -0,0 +1,27 @@ +From b178117bac33b4b1e7ce341a6f2eec493cca13f8 Mon Sep 17 00:00:00 2001 +From: Michael Tremer michael.tremer@ipfire.org +Date: Mon, 21 Dec 2020 16:25:46 +0000 +Subject: [PATCH] location: Fix list-networks-by-as + +Fixes: #12554 +Signed-off-by: Michael Tremer michael.tremer@ipfire.org +--- + src/python/location.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/python/location.in b/src/python/location.in +index b30beae..ad2ccf5 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -493,7 +493,7 @@ class CLI(object): + f = writer(sys.stdout, prefix="AS%s" % asn) + + # Print all matching networks +- for n in db.search_networks(asn=asn, family=ns.family): ++ for n in db.search_networks(asns=[asn], family=ns.family): + f.write(n) + + f.finish() +-- +2.26.2 +