* [PATCH] libloc: update to 0.9.5 and backport fix for #12554
@ 2020-12-23 14:03 Peter Müller
0 siblings, 0 replies; only message in thread
From: Peter Müller @ 2020-12-23 14:03 UTC (permalink / raw)
To: development
[-- Attachment #1: Type: text/plain, Size: 123799 bytes --]
This patch updates libloc to 0.9.5, deletes the upstream patchset from
version 0.9.4, and includes a latest upstream patch to backport a fix
for #12554.
Cc: Michael Tremer <michael.tremer(a)ipfire.org>
Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org>
---
lfs/libloc | 14 +-
src/patches/libloc-0.9.4-upstream.patch | 3850 -----------------
...9.5-location-Fix-list-networks-by-as.patch | 27 +
3 files changed, 34 insertions(+), 3857 deletions(-)
delete mode 100644 src/patches/libloc-0.9.4-upstream.patch
create mode 100644 src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch
diff --git a/lfs/libloc b/lfs/libloc
index 010ddf1cb..279840535 100644
--- a/lfs/libloc
+++ b/lfs/libloc
@@ -24,8 +24,8 @@
include Config
-VER = 0.9.4
-DB_DATE = 2020-09-21
+VER = 0.9.5
+DB_DATE = 2020-12-23
THISAPP = libloc-$(VER)
DL_FILE = $(THISAPP).tar.gz
@@ -40,11 +40,11 @@ TARGET = $(DIR_INFO)/$(THISAPP)
objects = $(DL_FILE) \
location-$(DB_DATE).db.xz
-$(DL_FILE) = https://source.ipfire.org/releases/libloc//$(DL_FILE)
+$(DL_FILE) = https://source.ipfire.org/releases/libloc/$(DL_FILE)
location-$(DB_DATE).db.xz = https://location.ipfire.org/databases/1/archive/location-$(DB_DATE).db.xz
-$(DL_FILE)_MD5 = 82770e9eba20f636c96e6fa42ff234b5
-location-$(DB_DATE).db.xz_MD5 = fa3069bf31170629d638317e283913c0
+$(DL_FILE)_MD5 = 41d8dc3fb4e498db958b7696cadd61f5
+location-$(DB_DATE).db.xz_MD5 = 3a71931555623fc3665f280c8e5c292a
install : $(TARGET)
@@ -78,8 +78,8 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
@$(PREBUILD)
@rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE)
- # Import changes from upstream
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.4-upstream.patch
+ # Import recent patches from upstream
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch
# Add patch for i586 to disable strong stack protector.
ifeq "$(BUILD_ARCH)" "i586"
diff --git a/src/patches/libloc-0.9.4-upstream.patch b/src/patches/libloc-0.9.4-upstream.patch
deleted file mode 100644
index a6ec1066b..000000000
--- a/src/patches/libloc-0.9.4-upstream.patch
+++ /dev/null
@@ -1,3850 +0,0 @@
-diff --git a/Makefile.am b/Makefile.am
-index a0431a6..dc594f8 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -91,11 +91,14 @@ EXTRA_DIST += \
- pkginclude_HEADERS = \
- src/loc/libloc.h \
- src/loc/as.h \
-+ src/loc/as-list.h \
- src/loc/compat.h \
- src/loc/country.h \
-+ src/loc/country-list.h \
- src/loc/database.h \
- src/loc/format.h \
- src/loc/network.h \
-+ src/loc/network-list.h \
- src/loc/private.h \
- src/loc/stringpool.h \
- src/loc/resolv.h \
-@@ -107,9 +110,12 @@ lib_LTLIBRARIES = \
- src_libloc_la_SOURCES = \
- src/libloc.c \
- src/as.c \
-+ src/as-list.c \
- src/country.c \
-+ src/country-list.c \
- src/database.c \
- src/network.c \
-+ src/network-list.c \
- src/resolv.c \
- src/stringpool.c \
- src/writer.c
-@@ -312,6 +318,7 @@ check_PROGRAMS = \
- src/test-database \
- src/test-as \
- src/test-network \
-+ src/test-network-list \
- src/test-country \
- src/test-signature
-
-@@ -351,6 +358,15 @@ src_test_network_CFLAGS = \
- src_test_network_LDADD = \
- src/libloc.la
-
-+src_test_network_list_SOURCES = \
-+ src/test-network-list.c
-+
-+src_test_network_list_CFLAGS = \
-+ $(TESTS_CFLAGS)
-+
-+src_test_network_list_LDADD = \
-+ src/libloc.la
-+
- src_test_stringpool_SOURCES = \
- src/test-stringpool.c
-
-@@ -390,7 +406,7 @@ MANPAGES_XML = $(patsubst %.txt,%.xml,$(MANPAGES_TXT))
- .PHONY: man
- man: $(MANPAGES) $(MANPAGES_HTML)
-
--if ENABLE_MANPAGES
-+if ENABLE_MAN_PAGES
- man_MANS = \
- $(MANPAGES)
- endif
-diff --git a/configure.ac b/configure.ac
-index 2364dfd..9eb9012 100644
---- a/configure.ac
-+++ b/configure.ac
-@@ -1,6 +1,6 @@
- AC_PREREQ(2.60)
- AC_INIT([libloc],
-- [0.9.4],
-+ [0.9.5],
- [location(a)lists.ipfire.org],
- [libloc],
- [https://location.ipfire.org/])
-@@ -43,16 +43,16 @@ AC_PROG_MKDIR_P
-
- # - man ------------------------------------------------------------------------
-
--have_manpages=no
--AC_ARG_ENABLE(manpages, AS_HELP_STRING([--disable-man-pages],
-+have_man_pages=no
-+AC_ARG_ENABLE(man_pages, AS_HELP_STRING([--disable-man-pages],
- [do not install man pages]))
--AS_IF([test "x$enable_manpages" != xno], [have_manpages=yes])
--AM_CONDITIONAL(ENABLE_MANPAGES, [test "x$have_manpages" = "xyes"])
-+AS_IF([test "x$enable_man_pages" != xno], [have_man_pages=yes])
-+AM_CONDITIONAL(ENABLE_MAN_PAGES, [test "x$have_man_pages" = "xyes"])
-
- AC_PATH_PROG([XSLTPROC], [xsltproc])
-
- AC_CHECK_PROGS(ASCIIDOC, [asciidoc])
--if test "${have_manpages}" = "yes" && test -z "${ASCIIDOC}"; then
-+if test "${have_man_pages}" = "yes" && test -z "${ASCIIDOC}"; then
- AC_MSG_ERROR([Required program 'asciidoc' not found])
- fi
- # - debug ----------------------------------------------------------------------
-diff --git a/src/.gitignore b/src/.gitignore
-index caf80b5..3ccbdb8 100644
---- a/src/.gitignore
-+++ b/src/.gitignore
-@@ -10,5 +10,6 @@ test-libloc
- test-database
- test-country
- test-network
-+test-network-list
- test-signature
- test-stringpool
-diff --git a/src/as-list.c b/src/as-list.c
-new file mode 100644
-index 0000000..5acbb8a
---- /dev/null
-+++ b/src/as-list.c
-@@ -0,0 +1,161 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2020 IPFire Development Team <info(a)ipfire.org>
-+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
-+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
-+
-+#include <errno.h>
-+#include <stdlib.h>
-+
-+#include <loc/as.h>
-+#include <loc/as-list.h>
-+#include <loc/private.h>
-+
-+struct loc_as_list {
-+ struct loc_ctx* ctx;
-+ int refcount;
-+
-+ struct loc_as** elements;
-+ size_t elements_size;
-+
-+ size_t size;
-+};
-+
-+static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
-+ DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
-+ list, size, list->elements_size + size);
-+
-+ struct loc_as** elements = reallocarray(list->elements,
-+ list->elements_size + size, sizeof(*list->elements));
-+ if (!elements)
-+ return -errno;
-+
-+ list->elements = elements;
-+ list->elements_size += size;
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
-+ struct loc_as_list** list) {
-+ struct loc_as_list* l = calloc(1, sizeof(*l));
-+ if (!l)
-+ return -ENOMEM;
-+
-+ l->ctx = loc_ref(ctx);
-+ l->refcount = 1;
-+
-+ DEBUG(l->ctx, "AS list allocated at %p\n", l);
-+ *list = l;
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
-+ list->refcount++;
-+
-+ return list;
-+}
-+
-+static void loc_as_list_free(struct loc_as_list* list) {
-+ DEBUG(list->ctx, "Releasing AS list at %p\n", list);
-+
-+ loc_as_list_clear(list);
-+
-+ loc_unref(list->ctx);
-+ free(list);
-+}
-+
-+LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
-+ if (!list)
-+ return NULL;
-+
-+ if (--list->refcount > 0)
-+ return list;
-+
-+ loc_as_list_free(list);
-+ return NULL;
-+}
-+
-+LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
-+ return list->size;
-+}
-+
-+LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
-+ return list->size == 0;
-+}
-+
-+LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
-+ if (!list->elements)
-+ return;
-+
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_as_unref(list->elements[i]);
-+
-+ free(list->elements);
-+ list->elements = NULL;
-+ list->elements_size = 0;
-+
-+ list->size = 0;
-+}
-+
-+LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
-+ // Check index
-+ if (index >= list->size)
-+ return NULL;
-+
-+ return loc_as_ref(list->elements[index]);
-+}
-+
-+LOC_EXPORT int loc_as_list_append(
-+ struct loc_as_list* list, struct loc_as* as) {
-+ if (loc_as_list_contains(list, as))
-+ return 0;
-+
-+ // Check if we have space left
-+ if (list->size >= list->elements_size) {
-+ int r = loc_as_list_grow(list, 64);
-+ if (r)
-+ return r;
-+ }
-+
-+ DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
-+
-+ list->elements[list->size++] = loc_as_ref(as);
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_as_list_contains(
-+ struct loc_as_list* list, struct loc_as* as) {
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ if (loc_as_cmp(as, list->elements[i]) == 0)
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_as_list_contains_number(
-+ struct loc_as_list* list, uint32_t number) {
-+ struct loc_as* as;
-+
-+ int r = loc_as_new(list->ctx, &as, number);
-+ if (r)
-+ return -1;
-+
-+ r = loc_as_list_contains(list, as);
-+ loc_as_unref(as);
-+
-+ return r;
-+}
-diff --git a/src/as.c b/src/as.c
-index e1fbb01..757bf3d 100644
---- a/src/as.c
-+++ b/src/as.c
-@@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
- }
-
- LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
-- as->name = strdup(name);
-+ if (as->name)
-+ free(as->name);
-+
-+ if (name)
-+ as->name = strdup(name);
-+ else
-+ as->name = NULL;
-
- return 0;
- }
-@@ -139,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
- if (!string)
- return 1;
-
-+ // Cannot match anything when name is not set
-+ if (!as->name)
-+ return 1;
-+
- // Search if string is in name
- if (strcasestr(as->name, string) != NULL)
- return 1;
-diff --git a/src/country-list.c b/src/country-list.c
-new file mode 100644
-index 0000000..cc36740
---- /dev/null
-+++ b/src/country-list.c
-@@ -0,0 +1,161 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2020 IPFire Development Team <info(a)ipfire.org>
-+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
-+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
-+
-+#include <errno.h>
-+#include <stdlib.h>
-+
-+#include <loc/country.h>
-+#include <loc/country-list.h>
-+#include <loc/private.h>
-+
-+struct loc_country_list {
-+ struct loc_ctx* ctx;
-+ int refcount;
-+
-+ struct loc_country** elements;
-+ size_t elements_size;
-+
-+ size_t size;
-+};
-+
-+static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
-+ DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
-+ list, size, list->elements_size + size);
-+
-+ struct loc_country** elements = reallocarray(list->elements,
-+ list->elements_size + size, sizeof(*list->elements));
-+ if (!elements)
-+ return -errno;
-+
-+ list->elements = elements;
-+ list->elements_size += size;
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
-+ struct loc_country_list** list) {
-+ struct loc_country_list* l = calloc(1, sizeof(*l));
-+ if (!l)
-+ return -ENOMEM;
-+
-+ l->ctx = loc_ref(ctx);
-+ l->refcount = 1;
-+
-+ DEBUG(l->ctx, "Country list allocated at %p\n", l);
-+ *list = l;
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
-+ list->refcount++;
-+
-+ return list;
-+}
-+
-+static void loc_country_list_free(struct loc_country_list* list) {
-+ DEBUG(list->ctx, "Releasing country list at %p\n", list);
-+
-+ loc_country_list_clear(list);
-+
-+ loc_unref(list->ctx);
-+ free(list);
-+}
-+
-+LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
-+ if (!list)
-+ return NULL;
-+
-+ if (--list->refcount > 0)
-+ return list;
-+
-+ loc_country_list_free(list);
-+ return NULL;
-+}
-+
-+LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
-+ return list->size;
-+}
-+
-+LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
-+ return list->size == 0;
-+}
-+
-+LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
-+ if (!list->elements)
-+ return;
-+
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_country_unref(list->elements[i]);
-+
-+ free(list->elements);
-+ list->elements = NULL;
-+ list->elements_size = 0;
-+
-+ list->size = 0;
-+}
-+
-+LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
-+ // Check index
-+ if (index >= list->size)
-+ return NULL;
-+
-+ return loc_country_ref(list->elements[index]);
-+}
-+
-+LOC_EXPORT int loc_country_list_append(
-+ struct loc_country_list* list, struct loc_country* country) {
-+ if (loc_country_list_contains(list, country))
-+ return 0;
-+
-+ // Check if we have space left
-+ if (list->size >= list->elements_size) {
-+ int r = loc_country_list_grow(list, 64);
-+ if (r)
-+ return r;
-+ }
-+
-+ DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
-+
-+ list->elements[list->size++] = loc_country_ref(country);
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_country_list_contains(
-+ struct loc_country_list* list, struct loc_country* country) {
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ if (loc_country_cmp(country, list->elements[i]) == 0)
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_country_list_contains_code(
-+ struct loc_country_list* list, const char* code) {
-+ struct loc_country* country;
-+
-+ int r = loc_country_new(list->ctx, &country, code);
-+ if (r)
-+ return -1;
-+
-+ r = loc_country_list_contains(list, country);
-+ loc_country_unref(country);
-+
-+ return r;
-+}
-diff --git a/src/country.c b/src/country.c
-index 2ba93e6..7aac0db 100644
---- a/src/country.c
-+++ b/src/country.c
-@@ -34,6 +34,9 @@ struct loc_country {
- };
-
- LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
-+ if (!loc_country_code_is_valid(country_code))
-+ return -EINVAL;
-+
- struct loc_country* c = calloc(1, sizeof(*c));
- if (!c)
- return -ENOMEM;
-diff --git a/src/database.c b/src/database.c
-index fa1dad0..4b8bf1d 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -38,8 +38,10 @@
-
- #include <loc/libloc.h>
- #include <loc/as.h>
-+#include <loc/as-list.h>
- #include <loc/compat.h>
- #include <loc/country.h>
-+#include <loc/country-list.h>
- #include <loc/database.h>
- #include <loc/format.h>
- #include <loc/network.h>
-@@ -99,11 +101,14 @@ struct loc_database_enumerator {
-
- // Search string
- char* string;
-- char country_code[3];
-- uint32_t asn;
-+ struct loc_country_list* countries;
-+ struct loc_as_list* asns;
- enum loc_network_flags flags;
- int family;
-
-+ // Flatten output?
-+ int flatten;
-+
- // Index of the AS we are looking at
- unsigned int as_index;
-
-@@ -115,6 +120,9 @@ struct loc_database_enumerator {
- struct loc_node_stack network_stack[MAX_STACK_DEPTH];
- int network_stack_depth;
- unsigned int* networks_visited;
-+
-+ // For subnet search
-+ struct loc_network_list* stack;
- };
-
- static int loc_database_read_magic(struct loc_database* db) {
-@@ -242,11 +250,11 @@ static int loc_database_read_signature(struct loc_database* db,
- char** dst, char* src, size_t length) {
- // Check for a plausible signature length
- if (length > LOC_SIGNATURE_MAX_LENGTH) {
-- ERROR(db->ctx, "Signature too long: %ld\n", length);
-+ ERROR(db->ctx, "Signature too long: %zu\n", length);
- return -EINVAL;
- }
-
-- DEBUG(db->ctx, "Reading signature of %ld bytes\n", length);
-+ DEBUG(db->ctx, "Reading signature of %zu bytes\n", length);
-
- // Allocate space
- *dst = malloc(length);
-@@ -611,7 +619,7 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
- }
-
- clock_t end = clock();
-- DEBUG(db->ctx, "Signature checked in %.4fms\n",
-+ INFO(db->ctx, "Signature checked in %.4fms\n",
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-
- CLEANUP:
-@@ -671,8 +679,10 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
- off_t lo = 0;
- off_t hi = db->as_count - 1;
-
-+#ifdef ENABLE_DEBUG
- // Save start time
- clock_t start = clock();
-+#endif
-
- while (lo <= hi) {
- off_t i = (lo + hi) / 2;
-@@ -685,11 +695,13 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
- // Check if this is a match
- uint32_t as_number = loc_as_get_number(*as);
- if (as_number == number) {
-+#ifdef ENABLE_DEBUG
- clock_t end = clock();
-
- // Log how fast this has been
- DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-+#endif
-
- return 0;
- }
-@@ -733,11 +745,13 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
- return -1;
- }
-
-+#ifdef ENABLE_DEBUG
- if (r == 0) {
- char* string = loc_network_str(*network);
- DEBUG(db->ctx, "Got network %s\n", string);
- free(string);
- }
-+#endif
-
- return r;
- }
-@@ -762,8 +776,7 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
- }
-
- // Check if the given IP address is inside the network
-- r = loc_network_match_address(*network, address);
-- if (r) {
-+ if (!loc_network_match_address(*network, address)) {
- DEBUG(db->ctx, "Searched address is not part of the network\n");
-
- loc_network_unref(*network);
-@@ -832,17 +845,21 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db,
-
- *network = NULL;
-
-+#ifdef ENABLE_DEBUG
- // Save start time
- clock_t start = clock();
-+#endif
-
- int r = __loc_database_lookup(db, address, network, &network_address,
- db->network_nodes_v1, 0);
-
-+#ifdef ENABLE_DEBUG
- clock_t end = clock();
-
- // Log how fast this has been
- DEBUG(db->ctx, "Executed network search in %.4fms\n",
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-+#endif
-
- return r;
- }
-@@ -889,8 +906,10 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
- off_t lo = 0;
- off_t hi = db->countries_count - 1;
-
-+#ifdef ENABLE_DEBUG
- // Save start time
- clock_t start = clock();
-+#endif
-
- while (lo <= hi) {
- off_t i = (lo + hi) / 2;
-@@ -905,11 +924,13 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
- int result = strcmp(code, cc);
-
- if (result == 0) {
-+#ifdef ENABLE_DEBUG
- clock_t end = clock();
-
- // Log how fast this has been
- DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
- (double)(end - start) / CLOCKS_PER_SEC * 1000);
-+#endif
-
- return 0;
- }
-@@ -932,8 +953,34 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
-
- // Enumerator
-
-+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
-+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
-+
-+ // Release all references
-+ loc_database_unref(enumerator->db);
-+ loc_unref(enumerator->ctx);
-+
-+ if (enumerator->string)
-+ free(enumerator->string);
-+
-+ if (enumerator->countries)
-+ loc_country_list_unref(enumerator->countries);
-+
-+ if (enumerator->asns)
-+ loc_as_list_unref(enumerator->asns);
-+
-+ // Free network search
-+ free(enumerator->networks_visited);
-+
-+ // Free subnet stack
-+ if (enumerator->stack)
-+ loc_network_list_unref(enumerator->stack);
-+
-+ free(enumerator);
-+}
-+
- LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
-- struct loc_database* db, enum loc_database_enumerator_mode mode) {
-+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
- struct loc_database_enumerator* e = calloc(1, sizeof(*e));
- if (!e)
- return -ENOMEM;
-@@ -944,11 +991,20 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
- e->mode = mode;
- e->refcount = 1;
-
-+ // Flatten output?
-+ e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
-+
- // Initialise graph search
-- //e->network_stack[++e->network_stack_depth] = 0;
- e->network_stack_depth = 1;
- e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
-
-+ // Allocate stack
-+ int r = loc_network_list_new(e->ctx, &e->stack);
-+ if (r) {
-+ loc_database_enumerator_free(e);
-+ return r;
-+ }
-+
- DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
-
- *enumerator = e;
-@@ -961,22 +1017,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
- return enumerator;
- }
-
--static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
-- DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
--
-- // Release all references
-- loc_database_unref(enumerator->db);
-- loc_unref(enumerator->ctx);
--
-- if (enumerator->string)
-- free(enumerator->string);
--
-- // Free network search
-- free(enumerator->networks_visited);
--
-- free(enumerator);
--}
--
- LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
- if (!enumerator)
- return NULL;
-@@ -998,40 +1038,38 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
- return 0;
- }
-
--LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
-- // Set empty country code
-- if (!country_code || !*country_code) {
-- *enumerator->country_code = '\0';
-- return 0;
-- }
-+LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
-+ struct loc_database_enumerator* enumerator) {
-+ if (!enumerator->countries)
-+ return NULL;
-
-- // Treat A1, A2, A3 as special country codes,
-- // but perform search for flags instead
-- if (strcmp(country_code, "A1") == 0) {
-- return loc_database_enumerator_set_flag(enumerator,
-- LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
-- } else if (strcmp(country_code, "A2") == 0) {
-- return loc_database_enumerator_set_flag(enumerator,
-- LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
-- } else if (strcmp(country_code, "A3") == 0) {
-- return loc_database_enumerator_set_flag(enumerator,
-- LOC_NETWORK_FLAG_ANYCAST);
-- }
-+ return loc_country_list_ref(enumerator->countries);
-+}
-
-- // Country codes must be two characters
-- if (!loc_country_code_is_valid(country_code))
-- return -EINVAL;
-+LOC_EXPORT int loc_database_enumerator_set_countries(
-+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
-+ if (enumerator->countries)
-+ loc_country_list_unref(enumerator->countries);
-
-- for (unsigned int i = 0; i < 3; i++) {
-- enumerator->country_code[i] = country_code[i];
-- }
-+ enumerator->countries = loc_country_list_ref(countries);
-
- return 0;
- }
-
--LOC_EXPORT int loc_database_enumerator_set_asn(
-- struct loc_database_enumerator* enumerator, unsigned int asn) {
-- enumerator->asn = asn;
-+LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
-+ struct loc_database_enumerator* enumerator) {
-+ if (!enumerator->asns)
-+ return NULL;
-+
-+ return loc_as_list_ref(enumerator->asns);
-+}
-+
-+LOC_EXPORT int loc_database_enumerator_set_asns(
-+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
-+ if (enumerator->asns)
-+ loc_as_list_unref(enumerator->asns);
-+
-+ enumerator->asns = loc_as_list_ref(asns);
-
- return 0;
- }
-@@ -1110,16 +1148,64 @@ static int loc_database_enumerator_stack_push_node(
- return 0;
- }
-
--LOC_EXPORT int loc_database_enumerator_next_network(
-- struct loc_database_enumerator* enumerator, struct loc_network** network) {
-- // Reset network
-- *network = NULL;
-+static int loc_database_enumerator_filter_network(
-+ struct loc_database_enumerator* enumerator, struct loc_network* network) {
-+ // Skip if the family does not match
-+ if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
-+ return 1;
-+ }
-
-- // Do not do anything if not in network mode
-- if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
-- return 0;
-+ // Skip if the country code does not match
-+ if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
-+ const char* country_code = loc_network_get_country_code(network);
-
-- int r;
-+ if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
-+ return 1;
-+ }
-+ }
-+
-+ // Skip if the ASN does not match
-+ if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
-+ uint32_t asn = loc_network_get_asn(network);
-+
-+ if (!loc_as_list_contains_number(enumerator->asns, asn)) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
-+ return 1;
-+ }
-+ }
-+
-+ // Skip if flags do not match
-+ if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
-+ DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
-+ return 1;
-+ }
-+
-+ // Do not filter
-+ return 0;
-+}
-+
-+static int __loc_database_enumerator_next_network(
-+ struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
-+ // Return top element from the stack
-+ while (1) {
-+ *network = loc_network_list_pop_first(enumerator->stack);
-+
-+ // Stack is empty
-+ if (!*network)
-+ break;
-+
-+ // Throw away any networks by filter
-+ if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
-+ loc_network_unref(*network);
-+ *network = NULL;
-+ continue;
-+ }
-+
-+ // Return result
-+ return 0;
-+ }
-
- DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
- enumerator->network_stack_depth);
-@@ -1149,7 +1235,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
- enumerator->db->network_nodes_v1 + node->offset;
-
- // Add edges to stack
-- r = loc_database_enumerator_stack_push_node(enumerator,
-+ int r = loc_database_enumerator_stack_push_node(enumerator,
- be32toh(n->one), 1, node->depth + 1);
-
- if (r)
-@@ -1175,54 +1261,142 @@ LOC_EXPORT int loc_database_enumerator_next_network(
- if (r)
- return r;
-
-- // Check if we are interested in this network
-+ // Return all networks when the filter is disabled
-+ if (!filter)
-+ return 0;
-
-- // Skip if the family does not match
-- if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
-+ // Check if we are interested in this network
-+ if (loc_database_enumerator_filter_network(enumerator, *network)) {
- loc_network_unref(*network);
- *network = NULL;
-
- continue;
- }
-
-- // Skip if the country code does not match
-- if (*enumerator->country_code &&
-- !loc_network_match_country_code(*network, enumerator->country_code)) {
-- loc_network_unref(*network);
-- *network = NULL;
-+ return 0;
-+ }
-+ }
-
-- continue;
-- }
-+ // Reached the end of the search
-+ return 0;
-+}
-
-- // Skip if the ASN does not match
-- if (enumerator->asn &&
-- !loc_network_match_asn(*network, enumerator->asn)) {
-- loc_network_unref(*network);
-- *network = NULL;
-+static int __loc_database_enumerator_next_network_flattened(
-+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
-+ // Fetch the next network
-+ int r = __loc_database_enumerator_next_network(enumerator, network, 1);
-+ if (r)
-+ return r;
-
-- continue;
-- }
-+ // End if we could not read another network
-+ if (!*network)
-+ return 0;
-
-- // Skip if flags do not match
-- if (enumerator->flags &&
-- !loc_network_match_flag(*network, enumerator->flags)) {
-- loc_network_unref(*network);
-- *network = NULL;
-+ struct loc_network* subnet = NULL;
-+ struct loc_network_list* subnets;
-
-- continue;
-+ // Create a list with all subnets
-+ r = loc_network_list_new(enumerator->ctx, &subnets);
-+ if (r)
-+ return r;
-+
-+ // Search all subnets from the database
-+ while (1) {
-+ // Fetch the next network in line
-+ r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
-+ if (r) {
-+ loc_network_unref(subnet);
-+ loc_network_list_unref(subnets);
-+
-+ return r;
-+ }
-+
-+ // End if we did not receive another subnet
-+ if (!subnet)
-+ break;
-+
-+ // Collect all subnets in a list
-+ if (loc_network_is_subnet(*network, subnet)) {
-+ r = loc_network_list_push(subnets, subnet);
-+ if (r) {
-+ loc_network_unref(subnet);
-+ loc_network_list_unref(subnets);
-+
-+ return r;
- }
-
-- return 0;
-+ loc_network_unref(subnet);
-+ continue;
-+ }
-+
-+ // If this is not a subnet, we push it back onto the stack and break
-+ r = loc_network_list_push(enumerator->stack, subnet);
-+ if (r) {
-+ loc_network_unref(subnet);
-+ loc_network_list_unref(subnets);
-+
-+ return r;
- }
-+
-+ loc_network_unref(subnet);
-+ break;
- }
-
-- // Reached the end of the search
-+ DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
-
-- // Mark all nodes as non-visited
-- for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
-- enumerator->networks_visited[i] = 0;
-+ // We can abort here if the network has no subnets
-+ if (loc_network_list_empty(subnets)) {
-+ loc_network_list_unref(subnets);
-
-- return 0;
-+ return 0;
-+ }
-+
-+ // If the network has any subnets, we will break it into smaller parts
-+ // without the subnets.
-+ struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
-+ if (!excluded) {
-+ loc_network_list_unref(subnets);
-+ return -1;
-+ }
-+
-+ // Merge subnets onto the stack
-+ r = loc_network_list_merge(enumerator->stack, subnets);
-+ if (r) {
-+ loc_network_list_unref(subnets);
-+ loc_network_list_unref(excluded);
-+
-+ return r;
-+ }
-+
-+ // Push excluded list onto the stack
-+ r = loc_network_list_merge(enumerator->stack, excluded);
-+ if (r) {
-+ loc_network_list_unref(subnets);
-+ loc_network_list_unref(excluded);
-+
-+ return r;
-+ }
-+
-+ loc_network_list_unref(subnets);
-+ loc_network_list_unref(excluded);
-+
-+ // Drop the network and restart the whole process again to pick the next network
-+ loc_network_unref(*network);
-+
-+ return __loc_database_enumerator_next_network_flattened(enumerator, network);
-+}
-+
-+LOC_EXPORT int loc_database_enumerator_next_network(
-+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
-+ // Do not do anything if not in network mode
-+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
-+ return 0;
-+
-+ // Flatten output?
-+ if (enumerator->flatten)
-+ return __loc_database_enumerator_next_network_flattened(enumerator, network);
-+
-+ return __loc_database_enumerator_next_network(enumerator, network, 1);
- }
-
- LOC_EXPORT int loc_database_enumerator_next_country(
-diff --git a/src/libloc.sym b/src/libloc.sym
-index b8296eb..ee333f1 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -37,6 +37,18 @@ global:
- loc_as_set_name;
- loc_as_unref;
-
-+ # AS List
-+ loc_as_list_append;
-+ loc_as_list_clear;
-+ loc_as_list_contains;
-+ loc_as_list_contains_number;
-+ loc_as_list_empty;
-+ loc_as_list_get;
-+ loc_as_list_new;
-+ loc_as_list_ref;
-+ loc_as_list_size;
-+ loc_as_list_unref;
-+
- # Country
- loc_country_cmp;
- loc_country_code_is_valid;
-@@ -49,6 +61,18 @@ global:
- loc_country_set_name;
- loc_country_unref;
-
-+ # Country List
-+ loc_country_list_append;
-+ loc_country_list_clear;
-+ loc_country_list_contains;
-+ loc_country_list_contains_code;
-+ loc_country_list_empty;
-+ loc_country_list_get;
-+ loc_country_list_new;
-+ loc_country_list_ref;
-+ loc_country_list_size;
-+ loc_country_list_unref;
-+
- # Database
- loc_database_add_as;
- loc_database_count_as;
-@@ -66,13 +90,15 @@ global:
- loc_database_verify;
-
- # Database Enumerator
-+ loc_database_enumerator_get_asns;
-+ loc_database_enumerator_get_countries;
- loc_database_enumerator_new;
- loc_database_enumerator_next_as;
- loc_database_enumerator_next_country;
- loc_database_enumerator_next_network;
- loc_database_enumerator_ref;
-- loc_database_enumerator_set_asn;
-- loc_database_enumerator_set_country_code;
-+ loc_database_enumerator_set_asns;
-+ loc_database_enumerator_set_countries;
- loc_database_enumerator_set_family;
- loc_database_enumerator_set_flag;
- loc_database_enumerator_set_string;
-@@ -80,24 +106,48 @@ global:
-
- # Network
- loc_network_address_family;
-+ loc_network_cmp;
-+ loc_network_exclude;
-+ loc_network_exclude_list;
- loc_network_format_first_address;
- loc_network_format_last_address;
- loc_network_get_asn;
- loc_network_get_country_code;
-+ loc_network_get_first_address;
-+ loc_network_get_last_address;
- loc_network_has_flag;
-- loc_network_is_subnet_of;
-+ loc_network_is_subnet;
-+ loc_network_match_address;
- loc_network_match_asn;
- loc_network_match_country_code;
- loc_network_match_flag;
- loc_network_new;
- loc_network_new_from_string;
-+ loc_network_overlaps;
-+ loc_network_prefix;
- loc_network_ref;
- loc_network_set_asn;
- loc_network_set_country_code;
- loc_network_set_flag;
- loc_network_str;
-+ loc_network_subnets;
- loc_network_unref;
-
-+ # Network List
-+ loc_network_list_clear;
-+ loc_network_list_contains;
-+ loc_network_list_dump;
-+ loc_network_list_empty;
-+ loc_network_list_get;
-+ loc_network_list_merge;
-+ loc_network_list_new;
-+ loc_network_list_pop;
-+ loc_network_list_pop_first;
-+ loc_network_list_push;
-+ loc_network_list_ref;
-+ loc_network_list_size;
-+ loc_network_list_unref;
-+
- # Writer
- loc_writer_add_as;
- loc_writer_add_country;
-diff --git a/src/loc/as-list.h b/src/loc/as-list.h
-new file mode 100644
-index 0000000..7b5c4e8
---- /dev/null
-+++ b/src/loc/as-list.h
-@@ -0,0 +1,41 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org>
-+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
-+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
-+
-+#ifndef LIBLOC_AS_LIST_H
-+#define LIBLOC_AS_LIST_H
-+
-+#include <loc/as.h>
-+#include <loc/libloc.h>
-+
-+struct loc_as_list;
-+
-+int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
-+struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
-+struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
-+
-+size_t loc_as_list_size(struct loc_as_list* list);
-+int loc_as_list_empty(struct loc_as_list* list);
-+void loc_as_list_clear(struct loc_as_list* list);
-+
-+struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
-+int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
-+
-+int loc_as_list_contains(
-+ struct loc_as_list* list, struct loc_as* as);
-+int loc_as_list_contains_number(
-+ struct loc_as_list* list, uint32_t number);
-+
-+#endif
-diff --git a/src/loc/country-list.h b/src/loc/country-list.h
-new file mode 100644
-index 0000000..a7f818a
---- /dev/null
-+++ b/src/loc/country-list.h
-@@ -0,0 +1,43 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org>
-+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
-+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
-+
-+#ifndef LIBLOC_COUNTRY_LIST_H
-+#define LIBLOC_COUNTRY_LIST_H
-+
-+#include <stdlib.h>
-+
-+#include <loc/libloc.h>
-+#include <loc/country.h>
-+
-+struct loc_country_list;
-+
-+int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
-+struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
-+struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
-+
-+size_t loc_country_list_size(struct loc_country_list* list);
-+int loc_country_list_empty(struct loc_country_list* list);
-+void loc_country_list_clear(struct loc_country_list* list);
-+
-+struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
-+int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
-+
-+int loc_country_list_contains(
-+ struct loc_country_list* list, struct loc_country* country);
-+int loc_country_list_contains_code(
-+ struct loc_country_list* list, const char* code);
-+
-+#endif
-diff --git a/src/loc/database.h b/src/loc/database.h
-index 43173dd..70801f0 100644
---- a/src/loc/database.h
-+++ b/src/loc/database.h
-@@ -25,6 +25,7 @@
- #include <loc/network.h>
- #include <loc/as.h>
- #include <loc/country.h>
-+#include <loc/country-list.h>
-
- struct loc_database;
- int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
-@@ -55,15 +56,24 @@ enum loc_database_enumerator_mode {
- LOC_DB_ENUMERATE_COUNTRIES = 3,
- };
-
-+enum loc_database_enumerator_flags {
-+ LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
-+};
-+
- struct loc_database_enumerator;
- int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
-- struct loc_database* db, enum loc_database_enumerator_mode mode);
-+ struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
- struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
- struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
-
- int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
--int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
--int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
-+struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
-+int loc_database_enumerator_set_countries(
-+ struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
-+struct loc_as_list* loc_database_enumerator_get_asns(
-+ struct loc_database_enumerator* enumerator);
-+int loc_database_enumerator_set_asns(
-+ struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
- int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
- int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
- int loc_database_enumerator_next_as(
-diff --git a/src/loc/network-list.h b/src/loc/network-list.h
-new file mode 100644
-index 0000000..bee21c4
---- /dev/null
-+++ b/src/loc/network-list.h
-@@ -0,0 +1,37 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2020 IPFire Development Team <info(a)ipfire.org>
-+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
-+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
-+
-+#ifndef LIBLOC_NETWORK_LIST_H
-+#define LIBLOC_NETWORK_LIST_H
-+
-+#include <loc/network.h>
-+
-+struct loc_network_list;
-+int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
-+struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
-+struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
-+size_t loc_network_list_size(struct loc_network_list* list);
-+int loc_network_list_empty(struct loc_network_list* list);
-+void loc_network_list_clear(struct loc_network_list* list);
-+void loc_network_list_dump(struct loc_network_list* list);
-+struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
-+int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
-+struct loc_network* loc_network_list_pop(struct loc_network_list* list);
-+struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
-+int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
-+int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
-+
-+#endif
-diff --git a/src/loc/network.h b/src/loc/network.h
-index 70c3803..af3dafd 100644
---- a/src/loc/network.h
-+++ b/src/loc/network.h
-@@ -21,6 +21,7 @@
-
- #include <loc/libloc.h>
- #include <loc/format.h>
-+#include <loc/network-list.h>
-
- enum loc_network_flags {
- LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
-@@ -37,8 +38,11 @@ struct loc_network* loc_network_ref(struct loc_network* network);
- struct loc_network* loc_network_unref(struct loc_network* network);
- char* loc_network_str(struct loc_network* network);
- int loc_network_address_family(struct loc_network* network);
-+unsigned int loc_network_prefix(struct loc_network* network);
-
-+const struct in6_addr* loc_network_get_first_address(struct loc_network* network);
- char* loc_network_format_first_address(struct loc_network* network);
-+const struct in6_addr* loc_network_get_last_address(struct loc_network* network);
- char* loc_network_format_last_address(struct loc_network* network);
- int loc_network_match_address(struct loc_network* network, const struct in6_addr* address);
-
-@@ -54,7 +58,14 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
- int loc_network_set_flag(struct loc_network* network, uint32_t flag);
- int loc_network_match_flag(struct loc_network* network, uint32_t flag);
-
--int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
-+int loc_network_cmp(struct loc_network* self, struct loc_network* other);
-+int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
-+int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
-+int loc_network_subnets(struct loc_network* network, struct loc_network** subnet1, struct loc_network** subnet2);
-+struct loc_network_list* loc_network_exclude(
-+ struct loc_network* self, struct loc_network* other);
-+struct loc_network_list* loc_network_exclude_list(
-+ struct loc_network* network, struct loc_network_list* list);
-
- #ifdef LIBLOC_PRIVATE
-
-diff --git a/src/network-list.c b/src/network-list.c
-new file mode 100644
-index 0000000..698d3ab
---- /dev/null
-+++ b/src/network-list.c
-@@ -0,0 +1,299 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2020 IPFire Development Team <info(a)ipfire.org>
-+
-+ This library is free software; you can redistribute it and/or
-+ modify it under the terms of the GNU Lesser General Public
-+ License as published by the Free Software Foundation; either
-+ version 2.1 of the License, or (at your option) any later version.
-+
-+ This library is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ Lesser General Public License for more details.
-+*/
-+
-+#include <errno.h>
-+#include <stdlib.h>
-+#include <time.h>
-+
-+#include <loc/libloc.h>
-+#include <loc/network.h>
-+#include <loc/private.h>
-+
-+struct loc_network_list {
-+ struct loc_ctx* ctx;
-+ int refcount;
-+
-+ struct loc_network** elements;
-+ size_t elements_size;
-+
-+ size_t size;
-+};
-+
-+static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
-+ DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
-+ list, size, list->elements_size + size);
-+
-+ struct loc_network** elements = reallocarray(list->elements,
-+ list->elements_size + size, sizeof(*list->elements));
-+ if (!elements)
-+ return -errno;
-+
-+ list->elements = elements;
-+ list->elements_size += size;
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
-+ struct loc_network_list** list) {
-+ struct loc_network_list* l = calloc(1, sizeof(*l));
-+ if (!l)
-+ return -ENOMEM;
-+
-+ l->ctx = loc_ref(ctx);
-+ l->refcount = 1;
-+
-+ DEBUG(l->ctx, "Network list allocated at %p\n", l);
-+ *list = l;
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
-+ list->refcount++;
-+
-+ return list;
-+}
-+
-+static void loc_network_list_free(struct loc_network_list* list) {
-+ DEBUG(list->ctx, "Releasing network list at %p\n", list);
-+
-+ // Remove all content
-+ loc_network_list_clear(list);
-+
-+ loc_unref(list->ctx);
-+ free(list);
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
-+ if (!list)
-+ return NULL;
-+
-+ if (--list->refcount > 0)
-+ return list;
-+
-+ loc_network_list_free(list);
-+ return NULL;
-+}
-+
-+LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
-+ return list->size;
-+}
-+
-+LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
-+ return list->size == 0;
-+}
-+
-+LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
-+ if (!list->elements)
-+ return;
-+
-+ for (unsigned int i = 0; i < list->size; i++)
-+ loc_network_unref(list->elements[i]);
-+
-+ free(list->elements);
-+ list->elements = NULL;
-+ list->elements_size = 0;
-+
-+ list->size = 0;
-+}
-+
-+LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
-+ struct loc_network* network;
-+ char* s;
-+
-+ for (unsigned int i = 0; i < list->size; i++) {
-+ network = list->elements[i];
-+
-+ s = loc_network_str(network);
-+
-+ INFO(list->ctx, "%4d: %s\n", i, s);
-+ free(s);
-+ }
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
-+ // Check index
-+ if (index >= list->size)
-+ return NULL;
-+
-+ return loc_network_ref(list->elements[index]);
-+}
-+
-+static off_t loc_network_list_find(struct loc_network_list* list,
-+ struct loc_network* network, int* found) {
-+ // Insert at the beginning for an empty list
-+ if (loc_network_list_empty(list))
-+ return 0;
-+
-+ off_t lo = 0;
-+ off_t hi = list->size - 1;
-+ int result;
-+
-+ // Since we are working on an ordered list, there is often a good chance that
-+ // the network we are looking for is at the end or has to go to the end.
-+ if (hi >= 0) {
-+ result = loc_network_cmp(network, list->elements[hi]);
-+
-+ // Match, so we are done
-+ if (result == 0) {
-+ *found = 1;
-+
-+ return hi;
-+
-+ // This needs to be added after the last one
-+ } else if (result > 0) {
-+ *found = 0;
-+
-+ return hi + 1;
-+ }
-+ }
-+
-+#ifdef ENABLE_DEBUG
-+ // Save start time
-+ clock_t start = clock();
-+#endif
-+
-+ off_t i = 0;
-+
-+ while (lo <= hi) {
-+ i = (lo + hi) / 2;
-+
-+ // Check if this is a match
-+ result = loc_network_cmp(network, list->elements[i]);
-+
-+ if (result == 0) {
-+ *found = 1;
-+
-+#ifdef ENABLE_DEBUG
-+ clock_t end = clock();
-+
-+ // Log how fast this has been
-+ DEBUG(list->ctx, "Found network in %.4fms at %jd\n",
-+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
-+#endif
-+
-+ return i;
-+ }
-+
-+ if (result > 0) {
-+ lo = i + 1;
-+ i++;
-+ } else {
-+ hi = i - 1;
-+ }
-+ }
-+
-+ *found = 0;
-+
-+#ifdef ENABLE_DEBUG
-+ clock_t end = clock();
-+
-+ // Log how fast this has been
-+ DEBUG(list->ctx, "Did not find network in %.4fms (last i = %jd)\n",
-+ (double)(end - start) / CLOCKS_PER_SEC * 1000, (intmax_t)i);
-+#endif
-+
-+ return i;
-+}
-+
-+LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
-+ int found = 0;
-+
-+ off_t index = loc_network_list_find(list, network, &found);
-+
-+ // The network has been found on the list. Nothing to do.
-+ if (found)
-+ return 0;
-+
-+ DEBUG(list->ctx, "%p: Inserting network %p at index %jd\n",
-+ list, network, (intmax_t)index);
-+
-+ // Check if we have space left
-+ if (list->size >= list->elements_size) {
-+ int r = loc_network_list_grow(list, 64);
-+ if (r)
-+ return r;
-+ }
-+
-+ // The list is now larger
-+ list->size++;
-+
-+ // Move all elements out of the way
-+ for (unsigned int i = list->size - 1; i > index; i--)
-+ list->elements[i] = list->elements[i - 1];
-+
-+ // Add the new element at the right place
-+ list->elements[index] = loc_network_ref(network);
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
-+ // Return nothing when empty
-+ if (loc_network_list_empty(list)) {
-+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-+ return NULL;
-+ }
-+
-+ struct loc_network* network = list->elements[--list->size];
-+
-+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-+
-+ return network;
-+}
-+
-+LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
-+ // Return nothing when empty
-+ if (loc_network_list_empty(list)) {
-+ DEBUG(list->ctx, "%p: Popped empty stack\n", list);
-+ return NULL;
-+ }
-+
-+ struct loc_network* network = list->elements[0];
-+
-+ // Move all elements to the top of the stack
-+ for (unsigned int i = 0; i < list->size - 1; i++) {
-+ list->elements[i] = list->elements[i+1];
-+ }
-+
-+ // The list is shorter now
-+ --list->size;
-+
-+ DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
-+
-+ return network;
-+}
-+
-+LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
-+ int found = 0;
-+
-+ loc_network_list_find(list, network, &found);
-+
-+ return found;
-+}
-+
-+LOC_EXPORT int loc_network_list_merge(
-+ struct loc_network_list* self, struct loc_network_list* other) {
-+ int r;
-+
-+ for (unsigned int i = 0; i < other->size; i++) {
-+ r = loc_network_list_push(self, other->elements[i]);
-+ if (r)
-+ return r;
-+ }
-+
-+ return 0;
-+}
-diff --git a/src/network.c b/src/network.c
-index 366caa2..a6b679c 100644
---- a/src/network.c
-+++ b/src/network.c
-@@ -29,6 +29,7 @@
- #include <loc/compat.h>
- #include <loc/country.h>
- #include <loc/network.h>
-+#include <loc/network-list.h>
- #include <loc/private.h>
-
- struct loc_network {
-@@ -97,6 +98,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
- return a;
- }
-
-+static struct in6_addr address_increment(const struct in6_addr* address) {
-+ struct in6_addr a = *address;
-+
-+ for (int octet = 15; octet >= 0; octet--) {
-+ if (a.s6_addr[octet] < 255) {
-+ a.s6_addr[octet]++;
-+ break;
-+ } else {
-+ a.s6_addr[octet] = 0;
-+ }
-+ }
-+
-+ return a;
-+}
-+
- LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
- struct in6_addr* address, unsigned int prefix) {
- // Address cannot be unspecified
-@@ -160,9 +176,11 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
- LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
- const char* address_string) {
- struct in6_addr first_address;
-- unsigned int prefix = 0;
- char* prefix_string;
-- int r = 1;
-+ unsigned int prefix = 128;
-+ int r = -EINVAL;
-+
-+ DEBUG(ctx, "Attempting to parse network %s\n", address_string);
-
- // Make a copy of the string to work on it
- char* buffer = strdup(address_string);
-@@ -171,29 +189,40 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
- // Split address and prefix
- address_string = strsep(&prefix_string, "/");
-
-- // Did we find a prefix?
-+ DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
-+
-+ // Parse the address
-+ r = loc_parse_address(ctx, address_string, &first_address);
-+ if (r) {
-+ DEBUG(ctx, "The address could not be parsed\n");
-+ goto FAIL;
-+ }
-+
-+ // If a prefix was given, we will try to parse it
- if (prefix_string) {
- // Convert prefix to integer
- prefix = strtol(prefix_string, NULL, 10);
-
-- if (prefix) {
-- // Parse the address
-- r = loc_parse_address(ctx, address_string, &first_address);
--
-- // Map the prefix to IPv6 if needed
-- if (IN6_IS_ADDR_V4MAPPED(&first_address))
-- prefix += 96;
-+ if (!prefix) {
-+ DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
-+ goto FAIL;
- }
-+
-+ // Map the prefix to IPv6 if needed
-+ if (IN6_IS_ADDR_V4MAPPED(&first_address))
-+ prefix += 96;
- }
-
-+FAIL:
- // Free temporary buffer
- free(buffer);
-
-- if (r == 0) {
-- r = loc_network_new(ctx, network, &first_address, prefix);
-- }
-+ // Exit if the parsing was unsuccessful
-+ if (r)
-+ return r;
-
-- return r;
-+ // Create a new network
-+ return loc_network_new(ctx, network, &first_address, prefix);
- }
-
- LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
-@@ -281,6 +310,18 @@ LOC_EXPORT int loc_network_address_family(struct loc_network* network) {
- return network->family;
- }
-
-+LOC_EXPORT unsigned int loc_network_prefix(struct loc_network* network) {
-+ switch (network->family) {
-+ case AF_INET6:
-+ return network->prefix;
-+
-+ case AF_INET:
-+ return network->prefix - 96;
-+ }
-+
-+ return 0;
-+}
-+
- static char* loc_network_format_address(struct loc_network* network, const struct in6_addr* address) {
- const size_t length = INET6_ADDRSTRLEN;
-
-@@ -314,10 +355,18 @@ static char* loc_network_format_address(struct loc_network* network, const struc
- return string;
- }
-
-+LOC_EXPORT const struct in6_addr* loc_network_get_first_address(struct loc_network* network) {
-+ return &network->first_address;
-+}
-+
- LOC_EXPORT char* loc_network_format_first_address(struct loc_network* network) {
- return loc_network_format_address(network, &network->first_address);
- }
-
-+LOC_EXPORT const struct in6_addr* loc_network_get_last_address(struct loc_network* network) {
-+ return &network->last_address;
-+}
-+
- LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
- return loc_network_format_address(network, &network->last_address);
- }
-@@ -325,14 +374,14 @@ LOC_EXPORT char* loc_network_format_last_address(struct loc_network* network) {
- LOC_EXPORT int loc_network_match_address(struct loc_network* network, const struct in6_addr* address) {
- // Address must be larger than the start address
- if (in6_addr_cmp(&network->first_address, address) > 0)
-- return 1;
-+ return 0;
-
- // Address must be smaller than the last address
- if (in6_addr_cmp(&network->last_address, address) < 0)
-- return 1;
-+ return 0;
-
- // The address is inside this network
-- return 0;
-+ return 1;
- }
-
- LOC_EXPORT const char* loc_network_get_country_code(struct loc_network* network) {
-@@ -392,20 +441,310 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
- return loc_network_has_flag(network, flag);
- }
-
--LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
-+LOC_EXPORT int loc_network_cmp(struct loc_network* self, struct loc_network* other) {
-+ // Compare address
-+ int r = in6_addr_cmp(&self->first_address, &other->first_address);
-+ if (r)
-+ return r;
-+
-+ // Compare prefix
-+ if (self->prefix > other->prefix)
-+ return 1;
-+ else if (self->prefix < other->prefix)
-+ return -1;
-+
-+ // Both networks are equal
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
-+ // Either of the start addresses must be in the other subnet
-+ if (loc_network_match_address(self, &other->first_address))
-+ return 1;
-+
-+ if (loc_network_match_address(other, &self->first_address))
-+ return 1;
-+
-+ // Or either of the end addresses is in the other subnet
-+ if (loc_network_match_address(self, &other->last_address))
-+ return 1;
-+
-+ if (loc_network_match_address(other, &self->last_address))
-+ return 1;
-+
-+ return 0;
-+}
-+
-+LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
-+ // The prefix must be smaller (this avoids the more complex comparisons later)
-+ if (self->prefix > other->prefix)
-+ return 0;
-+
- // If the start address of the other network is smaller than this network,
- // it cannot be a subnet.
-- if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
-+ if (in6_addr_cmp(&self->first_address, &other->first_address) > 0)
- return 0;
-
- // If the end address of the other network is greater than this network,
- // it cannot be a subnet.
-- if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
-+ if (in6_addr_cmp(&self->last_address, &other->last_address) < 0)
- return 0;
-
- return 1;
- }
-
-+LOC_EXPORT int loc_network_subnets(struct loc_network* network,
-+ struct loc_network** subnet1, struct loc_network** subnet2) {
-+ int r;
-+ *subnet1 = NULL;
-+ *subnet2 = NULL;
-+
-+ // New prefix length
-+ unsigned int prefix = network->prefix + 1;
-+
-+ // Check if the new prefix is valid
-+ if (valid_prefix(&network->first_address, prefix))
-+ return -1;
-+
-+ // Create the first half of the network
-+ r = loc_network_new(network->ctx, subnet1, &network->first_address, prefix);
-+ if (r)
-+ return r;
-+
-+ // The next subnet starts after the first one
-+ struct in6_addr first_address = address_increment(&(*subnet1)->last_address);
-+
-+ // Create the second half of the network
-+ r = loc_network_new(network->ctx, subnet2, &first_address, prefix);
-+ if (r)
-+ return r;
-+
-+ // Copy country code
-+ const char* country_code = loc_network_get_country_code(network);
-+ if (country_code) {
-+ loc_network_set_country_code(*subnet1, country_code);
-+ loc_network_set_country_code(*subnet2, country_code);
-+ }
-+
-+ // Copy ASN
-+ uint32_t asn = loc_network_get_asn(network);
-+ if (asn) {
-+ loc_network_set_asn(*subnet1, asn);
-+ loc_network_set_asn(*subnet2, asn);
-+ }
-+
-+ // Copy flags
-+ loc_network_set_flag(*subnet1, network->flags);
-+ loc_network_set_flag(*subnet2, network->flags);
-+
-+ return 0;
-+}
-+
-+static int __loc_network_exclude(struct loc_network* network,
-+ struct loc_network* other, struct loc_network_list* list) {
-+ struct loc_network* subnet1 = NULL;
-+ struct loc_network* subnet2 = NULL;
-+
-+ int r = loc_network_subnets(network, &subnet1, &subnet2);
-+ if (r)
-+ goto ERROR;
-+
-+ if (loc_network_cmp(other, subnet1) == 0) {
-+ r = loc_network_list_push(list, subnet2);
-+ if (r)
-+ goto ERROR;
-+
-+ } else if (loc_network_cmp(other, subnet2) == 0) {
-+ r = loc_network_list_push(list, subnet1);
-+ if (r)
-+ goto ERROR;
-+
-+ } else if (loc_network_is_subnet(subnet1, other)) {
-+ r = loc_network_list_push(list, subnet2);
-+ if (r)
-+ goto ERROR;
-+
-+ r = __loc_network_exclude(subnet1, other, list);
-+ if (r)
-+ goto ERROR;
-+
-+ } else if (loc_network_is_subnet(subnet2, other)) {
-+ r = loc_network_list_push(list, subnet1);
-+ if (r)
-+ goto ERROR;
-+
-+ r = __loc_network_exclude(subnet2, other, list);
-+ if (r)
-+ goto ERROR;
-+
-+ } else {
-+ ERROR(network->ctx, "We should never get here\n");
-+ r = 1;
-+ goto ERROR;
-+ }
-+
-+ERROR:
-+ if (subnet1)
-+ loc_network_unref(subnet1);
-+
-+ if (subnet2)
-+ loc_network_unref(subnet2);
-+
-+ return r;
-+}
-+
-+static int __loc_network_exclude_to_list(struct loc_network* self,
-+ struct loc_network* other, struct loc_network_list* list) {
-+ // Other must be a subnet of self
-+ if (!loc_network_is_subnet(self, other)) {
-+ DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
-+
-+ // Exit silently
-+ return 0;
-+ }
-+
-+ // We cannot perform this operation if both networks equal
-+ if (loc_network_cmp(self, other) == 0) {
-+ DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
-+
-+ // Exit silently
-+ return 0;
-+ }
-+
-+ return __loc_network_exclude(self, other, list);
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_exclude(
-+ struct loc_network* self, struct loc_network* other) {
-+ struct loc_network_list* list;
-+
-+#ifdef ENABLE_DEBUG
-+ char* n1 = loc_network_str(self);
-+ char* n2 = loc_network_str(other);
-+
-+ DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
-+
-+ free(n1);
-+ free(n2);
-+#endif
-+
-+ // Create a new list with the result
-+ int r = loc_network_list_new(self->ctx, &list);
-+ if (r) {
-+ ERROR(self->ctx, "Could not create network list: %d\n", r);
-+
-+ return NULL;
-+ }
-+
-+ r = __loc_network_exclude_to_list(self, other, list);
-+ if (r) {
-+ loc_network_list_unref(list);
-+
-+ return NULL;
-+ }
-+
-+ // Return the result
-+ return list;
-+}
-+
-+LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
-+ struct loc_network* network, struct loc_network_list* list) {
-+ struct loc_network_list* to_check;
-+
-+ // Create a new list with all networks to look at
-+ int r = loc_network_list_new(network->ctx, &to_check);
-+ if (r)
-+ return NULL;
-+
-+ struct loc_network* subnet = NULL;
-+ struct loc_network_list* subnets = NULL;
-+
-+ for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
-+ subnet = loc_network_list_get(list, i);
-+
-+ // Find all excluded networks
-+ if (!loc_network_list_contains(to_check, subnet)) {
-+ r = __loc_network_exclude_to_list(network, subnet, to_check);
-+ if (r) {
-+ loc_network_list_unref(to_check);
-+ loc_network_unref(subnet);
-+
-+ return NULL;
-+ }
-+ }
-+
-+ // Cleanup
-+ loc_network_unref(subnet);
-+ }
-+
-+ r = loc_network_list_new(network->ctx, &subnets);
-+ if (r) {
-+ loc_network_list_unref(to_check);
-+ return NULL;
-+ }
-+
-+ off_t smallest_subnet = 0;
-+
-+ while (!loc_network_list_empty(to_check)) {
-+ struct loc_network* subnet_to_check = loc_network_list_pop_first(to_check);
-+
-+ // Check whether the subnet to check is part of the input list
-+ if (loc_network_list_contains(list, subnet_to_check)) {
-+ loc_network_unref(subnet_to_check);
-+ continue;
-+ }
-+
-+ // Marks whether this subnet passed all checks
-+ int passed = 1;
-+
-+ for (unsigned int i = smallest_subnet; i < loc_network_list_size(list); i++) {
-+ subnet = loc_network_list_get(list, i);
-+
-+ // Drop this subnet if is a subnet of another subnet
-+ if (loc_network_is_subnet(subnet, subnet_to_check)) {
-+ passed = 0;
-+ loc_network_unref(subnet);
-+ break;
-+ }
-+
-+ // Break it down if it overlaps
-+ if (loc_network_overlaps(subnet, subnet_to_check)) {
-+ passed = 0;
-+
-+ __loc_network_exclude_to_list(subnet_to_check, subnet, to_check);
-+
-+ loc_network_unref(subnet);
-+ break;
-+ }
-+
-+ // If the subnet is strictly greater, we do not need to continue the search
-+ r = loc_network_cmp(subnet, subnet_to_check);
-+ if (r > 0) {
-+ loc_network_unref(subnet);
-+ break;
-+
-+ // If it is strictly smaller, we can continue the search from here next
-+ // time because all networks that are to be checked can only be larger
-+ // than this one.
-+ } else if (r < 0) {
-+ smallest_subnet = i;
-+ }
-+
-+ loc_network_unref(subnet);
-+ }
-+
-+ if (passed) {
-+ r = loc_network_list_push(subnets, subnet_to_check);
-+ }
-+
-+ loc_network_unref(subnet_to_check);
-+ }
-+
-+ loc_network_list_unref(to_check);
-+
-+ return subnets;
-+}
-+
- LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
- // Add country code
- loc_country_code_copy(dbobj->country_code, network->country_code);
-@@ -474,7 +813,7 @@ struct loc_network_tree_node {
- struct loc_network* network;
- };
-
--LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
-+int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
- struct loc_network_tree* t = calloc(1, sizeof(*t));
- if (!t)
- return -ENOMEM;
-@@ -494,7 +833,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
- return 0;
- }
-
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
-+struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
- return loc_network_tree_node_ref(tree->root);
- }
-
-@@ -566,7 +905,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
- return 0;
- }
-
--LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
-+int loc_network_tree_walk(struct loc_network_tree* tree,
- int(*filter_callback)(struct loc_network* network, void* data),
- int(*callback)(struct loc_network* network, void* data), void* data) {
- return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
-@@ -581,7 +920,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
- free(tree);
- }
-
--LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
-+struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
- if (--tree->refcount > 0)
- return tree;
-
-@@ -602,13 +941,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
- return 0;
- }
-
--LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
-+int loc_network_tree_dump(struct loc_network_tree* tree) {
- DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
-
- return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
- }
-
--LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
-+int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
- DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
-
- struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
-@@ -639,7 +978,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
- return 0;
- }
-
--LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
-+size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
- size_t counter = 0;
-
- int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
-@@ -661,11 +1000,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
- return counter;
- }
-
--LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
-+size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
- return __loc_network_tree_count_nodes(tree->root);
- }
-
--LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
-+int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
- struct loc_network_tree_node* n = calloc(1, sizeof(*n));
- if (!n)
- return -ENOMEM;
-@@ -680,7 +1019,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
- return 0;
- }
-
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
-+struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
- if (node)
- node->refcount++;
-
-@@ -703,7 +1042,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
- free(node);
- }
-
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
-+struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
- if (!node)
- return NULL;
-
-@@ -714,7 +1053,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
- return NULL;
- }
-
--LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
-+struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
- if (index == 0)
- node = node->zero;
- else
-@@ -726,10 +1065,10 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
- return loc_network_tree_node_ref(node);
- }
-
--LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
-+int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
- return (!!node->network);
- }
-
--LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
-+struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
- return loc_network_ref(node->network);
- }
-diff --git a/src/perl/Location.xs b/src/perl/Location.xs
-index dcf3f0d..b7676d2 100644
---- a/src/perl/Location.xs
-+++ b/src/perl/Location.xs
-@@ -125,7 +125,7 @@ database_countries(db)
- PPCODE:
- // Create Database enumerator
- struct loc_database_enumerator* enumerator;
-- int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
-+ int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
-
- if (err) {
- croak("Could not create a database enumerator\n");
-diff --git a/src/python/database.c b/src/python/database.c
-index 1013a58..0aa03cc 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -17,6 +17,8 @@
- #include <Python.h>
-
- #include <loc/libloc.h>
-+#include <loc/as.h>
-+#include <loc/as-list.h>
- #include <loc/database.h>
-
- #include "locationmodule.h"
-@@ -207,10 +209,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
- return (PyObject*)self;
- }
-
--static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
-+static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
- struct loc_database_enumerator* enumerator;
-
-- int r = loc_database_enumerator_new(&enumerator, self->db, what);
-+ int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
- return NULL;
-@@ -223,7 +225,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
- }
-
- static PyObject* Database_ases(DatabaseObject* self) {
-- return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
- }
-
- static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
-@@ -234,7 +236,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
-
- struct loc_database_enumerator* enumerator;
-
-- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
-+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
- return NULL;
-@@ -250,44 +252,142 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
- }
-
- static PyObject* Database_networks(DatabaseObject* self) {
-- return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
-+}
-+
-+static PyObject* Database_networks_flattened(DatabaseObject *self) {
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
- }
-
- static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
-- char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
-- const char* country_code = NULL;
-- unsigned int asn = 0;
-+ char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
-+ PyObject* country_codes = NULL;
-+ PyObject* asn_list = NULL;
- int flags = 0;
- int family = 0;
-+ int flatten = 0;
-
-- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
-+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
-+ &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
- return NULL;
-
- struct loc_database_enumerator* enumerator;
-- int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
-+ int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
-+ (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
- return NULL;
- }
-
- // Set country code we are searching for
-- if (country_code) {
-- r = loc_database_enumerator_set_country_code(enumerator, country_code);
-+ if (country_codes) {
-+ struct loc_country_list* countries;
-+ r = loc_country_list_new(loc_ctx, &countries);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not create country list");
-+ return NULL;
-+ }
-+
-+ for (int i = 0; i < PyList_Size(country_codes); i++) {
-+ PyObject* item = PyList_GetItem(country_codes, i);
-+
-+ if (!PyUnicode_Check(item)) {
-+ PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
-+ loc_country_list_unref(countries);
-+ return NULL;
-+ }
-+
-+ const char* country_code = PyUnicode_AsUTF8(item);
-+
-+ struct loc_country* country;
-+ r = loc_country_new(loc_ctx, &country, country_code);
-+ if (r) {
-+ if (r == -EINVAL) {
-+ PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
-+ } else {
-+ PyErr_SetString(PyExc_SystemError, "Could not create country");
-+ }
-+
-+ loc_country_list_unref(countries);
-+ return NULL;
-+ }
-+
-+ // Append it to the list
-+ r = loc_country_list_append(countries, country);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
-+
-+ loc_country_list_unref(countries);
-+ loc_country_unref(country);
-+ return NULL;
-+ }
-+
-+ loc_country_unref(country);
-+ }
-
-+ r = loc_database_enumerator_set_countries(enumerator, countries);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
-+
-+ loc_country_list_unref(countries);
- return NULL;
- }
-+
-+ loc_country_list_unref(countries);
- }
-
- // Set the ASN we are searching for
-- if (asn) {
-- r = loc_database_enumerator_set_asn(enumerator, asn);
-+ if (asn_list) {
-+ struct loc_as_list* asns;
-+ r = loc_as_list_new(loc_ctx, &asns);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not create AS list");
-+ return NULL;
-+ }
-+
-+ for (int i = 0; i < PyList_Size(asn_list); i++) {
-+ PyObject* item = PyList_GetItem(asn_list, i);
-+
-+ if (!PyLong_Check(item)) {
-+ PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
-
-+ loc_as_list_unref(asns);
-+ return NULL;
-+ }
-+
-+ unsigned long number = PyLong_AsLong(item);
-+
-+ struct loc_as* as;
-+ r = loc_as_new(loc_ctx, &as, number);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not create AS");
-+
-+ loc_as_list_unref(asns);
-+ loc_as_unref(as);
-+ return NULL;
-+ }
-+
-+ r = loc_as_list_append(asns, as);
-+ if (r) {
-+ PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
-+
-+ loc_as_list_unref(asns);
-+ loc_as_unref(as);
-+ return NULL;
-+ }
-+
-+ loc_as_unref(as);
-+ }
-+
-+ r = loc_database_enumerator_set_asns(enumerator, asns);
- if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
-+
-+ loc_as_list_unref(asns);
- return NULL;
- }
-+
-+ loc_as_list_unref(asns);
- }
-
- // Set the flags we are searching for
-@@ -317,7 +417,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
- }
-
- static PyObject* Database_countries(DatabaseObject* self) {
-- return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
- }
-
- static struct PyMethodDef Database_methods[] = {
-@@ -403,6 +503,13 @@ static struct PyGetSetDef Database_getsetters[] = {
- NULL,
- NULL,
- },
-+ {
-+ "networks_flattened",
-+ (getter)Database_networks_flattened,
-+ NULL,
-+ NULL,
-+ NULL,
-+ },
- {
- "vendor",
- (getter)Database_get_vendor,
-diff --git a/src/python/downloader.py b/src/python/downloader.py
-index 87bbb68..05f7872 100644
---- a/src/python/downloader.py
-+++ b/src/python/downloader.py
-@@ -119,8 +119,8 @@ class Downloader(object):
-
- headers = {}
- if timestamp:
-- headers["If-Modified-Since"] = timestamp.strftime(
-- "%a, %d %b %Y %H:%M:%S GMT",
-+ headers["If-Modified-Since"] = time.strftime(
-+ "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp),
- )
-
- t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
-@@ -195,7 +195,7 @@ class Downloader(object):
- db = Database(f.name)
-
- # Database is not recent
-- if timestamp and db.created_at < timestamp.timestamp():
-+ if timestamp and db.created_at < timestamp:
- return False
-
- log.info("Downloaded new database from %s" % (time.strftime(
-diff --git a/src/python/export.py b/src/python/export.py
-index d15c6f0..f0eae26 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -29,7 +29,7 @@ import _location
- log = logging.getLogger("location.export")
- log.propagate = 1
-
--flags = {
-+FLAGS = {
- _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
- _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
- _location.NETWORK_FLAG_ANYCAST : "A3",
-@@ -39,11 +39,8 @@ class OutputWriter(object):
- suffix = "networks"
- mode = "w"
-
-- def __init__(self, f, prefix=None, flatten=True):
-- self.f, self.prefix, self.flatten = f, prefix, flatten
--
-- # The previously written network
-- self._last_network = None
-+ def __init__(self, f, prefix=None):
-+ self.f, self.prefix = f, prefix
-
- # Immediately write the header
- self._write_header()
-@@ -60,18 +57,6 @@ class OutputWriter(object):
- def __repr__(self):
- return "<%s f=%s>" % (self.__class__.__name__, self.f)
-
-- def _flatten(self, network):
-- """
-- Checks if the given network needs to be written to file,
-- or if it is a subnet of the previously written network.
-- """
-- if self._last_network and network.is_subnet_of(self._last_network):
-- return True
--
-- # Remember this network for the next call
-- self._last_network = network
-- return False
--
- def _write_header(self):
- """
- The header of the file
-@@ -84,16 +69,8 @@ class OutputWriter(object):
- """
- pass
-
-- def _write_network(self, network):
-- self.f.write("%s\n" % network)
--
- def write(self, network):
-- if self.flatten and self._flatten(network):
-- log.debug("Skipping writing network %s" % network)
-- return
--
-- # Write the network to file
-- self._write_network(network)
-+ self.f.write("%s\n" % network)
-
- def finish(self):
- """
-@@ -114,7 +91,7 @@ class IpsetOutputWriter(OutputWriter):
- def _write_header(self):
- self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
-
-- def _write_network(self, network):
-+ def write(self, network):
- self.f.write("add %s %s\n" % (self.prefix, network))
-
-
-@@ -130,7 +107,7 @@ class NftablesOutputWriter(OutputWriter):
- def _write_footer(self):
- self.f.write("}\n")
-
-- def _write_network(self, network):
-+ def write(self, network):
- self.f.write(" %s,\n" % network)
-
-
-@@ -142,14 +119,9 @@ class XTGeoIPOutputWriter(OutputWriter):
- suffix = "iv"
- mode = "wb"
-
-- def _write_network(self, network):
-- for address in (network.first_address, network.last_address):
-- # Convert this into a string of bits
-- bytes = socket.inet_pton(
-- network.family, address,
-- )
--
-- self.f.write(bytes)
-+ def write(self, network):
-+ self.f.write(network._first_address)
-+ self.f.write(network._last_address)
-
-
- formats = {
-@@ -185,8 +157,14 @@ class Exporter(object):
-
- writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
-
-+ # Filter countries from special country codes
-+ country_codes = [
-+ country_code for country_code in countries if not country_code in FLAGS.values()
-+ ]
-+
- # Get all networks that match the family
-- networks = self.db.search_networks(family=family)
-+ networks = self.db.search_networks(family=family,
-+ country_codes=country_codes, asns=asns, flatten=True)
-
- # Walk through all networks
- for network in networks:
-@@ -203,10 +181,10 @@ class Exporter(object):
- pass
-
- # Handle flags
-- for flag in flags:
-+ for flag in FLAGS:
- if network.has_flag(flag):
- # Fetch the "fake" country code
-- country = flags[flag]
-+ country = FLAGS[flag]
-
- try:
- writers[country].write(network)
-diff --git a/src/python/importer.py b/src/python/importer.py
-index f19db4b..5f46bc3 100644
---- a/src/python/importer.py
-+++ b/src/python/importer.py
-@@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
- "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
-
- # Latin America and Caribbean Network Information Centre
-- "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-+ "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
-
- # Réseaux IP Européens
- #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-diff --git a/src/python/location-importer.in b/src/python/location-importer.in
-index 1467923..2dec89e 100644
---- a/src/python/location-importer.in
-+++ b/src/python/location-importer.in
-@@ -152,6 +152,7 @@ class CLI(object):
- last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
- CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
- CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
-+ CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
-
- -- autnums
- CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
-@@ -165,6 +166,7 @@ class CLI(object):
- -- networks
- CREATE TABLE IF NOT EXISTS networks(network inet, country text);
- CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
-+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
- CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
-
- -- overrides
-@@ -188,6 +190,8 @@ class CLI(object):
- );
- CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
- ON network_overrides(network);
-+ CREATE INDEX IF NOT EXISTS network_overrides_search
-+ ON network_overrides USING GIST(network inet_ops);
- """)
-
- return db
-@@ -234,32 +238,24 @@ class CLI(object):
-
- # Select all known networks
- rows = self.db.query("""
-- -- Get a (sorted) list of all known networks
-- WITH known_networks AS (
-- SELECT network FROM announcements
-- UNION
-- SELECT network FROM networks
-- ORDER BY network
-- )
--
- -- Return a list of those networks enriched with all
- -- other information that we store in the database
- SELECT
-- DISTINCT ON (known_networks.network)
-- known_networks.network AS network,
-- announcements.autnum AS autnum,
-+ DISTINCT ON (network)
-+ network,
-+ autnum,
-
- -- Country
- COALESCE(
- (
- SELECT country FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT country FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- networks.country
- ) AS country,
-@@ -268,50 +264,67 @@ class CLI(object):
- COALESCE(
- (
- SELECT is_anonymous_proxy FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT is_anonymous_proxy FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- FALSE
- ) AS is_anonymous_proxy,
- COALESCE(
- (
- SELECT is_satellite_provider FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT is_satellite_provider FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- FALSE
- ) AS is_satellite_provider,
- COALESCE(
- (
- SELECT is_anycast FROM network_overrides overrides
-- WHERE announcements.network <<= overrides.network
-+ WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
- ),
- (
- SELECT is_anycast FROM autnum_overrides overrides
-- WHERE announcements.autnum = overrides.number
-+ WHERE networks.autnum = overrides.number
- ),
- FALSE
-- ) AS is_anycast,
--
-- -- Must be part of returned values for ORDER BY clause
-- masklen(announcements.network) AS sort_a,
-- masklen(networks.network) AS sort_b
-- FROM known_networks
-- LEFT JOIN announcements ON known_networks.network <<= announcements.network
-- LEFT JOIN networks ON known_networks.network <<= networks.network
-- ORDER BY known_networks.network, sort_a DESC, sort_b DESC
-+ ) AS is_anycast
-+ FROM (
-+ SELECT
-+ known_networks.network AS network,
-+ announcements.autnum AS autnum,
-+ networks.country AS country,
-+
-+ -- Must be part of returned values for ORDER BY clause
-+ masklen(announcements.network) AS sort_a,
-+ masklen(networks.network) AS sort_b
-+ FROM (
-+ SELECT network FROM announcements
-+ UNION ALL
-+ SELECT network FROM networks
-+ UNION ALL
-+ SELECT network FROM network_overrides
-+ ) known_networks
-+ LEFT JOIN
-+ announcements ON known_networks.network <<= announcements.network
-+ LEFT JOIN
-+ networks ON known_networks.network <<= networks.network
-+ ORDER BY
-+ known_networks.network,
-+ sort_a DESC,
-+ sort_b DESC
-+ ) networks
- """)
-
- for row in rows:
-@@ -363,6 +376,16 @@ class CLI(object):
- CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
- ON COMMIT DROP;
- CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
-+
-+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
-+ ON COMMIT DROP;
-+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
-+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
-+ """)
-+
-+ # Remove all previously imported content
-+ self.db.execute("""
-+ TRUNCATE TABLE networks;
- """)
-
- for source in location.importer.WHOIS_SOURCES:
-@@ -370,31 +393,72 @@ class CLI(object):
- for block in f:
- self._parse_block(block)
-
-+ # Process all parsed networks from every RIR we happen to have access to,
-+ # insert the largest network chunks into the networks table immediately...
-+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
-+
-+ for family in (row.family for row in families):
-+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
-+
-+ self.db.execute("INSERT INTO networks(network, country) \
-+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
-+
-+ # ... determine any other prefixes for this network family, ...
-+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
-+
-+ # ... and insert networks with this prefix in case they provide additional
-+ # information (i. e. subnet of a larger chunk with a different country)
-+ for prefix in (row.prefix for row in prefixes):
-+ self.db.execute("""
-+ WITH candidates AS (
-+ SELECT
-+ _rirdata.network,
-+ _rirdata.country
-+ FROM
-+ _rirdata
-+ WHERE
-+ family(_rirdata.network) = %s
-+ AND
-+ masklen(_rirdata.network) = %s
-+ ),
-+ filtered AS (
-+ SELECT
-+ DISTINCT ON (c.network)
-+ c.network,
-+ c.country,
-+ masklen(networks.network),
-+ networks.country AS parent_country
-+ FROM
-+ candidates c
-+ LEFT JOIN
-+ networks
-+ ON
-+ c.network << networks.network
-+ ORDER BY
-+ c.network,
-+ masklen(networks.network) DESC NULLS LAST
-+ )
-+ INSERT INTO
-+ networks(network, country)
-+ SELECT
-+ network,
-+ country
-+ FROM
-+ filtered
-+ WHERE
-+ parent_country IS NULL
-+ OR
-+ country <> parent_country
-+ ON CONFLICT DO NOTHING""",
-+ family, prefix,
-+ )
-+
- self.db.execute("""
- INSERT INTO autnums(number, name)
- SELECT _autnums.number, _organizations.name FROM _autnums
- JOIN _organizations ON _autnums.organization = _organizations.handle
-- ON CONFLICT (number) DO UPDATE SET name = excluded.name
-- """)
--
-- self.db.execute("""
-- --- Purge any redundant entries
-- CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
-- AS
-- SELECT network FROM networks candidates
-- WHERE EXISTS (
-- SELECT FROM networks
-- WHERE
-- networks.network << candidates.network
-- AND
-- networks.country = candidates.country
-- );
--
-- CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
--
-- DELETE FROM networks WHERE EXISTS (
-- SELECT FROM _garbage WHERE networks.network = _garbage.network
-- );
-+ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
- """)
-
- # Download all extended sources
-@@ -405,6 +469,69 @@ class CLI(object):
- for line in f:
- self._parse_line(line)
-
-+ def _check_parsed_network(self, network):
-+ """
-+ Assistive function to detect and subsequently sort out parsed
-+ networks from RIR data (both Whois and so-called "extended sources"),
-+ which are or have...
-+
-+ (a) not globally routable (RFC 1918 space, et al.)
-+ (b) covering a too large chunk of the IP address space (prefix length
-+ is < 7 for IPv4 networks, and < 10 for IPv6)
-+ (c) "0.0.0.0" or "::" as a network address
-+ (d) are too small for being publicly announced (we have decided not to
-+ process them at the moment, as they significantly enlarge our
-+ database without providing very helpful additional information)
-+
-+ This unfortunately is necessary due to brain-dead clutter across
-+ various RIR databases, causing mismatches and eventually disruptions.
-+
-+ We will return False in case a network is not suitable for adding
-+ it to our database, and True otherwise.
-+ """
-+
-+ if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
-+ return False
-+
-+ if not network.is_global:
-+ log.warning("Skipping non-globally routable network: %s" % network)
-+ return False
-+
-+ if network.version == 4:
-+ if network.prefixlen < 7:
-+ log.warning("Skipping too big IP chunk: %s" % network)
-+ return False
-+
-+ if network.prefixlen > 24:
-+ log.debug("Skipping network too small to be publicly announced: %s" % network)
-+ return False
-+
-+ if str(network.network_address) == "0.0.0.0":
-+ log.warning("Skipping network based on 0.0.0.0: %s" % network)
-+ return False
-+
-+ elif network.version == 6:
-+ if network.prefixlen < 10:
-+ log.warning("Skipping too big IP chunk: %s" % network)
-+ return False
-+
-+ if network.prefixlen > 48:
-+ log.debug("Skipping network too small to be publicly announced: %s" % network)
-+ return False
-+
-+ if str(network.network_address) == "::":
-+ log.warning("Skipping network based on '::': %s" % network)
-+ return False
-+
-+ else:
-+ # This should not happen...
-+ log.warning("Skipping network of unknown family, this should not happen: %s" % network)
-+ return False
-+
-+ # In case we have made it here, the network is considered to
-+ # be suitable for libloc consumption...
-+ return True
-+
- def _parse_block(self, block):
- # Get first line to find out what type of block this is
- line = block[0]
-@@ -433,7 +560,7 @@ class CLI(object):
- autnum["asn"] = m.group(2)
-
- elif key == "org":
-- autnum[key] = val
-+ autnum[key] = val.upper()
-
- # Skip empty objects
- if not autnum:
-@@ -447,15 +574,22 @@ class CLI(object):
- )
-
- def _parse_inetnum_block(self, block):
-- logging.debug("Parsing inetnum block:")
-+ log.debug("Parsing inetnum block:")
-
- inetnum = {}
- for line in block:
-- logging.debug(line)
-+ log.debug(line)
-
- # Split line
- key, val = split_line(line)
-
-+ # Filter any inetnum records which are only referring to IP space
-+ # not managed by that specific RIR...
-+ if key == "netname":
-+ if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
-+ log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
-+ return
-+
- if key == "inetnum":
- start_address, delim, end_address = val.partition("-")
-
-@@ -467,7 +601,7 @@ class CLI(object):
- start_address = ipaddress.ip_address(start_address)
- end_address = ipaddress.ip_address(end_address)
- except ValueError:
-- logging.warning("Could not parse line: %s" % line)
-+ log.warning("Could not parse line: %s" % line)
- return
-
- # Set prefix to default
-@@ -484,23 +618,24 @@ class CLI(object):
- inetnum[key] = val
-
- elif key == "country":
-- if val == "UNITED STATES":
-- val = "US"
--
- inetnum[key] = val.upper()
-
- # Skip empty objects
-- if not inetnum:
-+ if not inetnum or not "country" in inetnum:
-+ return
-+
-+ # Skip objects with bogus country code 'ZZ'
-+ if inetnum.get("country") == "ZZ":
-+ log.warning("Skipping network with bogus country 'ZZ': %s" % \
-+ (inetnum.get("inet6num") or inetnum.get("inetnum")))
- return
-
- network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-- # Bail out in case we have processed a non-public IP network
-- if network.is_private:
-- logging.warning("Skipping non-globally routable network: %s" % network)
-+ if not self._check_parsed_network(network):
- return
-
-- self.db.execute("INSERT INTO networks(network, country) \
-+ self.db.execute("INSERT INTO _rirdata(network, country) \
- VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
- "%s" % network, inetnum.get("country"),
- )
-@@ -511,7 +646,9 @@ class CLI(object):
- # Split line
- key, val = split_line(line)
-
-- if key in ("organisation", "org-name"):
-+ if key == "organisation":
-+ org[key] = val.upper()
-+ elif key == "org-name":
- org[key] = val
-
- # Skip empty objects
-@@ -581,6 +718,9 @@ class CLI(object):
- log.warning("Invalid IP address: %s" % address)
- return
-
-+ if not self._check_parsed_network(network):
-+ return
-+
- self.db.execute("INSERT INTO networks(network, country) \
- VALUES(%s, %s) ON CONFLICT (network) DO \
- UPDATE SET country = excluded.country",
-diff --git a/src/python/location.in b/src/python/location.in
-index 44ad726..b30beae 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -253,6 +253,7 @@ class CLI(object):
- network = db.lookup(address)
- except ValueError:
- print(_("Invalid IP address: %s") % address, file=sys.stderr)
-+ return 2
-
- args = {
- "address" : address,
-@@ -398,10 +399,7 @@ class CLI(object):
-
- def handle_update(self, db, ns):
- if ns.cron and db:
-- now = datetime.datetime.utcnow()
--
-- # Parse the database timestamp
-- t = datetime.datetime.utcfromtimestamp(db.created_at)
-+ now = time.time()
-
- if ns.cron == "daily":
- delta = datetime.timedelta(days=1)
-@@ -410,22 +408,20 @@ class CLI(object):
- elif ns.cron == "monthly":
- delta = datetime.timedelta(days=30)
-
-+ delta = delta.total_seconds()
-+
- # Check if the database has recently been updated
-- if t >= (now - delta):
-+ if db.created_at >= (now - delta):
- log.info(
-- _("The database has been updated recently (%s)") % \
-- format_timedelta(now - t),
-+ _("The database has been updated recently"),
- )
- return 3
-
- # Fetch the timestamp we need from DNS
- t = location.discover_latest_version()
-
-- # Parse timestamp into datetime format
-- timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
--
- # Check the version of the local database
-- if db and timestamp and db.created_at >= timestamp.timestamp():
-+ if db and t and db.created_at >= t:
- log.info("Already on the latest version")
- return
-
-@@ -437,7 +433,7 @@ class CLI(object):
-
- # Try downloading a new database
- try:
-- t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
-+ t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
-
- # If no file could be downloaded, log a message
- except FileNotFoundError as e:
-@@ -453,13 +449,7 @@ class CLI(object):
-
- return 0
-
-- def handle_verify(self, ns):
-- try:
-- db = location.Database(ns.database)
-- except FileNotFoundError as e:
-- log.error("%s: %s" % (ns.database, e))
-- return 127
--
-+ def handle_verify(self, db, ns):
- # Verify the database
- with open(ns.public_key, "r") as f:
- if not db.verify(f):
-diff --git a/src/python/network.c b/src/python/network.c
-index 5496d1e..5b1369d 100644
---- a/src/python/network.c
-+++ b/src/python/network.c
-@@ -17,13 +17,33 @@
- #include <Python.h>
-
- #include <errno.h>
-+#include <limits.h>
-
- #include <loc/libloc.h>
- #include <loc/network.h>
-+#include <loc/network-list.h>
-
- #include "locationmodule.h"
- #include "network.h"
-
-+static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
-+ PyObject* list = PyList_New(0);
-+ if (!networks)
-+ return list;
-+
-+ while (!loc_network_list_empty(networks)) {
-+ struct loc_network* network = loc_network_list_pop(networks);
-+
-+ PyObject* n = new_network(&NetworkType, network);
-+ PyList_Append(list, n);
-+
-+ loc_network_unref(network);
-+ Py_DECREF(n);
-+ }
-+
-+ return list;
-+}
-+
- PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
- NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
- if (self) {
-@@ -114,10 +134,18 @@ static int Network_set_asn(NetworkObject* self, PyObject* value) {
- long int asn = PyLong_AsLong(value);
-
- // Check if the ASN is within the valid range
-- if (asn <= 0 || asn > UINT32_MAX) {
-+ if (asn <= 0) {
-+ PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
-+ return -1;
-+ }
-+
-+#if (__WORDSIZE > 32)
-+ // Check whether the input was longer than 32 bit
-+ if (asn > UINT32_MAX) {
- PyErr_Format(PyExc_ValueError, "Invalid ASN %ld", asn);
- return -1;
- }
-+#endif
-
- int r = loc_network_set_asn(self->network, asn);
- if (r)
-@@ -154,13 +182,28 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
- Py_RETURN_NONE;
- }
-
-+static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
-+ NetworkObject* other = NULL;
-+
-+ if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
-+ return NULL;
-+
-+ struct loc_network_list* list = loc_network_exclude(self->network, other->network);
-+
-+ // Convert to Python objects
-+ PyObject* obj = PyList_FromNetworkList(list);
-+ loc_network_list_unref(list);
-+
-+ return obj;
-+}
-+
- static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
- NetworkObject* other = NULL;
-
- if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
- return NULL;
-
-- if (loc_network_is_subnet_of(self->network, other->network))
-+ if (loc_network_is_subnet(other->network, self->network))
- Py_RETURN_TRUE;
-
- Py_RETURN_FALSE;
-@@ -181,6 +224,26 @@ static PyObject* Network_get_first_address(NetworkObject* self) {
- return obj;
- }
-
-+static PyObject* PyBytes_FromAddress(const struct in6_addr* address6) {
-+ struct in_addr address4;
-+
-+ // Convert IPv4 addresses to struct in_addr
-+ if (IN6_IS_ADDR_V4MAPPED(address6)) {
-+ address4.s_addr = address6->s6_addr32[3];
-+
-+ return PyBytes_FromStringAndSize((const char*)&address4, sizeof(address4));
-+ }
-+
-+ // Return IPv6 addresses as they are
-+ return PyBytes_FromStringAndSize((const char*)address6, sizeof(*address6));
-+}
-+
-+static PyObject* Network_get__first_address(NetworkObject* self) {
-+ const struct in6_addr* address = loc_network_get_first_address(self->network);
-+
-+ return PyBytes_FromAddress(address);
-+}
-+
- static PyObject* Network_get_last_address(NetworkObject* self) {
- char* address = loc_network_format_last_address(self->network);
-
-@@ -190,7 +253,19 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
- return obj;
- }
-
-+static PyObject* Network_get__last_address(NetworkObject* self) {
-+ const struct in6_addr* address = loc_network_get_last_address(self->network);
-+
-+ return PyBytes_FromAddress(address);
-+}
-+
- static struct PyMethodDef Network_methods[] = {
-+ {
-+ "exclude",
-+ (PyCFunction)Network_exclude,
-+ METH_VARARGS,
-+ NULL,
-+ },
- {
- "has_flag",
- (PyCFunction)Network_has_flag,
-@@ -241,6 +316,13 @@ static struct PyGetSetDef Network_getsetters[] = {
- NULL,
- NULL,
- },
-+ {
-+ "_first_address",
-+ (getter)Network_get__first_address,
-+ NULL,
-+ NULL,
-+ NULL,
-+ },
- {
- "last_address",
- (getter)Network_get_last_address,
-@@ -248,6 +330,13 @@ static struct PyGetSetDef Network_getsetters[] = {
- NULL,
- NULL,
- },
-+ {
-+ "_last_address",
-+ (getter)Network_get__last_address,
-+ NULL,
-+ NULL,
-+ NULL,
-+ },
- { NULL },
- };
-
-diff --git a/src/test-as.c b/src/test-as.c
-index 839a04c..2d61675 100644
---- a/src/test-as.c
-+++ b/src/test-as.c
-@@ -95,7 +95,7 @@ int main(int argc, char** argv) {
- // Enumerator
-
- struct loc_database_enumerator* enumerator;
-- err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
-+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
- if (err) {
- fprintf(stderr, "Could not create a database enumerator\n");
- exit(EXIT_FAILURE);
-diff --git a/src/test-database.c b/src/test-database.c
-index b4a75c4..da4b11c 100644
---- a/src/test-database.c
-+++ b/src/test-database.c
-@@ -38,6 +38,14 @@ const char* DESCRIPTION =
- "Maecenas ut venenatis nunc.";
- const char* LICENSE = "CC";
-
-+const char* networks[] = {
-+ "2001:db8::/32",
-+ "2001:db8:1000::/48",
-+ "2001:db8:2000::/48",
-+ "2001:db8:2020::/48",
-+ NULL,
-+};
-+
- static int attempt_to_open(struct loc_ctx* ctx, char* path) {
- FILE* f = fopen(path, "r");
- if (!f)
-@@ -139,6 +147,24 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ struct loc_network* network = NULL;
-+
-+ // Add some networks
-+ const char** n = networks;
-+ while (*n) {
-+ err = loc_writer_add_network(writer, &network, *n);
-+ if (err) {
-+ fprintf(stderr, "Could not add network %s\n", *n);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Set a country
-+ loc_network_set_country_code(network, "XX");
-+
-+ // Next one
-+ n++;
-+ }
-+
- FILE* f = tmpfile();
- if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
-@@ -170,6 +196,33 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ // Enumerator
-+ struct loc_database_enumerator* enumerator;
-+ err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
-+ if (err) {
-+ fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Walk through all networks
-+ while (1) {
-+ err = loc_database_enumerator_next_network(enumerator, &network);
-+ if (err) {
-+ fprintf(stderr, "Error fetching the next network: %d\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (!network)
-+ break;
-+
-+ char* s = loc_network_str(network);
-+ printf("Got network: %s\n", s);
-+ free(s);
-+ }
-+
-+ // Free the enumerator
-+ loc_database_enumerator_unref(enumerator);
-+
- // Close the database
- loc_database_unref(db);
- loc_unref(ctx);
-diff --git a/src/test-network-list.c b/src/test-network-list.c
-new file mode 100644
-index 0000000..6f32ff7
---- /dev/null
-+++ b/src/test-network-list.c
-@@ -0,0 +1,183 @@
-+/*
-+ libloc - A library to determine the location of someone on the Internet
-+
-+ Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org>
-+
-+ This program is free software; you can redistribute it and/or modify
-+ it under the terms of the GNU General Public License as published by
-+ the Free Software Foundation; either version 2 of the License, or
-+ (at your option) any later version.
-+
-+ This program is distributed in the hope that it will be useful,
-+ but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-+ GNU General Public License for more details.
-+*/
-+
-+#include <errno.h>
-+#include <stdio.h>
-+#include <stddef.h>
-+#include <stdlib.h>
-+#include <string.h>
-+#include <syslog.h>
-+
-+#include <loc/libloc.h>
-+#include <loc/network.h>
-+#include <loc/network-list.h>
-+
-+int main(int argc, char** argv) {
-+ int err;
-+
-+ struct loc_ctx* ctx;
-+ err = loc_new(&ctx);
-+ if (err < 0)
-+ exit(EXIT_FAILURE);
-+
-+ // Enable debug logging
-+ loc_set_log_priority(ctx, LOG_DEBUG);
-+
-+ // Create a network
-+ struct loc_network* network1;
-+ err = loc_network_new_from_string(ctx, &network1, "2001:db8::/32");
-+ if (err) {
-+ fprintf(stderr, "Could not create the network1\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet1;
-+ err = loc_network_new_from_string(ctx, &subnet1, "2001:db8:a::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet1\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet2;
-+ err = loc_network_new_from_string(ctx, &subnet2, "2001:db8:b::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet2\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet3;
-+ err = loc_network_new_from_string(ctx, &subnet3, "2001:db8:c::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet3\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet4;
-+ err = loc_network_new_from_string(ctx, &subnet4, "2001:db8:d::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet4\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet5;
-+ err = loc_network_new_from_string(ctx, &subnet5, "2001:db8:e::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet5\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ struct loc_network* subnet6;
-+ err = loc_network_new_from_string(ctx, &subnet6, "2001:db8:1::/48");
-+ if (err) {
-+ fprintf(stderr, "Could not create the subnet6\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Make a list with both subnets
-+ struct loc_network_list* subnets;
-+ err = loc_network_list_new(ctx, &subnets);
-+ if (err) {
-+ fprintf(stderr, "Could not create subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ size_t size = loc_network_list_size(subnets);
-+ if (size > 0) {
-+ fprintf(stderr, "The list is not empty: %zu\n", size);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ err = loc_network_list_push(subnets, subnet1);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet1 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (loc_network_list_empty(subnets)) {
-+ fprintf(stderr, "The subnets list reports that it is empty\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ err = loc_network_list_push(subnets, subnet2);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet2 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Add the fourth one next
-+ err = loc_network_list_push(subnets, subnet4);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet4 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Add the third one
-+ err = loc_network_list_push(subnets, subnet3);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet3 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Add more subnets
-+ err = loc_network_list_push(subnets, subnet5);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet5 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ err = loc_network_list_push(subnets, subnet6);
-+ if (err) {
-+ fprintf(stderr, "Could not add subnet6 to subnets list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(subnets);
-+
-+ size = loc_network_list_size(subnets);
-+ if (size != 6) {
-+ fprintf(stderr, "Network list is reporting an incorrect size: %zu\n", size);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Exclude subnet1 from network1
-+ struct loc_network_list* excluded = loc_network_exclude(network1, subnet1);
-+ if (!excluded) {
-+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnet1\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(excluded);
-+
-+ // Exclude all subnets from network1
-+ excluded = loc_network_exclude_list(network1, subnets);
-+ if (!excluded) {
-+ fprintf(stderr, "Received an empty result from loc_network_exclude() for subnets\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(excluded);
-+
-+ if (excluded)
-+ loc_network_list_unref(excluded);
-+
-+ loc_network_list_unref(subnets);
-+ loc_network_unref(network1);
-+ loc_network_unref(subnet1);
-+ loc_network_unref(subnet2);
-+ loc_unref(ctx);
-+
-+ return EXIT_SUCCESS;
-+}
-diff --git a/src/test-network.c b/src/test-network.c
-index d38f13d..dde13f1 100644
---- a/src/test-network.c
-+++ b/src/test-network.c
-@@ -14,6 +14,7 @@
- GNU General Public License for more details.
- */
-
-+#include <arpa/inet.h>
- #include <errno.h>
- #include <stdio.h>
- #include <stddef.h>
-@@ -37,12 +38,21 @@ int main(int argc, char** argv) {
- // Enable debug logging
- loc_set_log_priority(ctx, LOG_DEBUG);
-
-+#if 0
- struct loc_network_tree* tree;
- err = loc_network_tree_new(ctx, &tree);
- if (err) {
- fprintf(stderr, "Could not create the network tree\n");
- exit(EXIT_FAILURE);
- }
-+#endif
-+
-+ struct in6_addr address;
-+ err = inet_pton(AF_INET6, "2001:db8::1", &address);
-+ if (err != 1) {
-+ fprintf(stderr, "Could not parse IP address\n");
-+ exit(EXIT_FAILURE);
-+ }
-
- // Create a network
- struct loc_network* network1;
-@@ -58,12 +68,14 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+#if 0
- // Adding network to the tree
- err = loc_network_tree_add_network(tree, network1);
- if (err) {
- fprintf(stderr, "Could not add network to the tree\n");
- exit(EXIT_FAILURE);
- }
-+#endif
-
- // Check if the first and last addresses are correct
- char* string = loc_network_format_first_address(network1);
-@@ -88,6 +100,12 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+ err = loc_network_match_address(network1, &address);
-+ if (!err) {
-+ fprintf(stderr, "Network1 does not match address\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
- struct loc_network* network2;
- err = loc_network_new_from_string(ctx, &network2, "2001:db8:ffff::/48");
- if (err) {
-@@ -101,6 +119,7 @@ int main(int argc, char** argv) {
- exit(EXIT_FAILURE);
- }
-
-+#if 0
- // Adding network to the tree
- err = loc_network_tree_add_network(tree, network2);
- if (err) {
-@@ -117,20 +136,84 @@ int main(int argc, char** argv) {
-
- size_t nodes = loc_network_tree_count_nodes(tree);
- printf("The tree has %zu nodes\n", nodes);
-+#endif
-+
-+ // Check equals function
-+ err = loc_network_cmp(network1, network1);
-+ if (err) {
-+ fprintf(stderr, "Network is not equal with itself\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ err = loc_network_cmp(network1, network2);
-+ if (!err) {
-+ fprintf(stderr, "Networks equal unexpectedly\n");
-+ exit(EXIT_FAILURE);
-+ }
-
- // Check subnet function
-- err = loc_network_is_subnet_of(network1, network2);
-- if (err != 0) {
-+ err = loc_network_is_subnet(network1, network2);
-+ if (!err) {
- fprintf(stderr, "Subnet check 1 failed: %d\n", err);
- exit(EXIT_FAILURE);
- }
-
-- err = loc_network_is_subnet_of(network2, network1);
-- if (err != 1) {
-+ err = loc_network_is_subnet(network2, network1);
-+ if (err) {
- fprintf(stderr, "Subnet check 2 failed: %d\n", err);
- exit(EXIT_FAILURE);
- }
-
-+ // Make subnets
-+ struct loc_network* subnet1 = NULL;
-+ struct loc_network* subnet2 = NULL;
-+
-+ err = loc_network_subnets(network1, &subnet1, &subnet2);
-+ if (err || !subnet1 || !subnet2) {
-+ fprintf(stderr, "Could not find subnets of network: %d\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ char* s = loc_network_str(subnet1);
-+ printf("Received subnet1 = %s\n", s);
-+ free(s);
-+
-+ s = loc_network_str(subnet2);
-+ printf("Received subnet2 = %s\n", s);
-+ free(s);
-+
-+ if (!loc_network_is_subnet(network1, subnet1)) {
-+ fprintf(stderr, "Subnet1 is not a subnet\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (!loc_network_is_subnet(network1, subnet2)) {
-+ fprintf(stderr, "Subnet2 is not a subnet\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (!loc_network_overlaps(network1, subnet1)) {
-+ fprintf(stderr, "Network1 does not seem to contain subnet1\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (!loc_network_overlaps(network1, subnet2)) {
-+ fprintf(stderr, "Network1 does not seem to contain subnet2\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_unref(subnet1);
-+ loc_network_unref(subnet2);
-+
-+ struct loc_network_list* excluded = loc_network_exclude(network1, network2);
-+ if (!excluded) {
-+ fprintf(stderr, "Could not create excluded list\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ loc_network_list_dump(excluded);
-+ loc_network_list_unref(excluded);
-+
- // Create a database
- struct loc_writer* writer;
- err = loc_writer_new(ctx, &writer, NULL, NULL);
-@@ -160,6 +243,28 @@ int main(int argc, char** argv) {
- // Set ASN
- loc_network_set_asn(network4, 1024);
-
-+ // Try adding an invalid network
-+ struct loc_network* network;
-+ err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
-+ if (err != -EINVAL) {
-+ fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Try adding a single address
-+ err = loc_writer_add_network(writer, &network, "2001:db8::");
-+ if (err) {
-+ fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ // Try adding localhost
-+ err = loc_writer_add_network(writer, &network, "::1/128");
-+ if (err != -EINVAL) {
-+ fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
-+ exit(EXIT_FAILURE);
-+ }
-+
- FILE* f = tmpfile();
- if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
-@@ -177,7 +282,10 @@ int main(int argc, char** argv) {
- loc_network_unref(network2);
- loc_network_unref(network3);
- loc_network_unref(network4);
-+
-+#if 0
- loc_network_tree_unref(tree);
-+#endif
-
- // And open it again from disk
- struct loc_database* db;
-diff --git a/src/writer.c b/src/writer.c
-index 5939cff..c61a6df 100644
---- a/src/writer.c
-+++ b/src/writer.c
-@@ -147,8 +147,19 @@ static void loc_writer_free(struct loc_writer* writer) {
- EVP_PKEY_free(writer->private_key2);
-
- // Unref all AS
-- for (unsigned int i = 0; i < writer->as_count; i++) {
-- loc_as_unref(writer->as[i]);
-+ if (writer->as) {
-+ for (unsigned int i = 0; i < writer->as_count; i++) {
-+ loc_as_unref(writer->as[i]);
-+ }
-+ free(writer->as);
-+ }
-+
-+ // Unref all countries
-+ if (writer->countries) {
-+ for (unsigned int i = 0; i < writer->countries_count; i++) {
-+ loc_country_unref(writer->countries[i]);
-+ }
-+ free(writer->countries);
- }
-
- // Release network tree
-@@ -601,7 +612,7 @@ static int loc_writer_create_signature(struct loc_writer* writer,
- goto END;
- }
-
-- DEBUG(writer->ctx, "Successfully generated signature of %lu bytes\n", *length);
-+ DEBUG(writer->ctx, "Successfully generated signature of %zu bytes\n", *length);
- r = 0;
-
- // Dump signature
diff --git a/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch b/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch
new file mode 100644
index 000000000..ea29885a8
--- /dev/null
+++ b/src/patches/libloc-0.9.5-location-Fix-list-networks-by-as.patch
@@ -0,0 +1,27 @@
+From b178117bac33b4b1e7ce341a6f2eec493cca13f8 Mon Sep 17 00:00:00 2001
+From: Michael Tremer <michael.tremer(a)ipfire.org>
+Date: Mon, 21 Dec 2020 16:25:46 +0000
+Subject: [PATCH] location: Fix list-networks-by-as
+
+Fixes: #12554
+Signed-off-by: Michael Tremer <michael.tremer(a)ipfire.org>
+---
+ src/python/location.in | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/python/location.in b/src/python/location.in
+index b30beae..ad2ccf5 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -493,7 +493,7 @@ class CLI(object):
+ f = writer(sys.stdout, prefix="AS%s" % asn)
+
+ # Print all matching networks
+- for n in db.search_networks(asn=asn, family=ns.family):
++ for n in db.search_networks(asns=[asn], family=ns.family):
+ f.write(n)
+
+ f.finish()
+--
+2.26.2
+
--
2.26.2
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2020-12-23 14:03 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-12-23 14:03 [PATCH] libloc: update to 0.9.5 and backport fix for #12554 Peter Müller
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox