From mboxrd@z Thu Jan  1 00:00:00 1970
From: Michael Tremer <michael.tremer@ipfire.org>
To: location@lists.ipfire.org
Subject: [PATCH 01/10] importer: Store geofeed URLs from RIR data
Date: Tue, 27 Sep 2022 16:48:38 +0000
Message-ID: <20220927164847.3409646-1-michael.tremer@ipfire.org>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="===============0871696854905953350=="
List-Id: <location.lists.ipfire.org>

--===============0871696854905953350==
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable

Signed-off-by: Michael Tremer <michael.tremer(a)ipfire.org>
---
 src/scripts/location-importer.in | 40 +++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer=
.in
index 9faf23b..5bd5da3 100644
--- a/src/scripts/location-importer.in
+++ b/src/scripts/location-importer.in
@@ -182,6 +182,11 @@ class CLI(object):
 				CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(famil=
y(network));
 				CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(networ=
k inet_ops);
=20
+				-- geofeeds
+				CREATE TABLE IF NOT EXISTS network_geofeeds(network inet, url text);
+				CREATE UNIQUE INDEX IF NOT EXISTS network_geofeeds_unique
+					ON network_geofeeds(network);
+
 				-- overrides
 				CREATE TABLE IF NOT EXISTS autnum_overrides(
 					number bigint NOT NULL,
@@ -799,6 +804,16 @@ class CLI(object):
=20
 				inetnum[key].append(val)
=20
+			# Parse the geofeed attribute
+			elif key =3D=3D "geofeed":
+				inetnum["geofeed"] =3D val
+
+			# Parse geofeed when used as a remark
+			elif key =3D=3D "remark":
+				m =3D re.match(r"^(?:geofeed|Geofeed)\s+(https://.*)", val)
+				if m:
+					inetnum["geofeed"] =3D m.group(1)
+
 		# Skip empty objects
 		if not inetnum or not "country" in inetnum:
 			return
@@ -810,7 +825,6 @@ class CLI(object):
 		# them into the database, if _check_parsed_network() succeeded
 		for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"):
 			if self._check_parsed_network(single_network):
-
 				# Skip objects with unknown country codes if they are valid to avoid log=
 spam...
 				if validcountries and invalidcountries:
 					log.warning("Skipping network with bogus countr(y|ies) %s (original cou=
ntries: %s): %s" % \
@@ -823,6 +837,30 @@ class CLI(object):
 					"%s" % single_network, inetnum.get("country")[0], inetnum.get("country"=
), source_key,
 				)
=20
+				# Update any geofeed information
+				geofeed =3D inetnum.get("geofeed", None)
+
+				# Store/update any geofeeds
+				if geofeed:
+					self.db.execute("""
+						INSERT INTO
+							network_geofeeds(
+								network,
+								url
+							)
+						VALUES(
+							%s, %s
+						)
+						ON CONFLICT (network) DO
+							UPDATE SET url =3D excluded.url""",
+						"%s" % single_network, geofeed,
+					)
+
+				# Delete any previous geofeeds
+				else:
+					self.db.execute("DELETE FROM network_geofeeds WHERE network =3D %s",
+						"%s" % single_network)
+
 	def _parse_org_block(self, block, source_key):
 		org =3D {}
 		for line in block:
--=20
2.30.2


--===============0871696854905953350==--