From mboxrd@z Thu Jan  1 00:00:00 1970
From: Peter =?utf-8?q?M=C3=BCller?= <peter.mueller@ipfire.org>
To: location@lists.ipfire.org
Subject: Re-introducing inetnum parser, first attempt
Date: Tue, 01 Sep 2020 19:44:37 +0000
Message-ID: <46ac5f3c-36e4-c1a5-a9f6-fd9f6f4d21e9@ipfire.org>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="===============1177484414183951707=="
List-Id: <location.lists.ipfire.org>

--===============1177484414183951707==
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable

Good evening Michael,

below comes the diff of a rather hacky attempt to bring back the intetnum par=
ser we've
had in the past. Since I am not quite sure about that INSERT INTO networks() =
SQL statement
and handling conflicts with extended source files there, I thought letting yo=
u have a
look at it might be a good idea. :-)

What do you think? Is this the right direction?

Thanks, and best regards,
Peter M=C3=BCller

diff --git a/src/python/importer.py b/src/python/importer.py
index de20f37..586bd97 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES =3D (
 	"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
=20
 	# Asia Pacific Network Information Centre
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+	"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+	"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
 	"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES =3D (
 	# XXX ???
=20
 	# R=C3=A9seaux IP Europ=C3=A9ens
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
 	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index f5ae4a9..4d7cec4 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -393,6 +393,10 @@ class CLI(object):
 		if line.startswith("aut-num:"):
 			return self._parse_autnum_block(block)
=20
+		# inetnum
+		if line.startswith("inet6num:") or line.startswith("inetnum:"):
+			return self._parse_inetnum_block(block)
+
 		# organisation
 		elif line.startswith("organisation:"):
 			return self._parse_org_block(block)
@@ -422,6 +426,78 @@ class CLI(object):
 			autnum.get("asn"), autnum.get("org"),
 		)
=20
+	def _parse_inetnum_block(self, block):
+		logging.debug("Parsing inetnum block:")
+
+		inetnum =3D {}
+		for line in block:
+			logging.debug(line)
+
+			# Split line
+			key, val =3D split_line(line)
+
+			if key =3D=3D "inetnum":
+				start_address, delim, end_address =3D val.partition("-")
+
+				# Strip any excess space
+				start_address, end_address =3D start_address.rstrip(), end_address.strip=
()
+
+				# Skip invalid blocks
+				if start_address in ["0.0.0.0", "::/0", "0::/0",]:
+					return
+
+				# Convert to IP address
+				try:
+					start_address =3D ipaddress.ip_address(start_address)
+					end_address   =3D ipaddress.ip_address(end_address)
+				except ValueError:
+					logging.warning("Could not parse line: %s" % line)
+					return
+
+				# Set prefix to default
+				prefix =3D 32
+
+				# Count number of addresses in this subnet
+				num_addresses =3D int(end_address) - int(start_address)
+				if num_addresses:
+					prefix -=3D math.log(num_addresses, 2)
+
+				inetnum["inetnum"] =3D "%s/%.0f" % (start_address, prefix)
+
+			elif key =3D=3D "inet6num":
+				# Skip invalid blocks
+				if val in ["0.0.0.0", "::/0", "0::/0",]:
+					return
+
+				inetnum[key] =3D val
+
+			elif key =3D=3D "netname":
+				inetnum[key] =3D val
+
+			elif key =3D=3D "country":
+				if val =3D=3D "UNITED STATES":
+					val =3D "US"
+
+				inetnum[key] =3D val.upper()
+
+			elif key =3D=3D "descr":
+				if key in inetnum:
+					inetnum[key] +=3D "\n%s" % val
+				else:
+					inetnum[key] =3D val
+
+		# Skip empty objects
+		if not inetnum:
+			return
+
+		network =3D ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("i=
netnum"), strict=3DFalse)
+
+		self.db.execute("INSERT INTO networks(network, country) \
+			VALUES(%s, %s) ON CONFLICT (network) DO \
+			UPDATE SET country =3D excluded.country",
+			"%s" % str(network), inetnum.get("country"),
+		)
+
 	def _parse_org_block(self, block):
 		org =3D {}
 		for line in block:

--===============1177484414183951707==--