[PATCH] location-importer: Replace ARIN AS names source with one that offers human-readable names

["Peter Müller" <peter.mueller@ipfire.org>]

[-- Attachment #1: Type: text/plain, Size: 4366 bytes --]

This patch replaces our previous source for AS names in ARIN's realms
with another file provided by ARIN that contains human-readable names
for organizations ASNs have been allocated to.

Please note that a

	TRUNCATE autnums;

is necessary on machines previously running the old version of
location-importer, in order to make use of this changed data source.

Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org>
---
 src/scripts/location-importer.in | 47 ++++++++++++++++++--------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in
index 28a4f6c..96b3a20 100644
--- a/src/scripts/location-importer.in
+++ b/src/scripts/location-importer.in
@@ -3,7 +3,7 @@
 #                                                                             #
 # libloc - A library to determine the location of someone on the Internet     #
 #                                                                             #
-# Copyright (C) 2020-2022 IPFire Development Team <info(a)ipfire.org>           #
+# Copyright (C) 2020-2023 IPFire Development Team <info(a)ipfire.org>           #
 #                                                                             #
 # This library is free software; you can redistribute it and/or               #
 # modify it under the terms of the GNU Lesser General Public                  #
@@ -19,6 +19,7 @@
 
 import argparse
 import concurrent.futures
+import csv
 import http.client
 import ipaddress
 import json
@@ -1033,36 +1034,42 @@ class CLI(object):
 	def _import_as_names_from_arin(self):
 		downloader = location.importer.Downloader()
 
-		# XXX: Download AS names file from ARIN (note that these names appear to be quite
-		# technical, not intended for human consumption, as description fields in
-		# organisation handles for other RIRs are - however, this is what we have got,
-		# and in some cases, it might be still better than nothing)
-		for line in downloader.request_lines("https://ftp.arin.net/info/asn.txt"):
-			# Valid lines start with a space, followed by the number of the Autonomous System ...
-			if not line.startswith(" "):
+		# Download AS names file from ARIN and load it into CSV parser
+		for line in downloader.request_lines("https://ftp.arin.net/pub/resource_registry_service/asns.csv"):
+
+			# Valid lines start with a " ...
+			if not line.startswith("\""):
 				continue
 
 			# Split line and check if there is a valid ASN in it...
-			asn, name = line.split()[0:2]
+			for row in csv.reader([line]):
+				orgname = row[0]
+				orghandle = row[1]
+				firstasn = row[3]
+				lastasn = row[4]
 
 			try:
-				asn = int(asn)
+				firstasn = int(firstasn.strip("\""))
+				lastasn = int(lastasn.strip("\""))
 			except ValueError:
-				log.debug("Skipping ARIN AS names line not containing an integer for ASN")
+				log.debug("Skipping ARIN AS names line not containing valid integers for ASN")
 				continue
 
 			# Filter invalid ASNs...
-			if not self._check_parsed_asn(asn):
+			if not self._check_parsed_asn(firstasn):
 				continue
 
-			# Skip any AS name that appears to be a placeholder for a different RIR or entity...
-			if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|LACNIC|RIPE|IANA)(?:\d?$|\-)", name):
+			if firstasn > lastasn:
+				continue
+
+			# Filter any bulk AS assignments, since these are present for other RIRs where
+			# we get better data from elsewhere.
+			if not firstasn == lastasn:
 				continue
 
-			# Bail out in case the AS name contains anything we do not expect here...
-			if re.search(r"[^a-zA-Z0-9-_]", name):
-				log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \
-						(asn, name))
+			# Skip any AS name that appears to be a placeholder for a different RIR or entity...
+			if re.match(r"^(AFRINIC|APNIC|LACNIC|RIPE)$", orghandle.strip("\"")):
+				continue
 
 			# Things look good here, run INSERT statement and skip this one if we already have
 			# a (better?) name for this Autonomous System...
@@ -1073,8 +1080,8 @@ class CLI(object):
 					source
 				) VALUES (%s, %s, %s)
 				ON CONFLICT (number) DO NOTHING""",
-				asn,
-				name,
+				firstasn,
+				orgname.strip("\""),
 				"ARIN",
 			)
 
-- 
2.35.3