From mboxrd@z Thu Jan 1 00:00:00 1970 From: Peter =?utf-8?q?M=C3=BCller?= To: location@lists.ipfire.org Subject: [PATCH] location-importer: Replace ARIN AS names source with one that offers human-readable names Date: Sun, 10 Dec 2023 19:37:00 +0000 Message-ID: MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============1208896441212815588==" List-Id: --===============1208896441212815588== Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable This patch replaces our previous source for AS names in ARIN's realms with another file provided by ARIN that contains human-readable names for organizations ASNs have been allocated to. Please note that a TRUNCATE autnums; is necessary on machines previously running the old version of location-importer, in order to make use of this changed data source. Signed-off-by: Peter M=C3=BCller --- src/scripts/location-importer.in | 47 ++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer= .in index 28a4f6c..96b3a20 100644 --- a/src/scripts/location-importer.in +++ b/src/scripts/location-importer.in @@ -3,7 +3,7 @@ # = # # libloc - A library to determine the location of someone on the Internet = # # = # -# Copyright (C) 2020-2022 IPFire Development Team = # +# Copyright (C) 2020-2023 IPFire Development Team = # # = # # This library is free software; you can redistribute it and/or = # # modify it under the terms of the GNU Lesser General Public = # @@ -19,6 +19,7 @@ =20 import argparse import concurrent.futures +import csv import http.client import ipaddress import json @@ -1033,36 +1034,42 @@ class CLI(object): def _import_as_names_from_arin(self): downloader =3D location.importer.Downloader() =20 - # XXX: Download AS names file from ARIN (note that these names appear to b= e quite - # technical, not intended for human consumption, as description fields in - # organisation handles for other RIRs are - however, this is what we have = got, - # and in some cases, it might be still better than nothing) - for line in downloader.request_lines("https://ftp.arin.net/info/asn.txt"): - # Valid lines start with a space, followed by the number of the Autonomou= s System ... - if not line.startswith(" "): + # Download AS names file from ARIN and load it into CSV parser + for line in downloader.request_lines("https://ftp.arin.net/pub/resource_re= gistry_service/asns.csv"): + + # Valid lines start with a " ... + if not line.startswith("\""): continue =20 # Split line and check if there is a valid ASN in it... - asn, name =3D line.split()[0:2] + for row in csv.reader([line]): + orgname =3D row[0] + orghandle =3D row[1] + firstasn =3D row[3] + lastasn =3D row[4] =20 try: - asn =3D int(asn) + firstasn =3D int(firstasn.strip("\"")) + lastasn =3D int(lastasn.strip("\"")) except ValueError: - log.debug("Skipping ARIN AS names line not containing an integer for ASN= ") + log.debug("Skipping ARIN AS names line not containing valid integers for= ASN") continue =20 # Filter invalid ASNs... - if not self._check_parsed_asn(asn): + if not self._check_parsed_asn(firstasn): continue =20 - # Skip any AS name that appears to be a placeholder for a different RIR o= r entity... - if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|LACNIC|RIPE|IANA)(?= :\d?$|\-)", name): + if firstasn > lastasn: + continue + + # Filter any bulk AS assignments, since these are present for other RIRs = where + # we get better data from elsewhere. + if not firstasn =3D=3D lastasn: continue =20 - # Bail out in case the AS name contains anything we do not expect here... - if re.search(r"[^a-zA-Z0-9-_]", name): - log.debug("Skipping ARIN AS name for %s containing invalid characters: %= s" % \ - (asn, name)) + # Skip any AS name that appears to be a placeholder for a different RIR o= r entity... + if re.match(r"^(AFRINIC|APNIC|LACNIC|RIPE)$", orghandle.strip("\"")): + continue =20 # Things look good here, run INSERT statement and skip this one if we alr= eady have # a (better?) name for this Autonomous System... @@ -1073,8 +1080,8 @@ class CLI(object): source ) VALUES (%s, %s, %s) ON CONFLICT (number) DO NOTHING""", - asn, - name, + firstasn, + orgname.strip("\""), "ARIN", ) =20 --=20 2.35.3 --===============1208896441212815588==--