Location January 2024

location@lists.ipfire.org

1 discussions

[PATCH] location-importer: Replace ARIN AS names source with one that offers human-readable names
by Peter Müller 21 Feb '24

21 Feb '24

This patch replaces our previous source for AS names in ARIN's realms with another file provided by ARIN that contains human-readable names for organizations ASNs have been allocated to. Please note that a TRUNCATE autnums; is necessary on machines previously running the old version of location-importer, in order to make use of this changed data source. Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org> --- src/scripts/location-importer.in | 47 ++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in index 28a4f6c..96b3a20 100644 --- a/src/scripts/location-importer.in +++ b/src/scripts/location-importer.in @@ -3,7 +3,7 @@ # # # libloc - A library to determine the location of someone on the Internet # # # -# Copyright (C) 2020-2022 IPFire Development Team <info(a)ipfire.org> # +# Copyright (C) 2020-2023 IPFire Development Team <info(a)ipfire.org> # # # # This library is free software; you can redistribute it and/or # # modify it under the terms of the GNU Lesser General Public # @@ -19,6 +19,7 @@ import argparse import concurrent.futures +import csv import http.client import ipaddress import json @@ -1033,36 +1034,42 @@ class CLI(object): def _import_as_names_from_arin(self): downloader = location.importer.Downloader() - # XXX: Download AS names file from ARIN (note that these names appear to be quite - # technical, not intended for human consumption, as description fields in - # organisation handles for other RIRs are - however, this is what we have got, - # and in some cases, it might be still better than nothing) - for line in downloader.request_lines("https://ftp.arin.net/info/asn.txt"): - # Valid lines start with a space, followed by the number of the Autonomous System ... - if not line.startswith(" "): + # Download AS names file from ARIN and load it into CSV parser + for line in downloader.request_lines("https://ftp.arin.net/pub/resource_registry_service/asns.csv"): + + # Valid lines start with a " ... + if not line.startswith("\""): continue # Split line and check if there is a valid ASN in it... - asn, name = line.split()[0:2] + for row in csv.reader([line]): + orgname = row[0] + orghandle = row[1] + firstasn = row[3] + lastasn = row[4] try: - asn = int(asn) + firstasn = int(firstasn.strip("\"")) + lastasn = int(lastasn.strip("\"")) except ValueError: - log.debug("Skipping ARIN AS names line not containing an integer for ASN") + log.debug("Skipping ARIN AS names line not containing valid integers for ASN") continue # Filter invalid ASNs... - if not self._check_parsed_asn(asn): + if not self._check_parsed_asn(firstasn): continue - # Skip any AS name that appears to be a placeholder for a different RIR or entity... - if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|LACNIC|RIPE|IANA)(?:\d?$|\-)", name): + if firstasn > lastasn: + continue + + # Filter any bulk AS assignments, since these are present for other RIRs where + # we get better data from elsewhere. + if not firstasn == lastasn: continue - # Bail out in case the AS name contains anything we do not expect here... - if re.search(r"[^a-zA-Z0-9-_]", name): - log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \ - (asn, name)) + # Skip any AS name that appears to be a placeholder for a different RIR or entity... + if re.match(r"^(AFRINIC|APNIC|LACNIC|RIPE)$", orghandle.strip("\"")): + continue # Things look good here, run INSERT statement and skip this one if we already have # a (better?) name for this Autonomous System... @@ -1073,8 +1080,8 @@ class CLI(object): source ) VALUES (%s, %s, %s) ON CONFLICT (number) DO NOTHING""", - asn, - name, + firstasn, + orgname.strip("\""), "ARIN", ) -- 2.35.3

2 2

2024

2023

2022

2021

2020

2019

2018

2017

Location January 2024