Location June 2021

location@lists.ipfire.org

4 participants
15 discussions

[PATCH] location-importer.in: Import (technical) AS names from ARIN
by Peter Müller 08 Jun '21

08 Jun '21

ARIN and LACNIC, unfortunately, do not seem to publish data containing human readable AS names. For the former, we at least have a list of tecnical names, which this patch fetches and inserts into the autnums table. While some of them do not seem to be suitable for human consumption (i. e. being very cryptic), providing these data might be helpful neverthelesss. Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org> --- src/python/location-importer.in | 61 +++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/python/location-importer.in b/src/python/location-importer.in index aa3b8f7..2a9bf33 100644 --- a/src/python/location-importer.in +++ b/src/python/location-importer.in @@ -505,6 +505,9 @@ class CLI(object): for line in f: self._parse_line(line, source_key, validcountries) + # Download and import (technical) AS names from ARIN + self._import_as_names_from_arin() + def _check_parsed_network(self, network): """ Assistive function to detect and subsequently sort out parsed @@ -775,6 +778,64 @@ class CLI(object): "%s" % network, country, [country], source_key, ) + def _import_as_names_from_arin(self): + downloader = location.importer.Downloader() + + # XXX: Download AS names file from ARIN (note that these names appear to be quite + # technical, not intended for human consumption, as description fields in + # organisation handles for other RIRs are - however, this is what we have got, + # and in some cases, it might be still better than nothing) + try: + with downloader.request("https://ftp.arin.net/info/asn.txt", return_blocks=False) as f: + arin_as_names_file = f.body + except Exception as e: + log.error("failed to download and preprocess AS name file from ARIN: %s" % e) + return + + # Split downloaded body into lines and parse each of them... + for sline in arin_as_names_file.readlines(): + + # ... valid lines start with a space, followed by the number of the Autonomous System ... + if not sline.startswith(b" "): + continue + + # Split line and check if there is a valid ASN in it... + scontents = sline.split() + try: + asn = int(scontents[0]) + except ValueError: + log.debug("Skipping ARIN AS names line not containing an integer for ASN") + continue + + if not ((1 <= asn and asn <= 23455) or (23457 <= asn and asn <= 64495) or (131072 <= asn and asn <= 4199999999)): + log.debug("Skipping ARIN AS names line not containing a valid ASN: %s" % asn) + continue + + # Skip any AS name that appears to be a placeholder for a different RIR or entity... + as_name = scontents[1].decode("ascii") + + if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|DNIC|LACNIC|RIPE|IANA)\d{0,1}-*", as_name): + continue + + # Bail out in case the AS name contains anything we do not expect here... + if re.search(r"[^a-zA-Z0-9-_]", as_name): + log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \ + (asn, as_name)) + + # Things look good here, run INSERT statement and skip this one if we already have + # a (better?) name for this Autonomous System... + self.db.execute(""" + INSERT INTO autnums( + number, + name, + source + ) VALUES (%s, %s, %s) + ON CONFLICT (number) DO NOTHING""", + asn, + as_name, + "ARIN", + ) + def handle_update_announcements(self, ns): server = ns.server[0] -- 2.20.1

2 3

[PATCH v2 1/2] location-importer.in: add source column for overrides as well
by Peter Müller 08 Jun '21

08 Jun '21

This allows us to track changes introduced by IP feeds from 3rd parties, such as Amazon AWS, on the SQL server side. In order not to break existing tables (which would required TRUNCATE), there currently is no constraint set for the new column, but "NOT NULL" is planned in the future. Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org> --- src/python/location-importer.in | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/python/location-importer.in b/src/python/location-importer.in index aa3b8f7..78bfd55 100644 --- a/src/python/location-importer.in +++ b/src/python/location-importer.in @@ -183,6 +183,7 @@ class CLI(object): ); CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number ON autnum_overrides(number); + ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS source text; ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; CREATE TABLE IF NOT EXISTS network_overrides( @@ -196,6 +197,7 @@ class CLI(object): ON network_overrides(network); CREATE INDEX IF NOT EXISTS network_overrides_search ON network_overrides USING GIST(network inet_ops); + ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS source text; ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; """) @@ -997,14 +999,16 @@ class CLI(object): INSERT INTO network_overrides( network, country, + source, is_anonymous_proxy, is_satellite_provider, is_anycast, is_drop - ) VALUES (%s, %s, %s, %s, %s, %s) + ) VALUES (%s, %s, %s, %s, %s, %s, %s) ON CONFLICT (network) DO NOTHING""", "%s" % network, block.get("country"), + "manual", self._parse_bool(block, "is-anonymous-proxy"), self._parse_bool(block, "is-satellite-provider"), self._parse_bool(block, "is-anycast"), @@ -1027,15 +1031,17 @@ class CLI(object): number, name, country, + source, is_anonymous_proxy, is_satellite_provider, is_anycast, is_drop - ) VALUES(%s, %s, %s, %s, %s, %s, %s) + ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING""", autnum, block.get("name"), block.get("country"), + "manual", self._parse_bool(block, "is-anonymous-proxy"), self._parse_bool(block, "is-satellite-provider"), self._parse_bool(block, "is-anycast"), -- 2.20.1

1 1

[PATCH] Implement an additional flag for hostile networks safe to drop
by Peter Müller 07 Jun '21

07 Jun '21

This patch implements an additional flag intended for networks and Autonomous Systems being considered hostile. While libloc does not and should not be an opinionated database, reality shows it is being used this way. Hereby, we assign "XD" (drop) as a custom country code for networks being flagged this way. According to ISO, "XA" to "XZ" are reserved for "user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html), so this is a safe thing to do. This patch does not interfere with "A1" to "A3", which we currently assign outside standardised country code ranges for historical reasons. Neither does it specify any policy or source for tagging networks with a "drop" flag. Doing so is beyond the scope of this - technical - approach. To avoid confusions with the SQL "DROP" command, "is_drop" will be used as a column name for database operations. Thanks to Michael for his remarks and ideas during the run-up. Cc: Michael Tremer <michael.tremer(a)ipfire.org> Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org> --- man/location.txt | 4 ++-- po/de.po | 3 +++ src/loc/network.h | 3 ++- src/perl/Location.xs | 2 ++ src/python/export.py | 3 ++- src/python/location-importer.in | 32 +++++++++++++++++++++++++++----- src/python/location.in | 19 ++++++++++++++++--- src/python/locationmodule.c | 5 ++++- 8 files changed, 58 insertions(+), 13 deletions(-) diff --git a/man/location.txt b/man/location.txt index df1da53..b38f21c 100644 --- a/man/location.txt +++ b/man/location.txt @@ -10,7 +10,7 @@ location - Query the location database `location list-countries [--show-name] [--show-continent]` `location list-networks-by-as ASN` `location list-networks-by-cc COUNTRY_CODE` -`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` +`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]` `location lookup ADDRESS [ADDRESS...]` `location search-as STRING` `location update [--cron=daily|weekly|monthly]` @@ -73,7 +73,7 @@ or countries. + See above for usage of the '--family' and '--format' parameters. -'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast]':: +'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]':: Lists all networks that have a certain flag. + See above for usage of the '--family' and '--format' parameters. diff --git a/po/de.po b/po/de.po index 3b073d6..3cbcdd7 100644 --- a/po/de.po +++ b/po/de.po @@ -152,6 +152,9 @@ msgstr "" msgid "Anycasts" msgstr "" +msgid "Hostile Networks safe to drop" +msgstr "" + msgid "Lists all countries" msgstr "" diff --git a/src/loc/network.h b/src/loc/network.h index af3dafd..a30f653 100644 --- a/src/loc/network.h +++ b/src/loc/network.h @@ -1,7 +1,7 @@ /* libloc - A library to determine the location of someone on the Internet - Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org> + Copyright (C) 2017-2021 IPFire Development Team <info(a)ipfire.org> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -27,6 +27,7 @@ enum loc_network_flags { LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1 LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2 LOC_NETWORK_FLAG_ANYCAST = (1 << 2), // A3 + LOC_NETWORK_FLAG_DROP = (1 << 3), // XD }; struct loc_network; diff --git a/src/perl/Location.xs b/src/perl/Location.xs index b7676d2..73f85b4 100644 --- a/src/perl/Location.xs +++ b/src/perl/Location.xs @@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag) iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER; else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0) iv |= LOC_NETWORK_FLAG_ANYCAST; + else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0) + iv |= LOC_NETWORK_FLAG_DROP; else croak("Invalid flag"); diff --git a/src/python/export.py b/src/python/export.py index f0eae26..3b9e1e0 100644 --- a/src/python/export.py +++ b/src/python/export.py @@ -3,7 +3,7 @@ # # # libloc - A library to determine the location of someone on the Internet # # # -# Copyright (C) 2020 IPFire Development Team <info(a)ipfire.org> # +# Copyright (C) 2020-2021 IPFire Development Team <info(a)ipfire.org> # # # # This library is free software; you can redistribute it and/or # # modify it under the terms of the GNU Lesser General Public # @@ -33,6 +33,7 @@ FLAGS = { _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", _location.NETWORK_FLAG_ANYCAST : "A3", + _location.NETWORK_FLAG_DROP : "XD", } class OutputWriter(object): diff --git a/src/python/location-importer.in b/src/python/location-importer.in index f796652..ea1e8f5 100644 --- a/src/python/location-importer.in +++ b/src/python/location-importer.in @@ -182,6 +182,7 @@ class CLI(object): ); CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number ON autnum_overrides(number); + ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; CREATE TABLE IF NOT EXISTS network_overrides( network inet NOT NULL, @@ -194,6 +195,7 @@ class CLI(object): ON network_overrides(network); CREATE INDEX IF NOT EXISTS network_overrides_search ON network_overrides USING GIST(network inet_ops); + ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean; """) return db @@ -301,7 +303,20 @@ class CLI(object): WHERE networks.autnum = overrides.number ), FALSE - ) AS is_anycast + ) AS is_anycast, + COALESCE( + ( + SELECT is_drop FROM network_overrides overrides + WHERE networks.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_drop FROM autnum_overrides overrides + WHERE networks.autnum = overrides.number + ), + FALSE + ) AS is_drop FROM ( SELECT known_networks.network AS network, @@ -350,6 +365,9 @@ class CLI(object): if row.is_anycast: network.set_flag(location.NETWORK_FLAG_ANYCAST) + if row.is_drop: + network.set_flag(location.NETWORK_FLAG_DROP) + # Add all countries log.info("Writing countries...") rows = self.db.query("SELECT * FROM countries ORDER BY country_code") @@ -966,14 +984,16 @@ class CLI(object): country, is_anonymous_proxy, is_satellite_provider, - is_anycast - ) VALUES (%s, %s, %s, %s, %s) + is_anycast, + is_drop + ) VALUES (%s, %s, %s, %s, %s, %s) ON CONFLICT (network) DO NOTHING""", "%s" % network, block.get("country"), self._parse_bool(block, "is-anonymous-proxy"), self._parse_bool(block, "is-satellite-provider"), self._parse_bool(block, "is-anycast"), + self._parse_bool(block, "drop"), ) elif type == "aut-num": @@ -994,8 +1014,9 @@ class CLI(object): country, is_anonymous_proxy, is_satellite_provider, - is_anycast - ) VALUES(%s, %s, %s, %s, %s, %s) + is_anycast, + is_drop + ) VALUES(%s, %s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING""", autnum, block.get("name"), @@ -1003,6 +1024,7 @@ class CLI(object): self._parse_bool(block, "is-anonymous-proxy"), self._parse_bool(block, "is-satellite-provider"), self._parse_bool(block, "is-anycast"), + self._parse_bool(block, "drop"), ) else: diff --git a/src/python/location.in b/src/python/location.in index e02b4e8..0c89d75 100644 --- a/src/python/location.in +++ b/src/python/location.in @@ -3,7 +3,7 @@ # # # libloc - A library to determine the location of someone on the Internet # # # -# Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org> # +# Copyright (C) 2017-2021 IPFire Development Team <info(a)ipfire.org> # # # # This library is free software; you can redistribute it and/or # # modify it under the terms of the GNU Lesser General Public # @@ -146,6 +146,9 @@ class CLI(object): list_networks_by_flags.add_argument("--anycast", action="store_true", help=_("Anycasts"), ) + list_networks_by_flags.add_argument("--drop", + action="store_true", help=_("Hostile Networks safe to drop"), + ) list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4")) list_networks_by_flags.add_argument("--format", choices=location.export.formats.keys(), default="list") @@ -305,6 +308,12 @@ class CLI(object): _("Anycast"), _("yes"), )) + # Hostile Network + if network.has_flag(location.NETWORK_FLAG_DROP): + print(format % ( + _("Hostile Network safe to drop"), _("yes"), + )) + return ret def handle_dump(self, db, ns): @@ -346,6 +355,7 @@ class CLI(object): location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:", location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:", location.NETWORK_FLAG_ANYCAST : "is-anycast:", + location.NETWORK_FLAG_DROP : "drop:", } # Iterate over all networks @@ -523,6 +533,9 @@ class CLI(object): if ns.anycast: flags |= location.NETWORK_FLAG_ANYCAST + if ns.drop: + flags |= location.NETWORK_FLAG_DROP + if not flags: raise ValueError(_("You must at least pass one flag")) @@ -551,7 +564,7 @@ class CLI(object): asns.append(object) elif location.country_code_is_valid(object) \ - or object in ("A1", "A2", "A3"): + or object in ("A1", "A2", "A3", "XD"): countries.append(object) else: @@ -560,7 +573,7 @@ class CLI(object): # Default to exporting all countries if not countries and not asns: - countries = ["A1", "A2", "A3"] + [country.code for country in db.countries] + countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries] # Select the output format writer = self.__get_output_formatter(ns) diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c index 5b72be9..5dd4ec6 100644 --- a/src/python/locationmodule.c +++ b/src/python/locationmodule.c @@ -1,7 +1,7 @@ /* libloc - A library to determine the location of someone on the Internet - Copyright (C) 2017 IPFire Development Team <info(a)ipfire.org> + Copyright (C) 2017-2021 IPFire Development Team <info(a)ipfire.org> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) { if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST)) return NULL; + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP)) + return NULL; + // Add latest database version if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST)) return NULL; -- 2.26.2

2 1

[PATCH] overrides-{a{1,3},other}: regular batch of various overrides
by Peter Müller 03 Jun '21

03 Jun '21

Including location pinning for various LeaseWeb AS, as their customers seem to tamper with RIR data a lot. Fortunately for use, they use one AS per PoP, so we can trace back locations quite easy. :-) AS209242 is especially - um - interesting. Given Cloudflare's nature, it is impossible to tell where these shady prefixes announced by it are located. Most of them point to letterbox companies, hosting questionable services at best. Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org> --- overrides/override-a1.txt | 32 +++++++++- overrides/override-a3.txt | 30 ++++++++++ overrides/override-other.txt | 112 ++++++++++++++++++++++++++++++++++- 3 files changed, 172 insertions(+), 2 deletions(-) diff --git a/overrides/override-a1.txt b/overrides/override-a1.txt index 3a65232..b884c5d 100644 --- a/overrides/override-a1.txt +++ b/overrides/override-a1.txt @@ -44,6 +44,11 @@ descr: VPN de Mexico, S.A. de C.V. remarks: VPN provider is-anonymous-proxy: yes +aut-num: AS32781 +descr: Defender cloud international LLC +remarks: VPN provider [high confidence, but not proofed] +is-anonymous-proxy: yes + aut-num: AS34962 descr: Epik Network remarks: Shady ISP and registrar, many prefixes announced refer to "anonymize" infrastructure @@ -236,6 +241,11 @@ descr: Business VPN LLC remarks: VPN provider is-anonymous-proxy: yes +aut-num: AS398271 +descr: HardenedVPN[.]com LLC +remarks: VPN provider +is-anonymous-proxy: yes + net: 2.57.171.0/24 descr: VPN Consumer Network remarks: VPN provider @@ -476,6 +486,11 @@ descr: Shtrauh Andrey remarks: VPN provider [high confidence, but not proofed] is-anonymous-proxy: yes +net: 45.147.56.0/23 +descr: OLMERA GROUP LTD +remarks: VPN provider [high confidence, but not proofed] +is-anonymous-proxy: yes + net: 45.151.115.0/24 descr: ikoProxies [high confidence, but not proofed] remarks: VPN provider located in NL @@ -489,7 +504,7 @@ is-anonymous-proxy: yes net: 45.192.0.0/12 descr: Cloud Innovation Ltd. -remarks: hijacked AFRINIC IP chunk owned by an offshore company, routed to several dirty networks worldwide, cannot tell what is going on here +remarks: hijacked (?) AFRINIC IP chunk owned by an offshore company, routed to several dirty networks worldwide, cannot tell what is going on here is-anonymous-proxy: yes net: 45.220.72.0/22 @@ -1131,6 +1146,11 @@ remarks: (Rogue) VPN provider is-anonymous-proxy: yes country: EU +net: 185.164.59.0/24 +descr: Buyproxies / Yuli Azarch +remarks: VPN provider [high confidence, but not proofed] +is-anonymous-proxy: yes + net: 185.165.153.0/24 descr: Privacy Matters VPN service / nVPN / David Craig remarks: (Rogue) VPN provider @@ -1384,6 +1404,11 @@ descr: VPNClientPublics remarks: VPN provider is-anonymous-proxy: yes +net: 198.44.224.0/19 +descr: Defender cloud international LLC +remarks: VPN provider [high confidence, but not proofed] +is-anonymous-proxy: yes + net: 199.249.223.0/24 descr: Quintex Alliance Consulting remarks: Tor relay provider @@ -1454,6 +1479,11 @@ descr: SecuredConnectivity remarks: VPN provider is-anonymous-proxy: yes +net: 206.123.128.0/19 +descr: Secure Internet LLC +remarks: VPN provider +is-anonymous-proxy: yes + net: 209.107.176.0/20 descr: Google VPN remarks: VPN provider diff --git a/overrides/override-a3.txt b/overrides/override-a3.txt index ec246c1..daacc95 100644 --- a/overrides/override-a3.txt +++ b/overrides/override-a3.txt @@ -30,11 +30,21 @@ descr: Akamai Technologies, Inc. remarks: Worldwide CDN, does not make sense to assign their networks to a specific country is-anycast: yes +aut-num: AS20577 +descr: TRADEWEB EUROPE LIMITED +remarks: Generic anycast network [high confidence, but not proofed] +is-anycast: yes + aut-num: AS20940 descr: Akamai International BV remarks: Worldwide CDN, does not make sense to assign their networks to a specific country is-anycast: yes +aut-num: AS28064 +descr: Smart Data Center S.A. +remarks: Generic anycast network [high confidence, but not proofed] +is-anycast: yes + aut-num: AS31529 descr: DENIC eG remarks: TLD operator's anycast network @@ -45,6 +55,11 @@ descr: Citigroup remarks: Public anycast DNS resolver network [high confidence, but not proofed] is-anycast: yes +aut-num: AS34738 +descr: WEBHOST LIMITED +remarks: Generic anycast network [high confidence, but not proofed] +is-anycast: yes + aut-num: AS34868 descr: wasted.io Ltd. remarks: Generic anycast network @@ -80,6 +95,11 @@ descr: coreIT.pl remarks: Generic anycast network is-anycast: yes +aut-num: AS60626 +descr: Leaseweb CDN B.V. +remarks: Generic anycast network +is-anycast: yes + aut-num: AS60890 descr: CentralNic Ltd remarks: Generic anycast network @@ -95,6 +115,11 @@ descr: OpenTLD BV remarks: TLD operator's anycast network is-anycast: yes +aut-num: AS61072 +descr: EZNet LIMITED +remarks: Generic anycast network +is-anycast: yes + aut-num: AS62766 descr: SJB Communications remarks: Generic anycast network @@ -160,6 +185,11 @@ descr: ipcom GmbH remarks: Generic anycast network is-anycast: yes +aut-num: AS209242 +descr: Cloudflare London, LLC / Cloudflare Spectrum +remarks: CFs' "bring your own IP networks and we will announce it" AS, lots of shady announcements originated from there, clients are located worldwide as single peer is CF's main AS +is-anycast: yes + aut-num: AS209813 descr: Fast Content Delivery Ltd. remarks: Generic anycast network diff --git a/overrides/override-other.txt b/overrides/override-other.txt index 0b521d1..e387dbd 100644 --- a/overrides/override-other.txt +++ b/overrides/override-other.txt @@ -13,6 +13,11 @@ # Please keep this file sorted. # +aut-num: AS1820 +descr: WNET TELECOM USA Corp. +remarks: traces back to various locations in UA, seems to tamper with RIR data +country: UA + aut-num: AS3216 descr: PJSC "Vimpelcom" remarks: ISP located in RU, but some RIR data for announced prefixes contain garbage @@ -23,6 +28,11 @@ descr: XNNET LLC remarks: traces back to an unknown oversea location (HK?), seems to tamper with RIR data country: HK +aut-num: AS7203 +descr: Leaseweb USA, Inc. +remarks: ISP located in US, but some RIR data for announced prefixes contain garbage +country: US + aut-num: AS8359 descr: MTS PJSC remarks: ISP located in RU, but some RIR data for announced prefixes contain garbage @@ -58,16 +68,36 @@ descr: Yes Networks Unlimited Ltd remarks: traces to UA, but some RIR entries seem to contain garbage (VG) country: UA +aut-num: AS24961 +descr: myLoc managed IT AG +remarks: ISP located in DE, but some RIR data for announced prefixes contain garbage +country: DE + aut-num: AS25098 descr: Netcalibre Ltd. -remarks: ISP located in GB, but some RIR data for announced prefixes contain garbage/ +remarks: ISP located in GB, but some RIR data for announced prefixes contain garbage country: GB +aut-num: AS27411 +descr: Leaseweb USA, Inc. +remarks: ISP located in US, but some RIR data for announced prefixes contain garbage +country: US + aut-num: AS28098 descr: ABGON Comunicaciones remarks: ISP located in CL, but some RIR data for announced prefixes contain garbage (BZ) country: CL +aut-num: AS28753 +descr: Leaseweb Deutschland GmbH +remarks: ISP located in Frankfurt/Main, DE, but many RIR data for announced prefixes contain garbage +country: DE + +aut-num: AS30633 +descr: Leaseweb USA, Inc. +remarks: ISP located in US, but some RIR data for announced prefixes contain garbage (BZ) +country: US + aut-num: AS35042 descr: IP Interactive UG (haftungsbeschraenkt) remarks: ISP located in BG, but RIR data for announced prefixes contain garbage @@ -163,6 +193,11 @@ descr: Global Colocation Limited remarks: part of a dirty ISP conglomerate most likely operating out of SE country: SE +aut-num: AS48721 +descr: ADM Service Ltd. +remarks: traces back to Vilnius, LT +country: LT + aut-num: AS49466 descr: KLAYER LLC remarks: part of the "Asline" IP hijacking gang, traces back to AP region @@ -233,6 +268,11 @@ descr: INNETRA PC remarks: another shady customer of "DDoS Guard Ltd.", jurisdiction is probably RU, but traceroutes dead-end somewhere else in EU country: EU +aut-num: AS59253 +descr: Leaseweb Asia Pacific pte. ltd. +remarks: ISP located in SG, but some RIR data for announced prefixes contain garbage +country: SG + aut-num: AS59580 descr: Batterflyai Media Ltd. remarks: ISP located in RU, but some RIR data for announced prefixes contain garbage @@ -248,6 +288,11 @@ descr: Inter Connects Inc. / Jing Yun remarks: part of a dirty ISP conglomerate operating most likely out of SE, hijacking AfriNIC networks country: SE +aut-num: AS60781 +descr: LeaseWeb Netherlands B.V. +remarks: ISP located in Amsterdam, NL, but many RIR data for announced prefixes contain garbage +country: NL + aut-num: AS61977 descr: Vivo Trade L.P. remarks: another shady customer of "DDoS Guard Ltd." @@ -283,11 +328,31 @@ descr: NForce Entertainment BV remarks: currently hijacks a single stolen /20 AfriNIC IPv4 net, hosted in NL country: NL +aut-num: AS132369 +descr: XIANGAO INTERNATIONAL TELECOMMUNICATION LIMITED +remarks: ISP located in HK, tampers with RIR data +country: HK + +aut-num: AS133752 +descr: Leaseweb Asia Pacific pte. ltd. +remarks: ISP located in HK, some RIR data for announced prefixes contain garbage +country: HK + +aut-num: AS134351 +descr: Leaseweb Japan K.K. +remarks: ISP located in JP, some RIR data for announced prefixes contain garbage +country: JP + aut-num: AS134548 descr: DXTL Tseung Kwan O Service remarks: tampers with RIR data, traces back to AP region country: AP +aut-num: AS136988 +descr: Leaseweb Australia Pty. Ltd. +remarks: ISP located in AU, some RIR data for announced prefixes contain garbage +country: AU + aut-num: AS137443 descr: Anchnet Asia Limited remarks: IP hijacker located in HK, tampers with RIR data @@ -298,11 +363,26 @@ descr: Clayer Limited remarks: part of the "Asline" IP hijacking gang, tampers with RIR data, traces back to AP region country: AP +aut-num: AS138571 +descr: SUPERCLOUDS LIMITED +remarks: ISP located in HK, tampers with RIR data +country: HK + +aut-num: AS139242 +descr: Cloudflare Sydney, LLC +remarks: ... but CF failed to set the country for announced prefixes to AU as well :-/ +country: AU + aut-num: AS139330 descr: SANREN DATA LIMITED remarks: IP hijacker located somewhere in AP region, tampers with RIR data country: AP +aut-num: AS139811 +descr: ANLIAN NETWORK TECHNOLOGY CO., LIMITED +remarks: ISP located in HK, tampers with RIR data +country: HK + aut-num: AS140733 descr: Wujidun Network Limited remarks: part of the "Asline" IP hijacking gang, tampers with RIR data, traces back to AP region @@ -323,6 +403,11 @@ descr: Datashield, Inc. remarks: fake offshore location (SC), traces back to NL country: NL +aut-num: AS200740 +descr: Network Management Ltd. +remarks: traceroutes dead-end somewhere in or near RU +country: RU + aut-num: AS200845 descr: AVATEL TELECOM remarks: ISP located in ES, but some RIR data for announced prefixes contain garbage @@ -388,6 +473,11 @@ descr: Hauer Hosting Services Limited remarks: ISP located in ES, but some RIR data for announced prefixes contain garbage country: ES +aut-num: AS205544 +descr: LEASEWEB UK LIMITED +remarks: ISP located in London, GB, but many RIR data for announced prefixes contain garbage +country: GB + aut-num: AS206397 descr: Genius Guard / Genius Security Ltd. remarks: another shady customer of "DDoS Guard Ltd.", probably located in RU @@ -403,6 +493,11 @@ descr: Xtudio Networks S.L.U. remarks: ISP located in ES, but some RIR data for announced prefixes contain garbage country: ES +aut-num: AS207569 +descr: Network Management Ltd. +remarks: traceroutes dead-end somewhere in or near RU +country: RU + aut-num: AS207616 descr: Altrosky Technology Ltd. remarks: fake offshore location (SC), traces back to CZ and NL @@ -458,6 +553,11 @@ descr: Harry Dowd remarks: ISP located in GB, but RIR data for announced prefixes contain garbage country: GB +aut-num: AS212962 +descr: Quality Area Ltd. +remarks: traceroutes dead-end somewhere near Crimera, UA +country: UA + aut-num: AS213035 descr: Serverion BV remarks: ISP located in NL, but RIR data for most announced prefixes contain garbage @@ -488,6 +588,16 @@ descr: Datapacket Maroc SARL remarks: bulletproof ISP (strongly linked to AS202425) located in NL country: NL +aut-num: AS394380 +descr: Leaseweb USA, Inc. +remarks: ISP located in Dallas, TX, US, but some RIR data for announced prefixes contain garbage +country: US + +aut-num: AS395954 +descr: Leaseweb USA, Inc. +remarks: ISP located in US, but some RIR data for announced prefixes contain garbage +country: US + aut-num: AS398478 descr: PEG TECH INC remarks: ISP located in HK, tampers with RIR data -- 2.26.2

1 0

[PATCH v2] location-importer.in: track original countries as well
by Peter Müller 02 Jun '21

02 Jun '21

This helps us to determine how many network objects have more than one country set, and what their original country code set looked like. The second version of this patch uses ALTER TABLE to add the column for original countries, preventing existing SQL setups from breaking. Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org> --- src/python/location-importer.in | 52 ++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/python/location-importer.in b/src/python/location-importer.in index e5f55af..c3f908a 100644 --- a/src/python/location-importer.in +++ b/src/python/location-importer.in @@ -165,6 +165,7 @@ class CLI(object): -- networks CREATE TABLE IF NOT EXISTS networks(network inet, country text); + ALTER TABLE networks ADD COLUMN IF NOT EXISTS original_countries text[]; CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); @@ -377,7 +378,7 @@ class CLI(object): ON COMMIT DROP; CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); - CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL, original_countries text[] NOT NULL) ON COMMIT DROP; CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); @@ -407,8 +408,8 @@ class CLI(object): for family in (row.family for row in families): smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) - self.db.execute("INSERT INTO networks(network, country) \ - SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) + self.db.execute("INSERT INTO networks(network, country, original_countries) \ + SELECT network, country, original_countries FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) # ... determine any other prefixes for this network family, ... prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ @@ -421,7 +422,8 @@ class CLI(object): WITH candidates AS ( SELECT _rirdata.network, - _rirdata.country + _rirdata.country, + _rirdata.original_countries FROM _rirdata WHERE @@ -434,6 +436,7 @@ class CLI(object): DISTINCT ON (c.network) c.network, c.country, + c.original_countries, masklen(networks.network), networks.country AS parent_country FROM @@ -447,10 +450,11 @@ class CLI(object): masklen(networks.network) DESC NULLS LAST ) INSERT INTO - networks(network, country) + networks(network, country, original_countries) SELECT network, - country + country, + original_countries FROM filtered WHERE @@ -617,28 +621,36 @@ class CLI(object): inetnum[key] = [ipaddress.ip_network(val, strict=False)] elif key == "country": - inetnum[key] = val.upper() + # Catch RIR data objects with more than one country code... + if not key in inetnum.keys(): + inetnum[key] = [] + else: + if val.upper() in inetnum.get("country"): + # ... but keep this list distinct... + continue + + inetnum[key].append(val.upper()) # Skip empty objects if not inetnum or not "country" in inetnum: return + # Prepare skipping objects with unknown country codes... + invalidcountries = [singlecountry for singlecountry in inetnum.get("country") if singlecountry not in validcountries] + # Iterate through all networks enumerated from above, check them for plausibility and insert # them into the database, if _check_parsed_network() succeeded for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"): if self._check_parsed_network(single_network): - - # Skip objects with unknown country codes - to avoid log spam for invalid or too small - # networks, this check is - kinda ugly - done at this point - if validcountries and inetnum.get("country") not in validcountries: - log.warning("Skipping network with bogus country '%s': %s" % \ - (inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum"))) + # Skip objects with unknown country codes if they are valid to avoid log spam... + if validcountries and invalidcountries: + log.warning("Skipping network with bogus countr(y|ies) %s (original countries: %s): %s" % \ + (invalidcountries, inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum"))) break - # Everything is fine here, run INSERT statement... - self.db.execute("INSERT INTO _rirdata(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", - "%s" % single_network, inetnum.get("country"), + self.db.execute("INSERT INTO _rirdata(network, country, original_countries) \ + VALUES(%s, %s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", + "%s" % single_network, inetnum.get("country")[0], inetnum.get("country"), ) def _parse_org_block(self, block): @@ -729,10 +741,10 @@ class CLI(object): if not self._check_parsed_network(network): return - self.db.execute("INSERT INTO networks(network, country) \ - VALUES(%s, %s) ON CONFLICT (network) DO \ + self.db.execute("INSERT INTO networks(network, country, original_countries) \ + VALUES(%s, %s, %s) ON CONFLICT (network) DO \ UPDATE SET country = excluded.country", - "%s" % network, country, + "%s" % network, country, [country], ) def handle_update_announcements(self, ns): -- 2.20.1

2 6

2024

2023

2022

2021

2020

2019

2018

2017

Location June 2021