Re: [PATCH] location-importer.in: import additional IP information for Amazon AWS IP networks

[Michael Tremer <michael.tremer@ipfire.org>]

[-- Attachment #1: Type: text/plain, Size: 5663 bytes --]

Hello Peter,

Thanks for this, I guess this would affect quite a few people out there…

However, is it a good idea to use the overrides table for this? Should that not be reserved for the pure overrides?

There is no way to view these changes. Is that something we can live with?

-Michael

> On 10 Apr 2021, at 13:28, Peter Müller <peter.mueller(a)ipfire.org> wrote:
> 
> Amazon publishes information regarding some of their IP networks
> primarily used for AWS cloud services in a machine-readable format. To
> improve libloc lookup results for these, we have little choice other
> than importing and parsing them.
> 
> Unfortunately, there seems to be no machine-readable list of the
> locations of their data centers or availability zones available. If
> there _is_ any, please let the author know.
> 
> Fixes: #12594
> 
> Signed-off-by: Peter Müller <peter.mueller(a)ipfire.org>
> ---
> src/python/location-importer.in | 110 ++++++++++++++++++++++++++++++++
> 1 file changed, 110 insertions(+)
> 
> diff --git a/src/python/location-importer.in b/src/python/location-importer.in
> index 1e08458..5be1d61 100644
> --- a/src/python/location-importer.in
> +++ b/src/python/location-importer.in
> @@ -19,6 +19,7 @@
> 
> import argparse
> import ipaddress
> +import json
> import logging
> import math
> import re
> @@ -931,6 +932,10 @@ class CLI(object):
> 				TRUNCATE TABLE network_overrides;
> 			""")
> 
> +			# Update overrides for various cloud providers big enough to publish their own IP
> +			# network allocation lists in a machine-readable format...
> +			self._update_overrides_for_aws()
> +
> 			for file in ns.files:
> 				log.info("Reading %s..." % file)
> 
> @@ -998,6 +1003,111 @@ class CLI(object):
> 						else:
> 							log.warning("Unsupported type: %s" % type)
> 
> +	def _update_overrides_for_aws(self):
> +		# Download Amazon AWS IP allocation file to create overrides...
> +		downloader = location.importer.Downloader()
> +
> +		try:
> +			with downloader.request("https://ip-ranges.amazonaws.com/ip-ranges.json", return_blocks=False) as f:
> +				aws_ip_dump = json.load(f.body)
> +		except Exception as e:
> +			log.error("unable to preprocess Amazon AWS IP ranges: %s" % e)
> +			return
> +
> +		# XXX: Set up a dictionary for mapping a region name to a country. Unfortunately,
> +		# there seems to be no machine-readable version available of this other than
> +		# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
> +		# (worse, it seems to be incomplete :-/ ); https://www.cloudping.cloud/endpoints
> +		# was helpful here as well.
> +		aws_region_country_map = {
> +				"af-south-1": "ZA",
> +				"ap-east-1": "HK",
> +				"ap-south-1": "IN",
> +				"ap-south-2": "IN",
> +				"ap-northeast-3": "JP",
> +				"ap-northeast-2": "KR",
> +				"ap-southeast-1": "SG",
> +				"ap-southeast-2": "AU",
> +				"ap-southeast-3": "MY",
> +				"ap-northeast-1": "JP",
> +				"ca-central-1": "CA",
> +				"eu-central-1": "DE",
> +				"eu-central-2": "CH",
> +				"eu-west-1": "IE",
> +				"eu-west-2": "GB",
> +				"eu-south-1": "IT",
> +				"eu-south-2": "ES",
> +				"eu-west-3": "FR",
> +				"eu-north-1": "SE",
> +				"me-south-1": "BH",
> +				"sa-east-1": "BR"
> +				}
> +
> +		# Fetch all valid country codes to check parsed networks aganist...
> +		rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
> +		validcountries = []
> +
> +		for row in rows:
> +			validcountries.append(row.country_code)
> +
> +		with self.db.transaction():
> +			for snetwork in aws_ip_dump["prefixes"] + aws_ip_dump["ipv6_prefixes"]:
> +				try:
> +					network = ipaddress.ip_network(snetwork.get("ip_prefix") or snetwork.get("ipv6_prefix"), strict=False)
> +				except ValueError:
> +					log.warning("Unable to parse line: %s" % snetwork)
> +					continue
> +
> +				# Sanitize parsed networks...
> +				if not self._check_parsed_network(network):
> +					continue
> +
> +				# Determine region of this network...
> +				region = snetwork["region"]
> +				cc = None
> +				is_anycast = False
> +
> +				# Any region name starting with "us-" will get "US" country code assigned straight away...
> +				if region.startswith("us-"):
> +					cc = "US"
> +				elif region.startswith("cn-"):
> +					# ... same goes for China ...
> +					cc = "CN"
> +				elif region == "GLOBAL":
> +					# ... funny region name for anycast-like networks ...
> +					is_anycast = True
> +				elif region in aws_region_country_map:
> +					# ... assign looked up country code otherwise ...
> +					cc = aws_region_country_map[region]
> +				else:
> +					# ... and bail out if we are missing something here
> +					log.warning("Unable to determine country code for line: %s" % snetwork)
> +					continue
> +
> +				# Skip networks with unknown country codes
> +				if not is_anycast and validcountries and cc not in validcountries:
> +					log.warning("Skipping Amazon AWS network with bogus country '%s': %s" % \
> +						(cc, network))
> +					return
> +
> +				# Conduct SQL statement...
> +				self.db.execute("""
> +					INSERT INTO network_overrides(
> +						network,
> +						country,
> +						is_anonymous_proxy,
> +						is_satellite_provider,
> +						is_anycast
> +					) VALUES (%s, %s, %s, %s, %s)
> +					ON CONFLICT (network) DO NOTHING""",
> +					"%s" % network,
> +					cc,
> +					None,
> +					None,
> +					is_anycast,
> +				)
> +
> +
> 	@staticmethod
> 	def _parse_bool(block, key):
> 		val = block.get(key)
> -- 
> 2.26.2