From mboxrd@z Thu Jan  1 00:00:00 1970
From: Michael Tremer <michael.tremer@ipfire.org>
To: location@lists.ipfire.org
Subject: Re: [PATCH] location-importer.in: process unaligned IP ranges in RIR
 data files correctly
Date: Mon, 29 Mar 2021 21:27:24 +0100
Message-ID: <55002AB9-77BA-43FF-895F-36CACEDE5368@ipfire.org>
In-Reply-To: <8d42901f-e1be-285f-2060-639218e2b694@ipfire.org>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="===============7575780521212185400=="
List-Id: <location.lists.ipfire.org>

--===============7575780521212185400==
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable

Thank you for this.

Are there any other things coming or can I go ahead and tag another version t=
o roll these changes out into production?

-Michael

> On 29 Mar 2021, at 21:24, Peter M=C3=BCller <peter.mueller(a)ipfire.org> wr=
ote:
>=20
> The IP range given in an inetnum object apparently not necessarily
> matches distinct subnet boundaries. As a result, the current attempt to
> calculate its CIDR mask resulted in faulty subnets not covering the
> entire IP range.
>=20
> This patch leaves the task of enumerating subnets to the ipaddress
> module itself, which handles things much more robust. Since the output
> may contain of several subnets, a list for the inetnum key is necessary
> as well as a loop over them when conducting the SQL statements.
>=20
> Fixes: #12595
>=20
> Cc: Michael Tremer <michael.tremer(a)ipfire.org>
> Signed-off-by: Peter M=C3=BCller <peter.mueller(a)ipfire.org>
> ---
> src/python/location-importer.in | 31 +++++++++++--------------------
> 1 file changed, 11 insertions(+), 20 deletions(-)
>=20
> diff --git a/src/python/location-importer.in b/src/python/location-importer=
.in
> index 2506925..e2f201b 100644
> --- a/src/python/location-importer.in
> +++ b/src/python/location-importer.in
> @@ -3,7 +3,7 @@
> #                                                                          =
   #
> # libloc - A library to determine the location of someone on the Internet  =
   #
> #                                                                          =
   #
> -# Copyright (C) 2020 IPFire Development Team <info(a)ipfire.org>          =
      #
> +# Copyright (C) 2020-2021 IPFire Development Team <info(a)ipfire.org>     =
      #
> #                                                                          =
   #
> # This library is free software; you can redistribute it and/or            =
   #
> # modify it under the terms of the GNU Lesser General Public               =
   #
> @@ -604,18 +604,10 @@ class CLI(object):
> 					log.warning("Could not parse line: %s" % line)
> 					return
>=20
> -				# Set prefix to default
> -				prefix =3D 32
> -
> -				# Count number of addresses in this subnet
> -				num_addresses =3D int(end_address) - int(start_address)
> -				if num_addresses:
> -					prefix -=3D math.log(num_addresses, 2)
> -
> -				inetnum["inetnum"] =3D "%s/%.0f" % (start_address, prefix)
> +				inetnum["inetnum"] =3D list(ipaddress.summarize_address_range(start_ad=
dress, end_address))
>=20
> 			elif key =3D=3D "inet6num":
> -				inetnum[key] =3D val
> +				inetnum[key] =3D [ipaddress.ip_network(val, strict=3DFalse)]
>=20
> 			elif key =3D=3D "country":
> 				inetnum[key] =3D val.upper()
> @@ -630,15 +622,14 @@ class CLI(object):
> 				(inetnum.get("inet6num") or inetnum.get("inetnum")))
> 			return
>=20
> -		network =3D ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get(=
"inetnum"), strict=3DFalse)
> -
> -		if not self._check_parsed_network(network):
> -			return
> -
> -		self.db.execute("INSERT INTO _rirdata(network, country) \
> -			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country =3D excluded=
.country",
> -			"%s" % network, inetnum.get("country"),
> -		)
> +		# Iterate through all networks enumerated from above, check them for pla=
usibility and insert
> +		# them into the database, if _check_parsed_network() succeeded
> +		for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"):
> +			if self._check_parsed_network(single_network):
> +				self.db.execute("INSERT INTO _rirdata(network, country) \
> +					VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country =3D exclud=
ed.country",
> +					"%s" % single_network, inetnum.get("country"),
> +				)
>=20
> 	def _parse_org_block(self, block):
> 		org =3D {}
> --=20
> 2.26.2


--===============7575780521212185400==--