From mboxrd@z Thu Jan  1 00:00:00 1970
From: Michael Tremer <michael.tremer@ipfire.org>
To: location@lists.ipfire.org
Subject: Re: [PATCH] location-importer.in: Conduct sanity checks per DROP list
Date: Fri, 19 Aug 2022 09:25:31 +0100
Message-ID: <AA718931-2C3E-44E2-A4E2-821A6817320A@ipfire.org>
In-Reply-To: <db340fda-1c2d-ce45-9660-619e61d27f85@ipfire.org>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="===============3686558152080402134=="
List-Id: <location.lists.ipfire.org>

--===============3686558152080402134==
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: quoted-printable

Hello Matthias,

This patch is against libloc:

  https://git.ipfire.org/?p=3Dlocation/libloc.git;a=3Dsummary

-Michael

> On 19 Aug 2022, at 09:21, Matthias Fischer <matthias.fischer(a)ipfire.org> =
wrote:
>=20
> Hi,
>=20
> being curious I wanted to test this patch, but couldn't find
> 'location-importer.in' in 'src/scripts'. No chance to apply this patch.
>=20
> Finally I found '.../build/usr/bin/location-importer'. Hm. Ok.
>=20
> But how do I apply this patch?
>=20
> Still being curious... ;-)
>=20
> Matthias
>=20
> On 18.08.2022 23:42, Peter M=C3=BCller wrote:
>> Previously, the lack of distinction between different DROP lists caused
>> only the last one to be persisted.
>>=20
>> Signed-off-by: Peter M=C3=BCller <peter.mueller(a)ipfire.org>
>> ---
>> src/scripts/location-importer.in | 58 +++++++++++++++++++++-----------
>> 1 file changed, 39 insertions(+), 19 deletions(-)
>>=20
>> diff --git a/src/scripts/location-importer.in b/src/scripts/location-impor=
ter.in
>> index 8d47497..e4a9ab0 100644
>> --- a/src/scripts/location-importer.in
>> +++ b/src/scripts/location-importer.in
>> @@ -1427,18 +1427,21 @@ class CLI(object):
>> 	def _update_overrides_for_spamhaus_drop(self):
>> 		downloader =3D location.importer.Downloader()
>>=20
>> -		ip_urls =3D [
>> -					"https://www.spamhaus.org/drop/drop.txt",
>> -					"https://www.spamhaus.org/drop/edrop.txt",
>> -					"https://www.spamhaus.org/drop/dropv6.txt"
>> +		ip_lists =3D [
>> +					("Spamhaus DROP list", "https://www.spamhaus.org/drop/drop.txt"),
>> +					("Spamhaus EDROP list", "https://www.spamhaus.org/drop/edrop.txt"),
>> +					("Spamhaus DROPv6 list", "https://www.spamhaus.org/drop/dropv6.txt")
>> 				]
>>=20
>> -		asn_urls =3D [
>> -					"https://www.spamhaus.org/drop/asndrop.txt"
>> +		asn_lists =3D [
>> +					("Spamhaus ASN-DROP list", "https://www.spamhaus.org/drop/asndrop.tx=
t")
>> 				]
>>=20
>> -		for url in ip_urls:
>> -			# Fetch IP list
>> +		for list in ip_lists:
>> +			name =3D list[0]
>> +			url =3D list[1]
>> +
>> +			# Fetch IP list from given URL
>> 			f =3D downloader.retrieve(url)
>>=20
>> 			# Split into lines
>> @@ -1448,11 +1451,11 @@ class CLI(object):
>> 			# downloads.
>> 			if len(fcontent) > 10:
>> 				self.db.execute("""
>> -					DELETE FROM autnum_overrides WHERE source =3D 'Spamhaus ASN-DROP lis=
t';
>> -					DELETE FROM network_overrides WHERE source =3D 'Spamhaus DROP lists';
>> -				""")
>> +					DELETE FROM network_overrides WHERE source =3D '%s';
>> +				""" % name,
>> +				)
>> 			else:
>> -				log.error("Spamhaus DROP URL %s returned likely bogus file, ignored" =
% url)
>> +				log.error("%s (%s) returned likely bogus file, ignored" % (name, url))
>> 				continue
>>=20
>> 			# Iterate through every line, filter comments and add remaining network=
s to
>> @@ -1475,8 +1478,8 @@ class CLI(object):
>>=20
>> 					# Sanitize parsed networks...
>> 					if not self._check_parsed_network(network):
>> -						log.warning("Skipping bogus network found in Spamhaus DROP URL %s: =
%s" % \
>> -							(url, network))
>> +						log.warning("Skipping bogus network found in %s (%s): %s" % \
>> +							(name, url, network))
>> 						continue
>>=20
>> 					# Conduct SQL statement...
>> @@ -1488,14 +1491,31 @@ class CLI(object):
>> 						) VALUES (%s, %s, %s)
>> 						ON CONFLICT (network) DO UPDATE SET is_drop =3D True""",
>> 						"%s" % network,
>> -						"Spamhaus DROP lists",
>> +						name,
>> 						True
>> 					)
>>=20
>> -		for url in asn_urls:
>> +		for list in asn_lists:
>> +			name =3D list[0]
>> +			url =3D list[1]
>> +
>> 			# Fetch URL
>> 			f =3D downloader.retrieve(url)
>>=20
>> +			# Split into lines
>> +			fcontent =3D f.readlines()
>> +
>> +			# Conduct a very basic sanity check to rule out CDN issues causing bog=
us DROP
>> +			# downloads.
>> +			if len(fcontent) > 10:
>> +				self.db.execute("""
>> +					DELETE FROM autnum_overrides WHERE source =3D '%s';
>> +				""" % name,
>> +				)
>> +			else:
>> +				log.error("%s (%s) returned likely bogus file, ignored" % (name, url))
>> +				continue
>> +
>> 			# Iterate through every line, filter comments and add remaining ASNs to
>> 			# the override table in case they are valid...
>> 			with self.db.transaction():
>> @@ -1518,8 +1538,8 @@ class CLI(object):
>>=20
>> 					# Filter invalid ASNs...
>> 					if not self._check_parsed_asn(asn):
>> -						log.warning("Skipping bogus ASN found in Spamhaus DROP URL %s: %s" =
% \
>> -							(url, asn))
>> +						log.warning("Skipping bogus ASN found in %s (%s): %s" % \
>> +							(name, url, asn))
>> 						continue
>>=20
>> 					# Conduct SQL statement...
>> @@ -1531,7 +1551,7 @@ class CLI(object):
>> 						) VALUES (%s, %s, %s)
>> 						ON CONFLICT (number) DO UPDATE SET is_drop =3D True""",
>> 						"%s" % asn,
>> -						"Spamhaus ASN-DROP list",
>> +						name,
>> 						True
>> 					)
>>=20
>=20


--===============3686558152080402134==--