Reviewed-by: Michael Tremer > On 20 Aug 2021, at 21:04, Adolf Belka wrote: > > - Implement python3 version of certdata2pem.py script from fedora > - Modify build.sh to work with python3 script that uses p11-kit based on fedora > approach - https://src.fedoraproject.org/rpms/ca-certificates/tree/rawhide > - Extraction of cert files now uses p11-kit which requires libtasn1 as a build > dependency > - Updated rootfile > - Updated ca-certificates installed into a vm and confirmed to download a file from an > https site with the same results as with existing ca-certfictaes system > > Tested-by: Adolf Belka > Signed-off-by: Adolf Belka > --- > config/ca-certificates/build.sh | 48 +++-- > config/ca-certificates/certdata2pem.py | 260 ++++++++++++++++++++---- > config/rootfiles/common/ca-certificates | 5 +- > lfs/ca-certificates | 2 +- > 4 files changed, 248 insertions(+), 67 deletions(-) > > diff --git a/config/ca-certificates/build.sh b/config/ca-certificates/build.sh > index c868ed94a..8e64f9e9f 100644 > --- a/config/ca-certificates/build.sh > +++ b/config/ca-certificates/build.sh > @@ -3,13 +3,34 @@ > set -e > > # Create file layout. > -mkdir -pv certs certs/legacy-default certs/legacy-disable > +mkdir -pv certs > +mkdir -pv /etc/pki/ca-trust/source > cp certdata.txt certs > cd certs > > -python ../certdata2pem.py > +python3 ../certdata2pem.py > > cd .. > + > + > +cat < ca-bundle.trust.p11-kit > +# This is a bundle of X.509 certificates of public Certificate > +# Authorities. It was generated from the Mozilla root CA list. > +# These certificates and trust/distrust attributes use the file format accepted > +# by the p11-kit-trust module. > +# > +# Source: mozilla/security/nss/lib/ckfw/builtins/certdata.txt > +# > +EOF > + > + > +P11FILES=`find certs -name \*.tmp-p11-kit | wc -l` > +if [ $P11FILES -ne 0 ]; then > + for p in certs/*.tmp-p11-kit; do > + cat "$p" >> /etc/pki/ca-trust/source/ca-bundle.trust.p11-kit > + done > +fi > + > cat < ca-bundle.crt > # This is a bundle of X.509 certificates of public Certificate > # Authorities. It was generated from the Mozilla root CA list. > @@ -28,24 +49,11 @@ cat < ca-bundle.trust.crt > # > EOF > > -for f in certs/*.crt; do > - [ -z "${f}" ] && continue > - > - tbits=$(sed -n '/^# openssl-trust/{s/^.*=//;p;}' ${f}) > - case "${tbits}" in > - *serverAuth*) > - openssl x509 -text -in "${f}" >> ca-bundle.crt > - ;; > - esac > +trust extract --comment --filter=certificates --format=openssl-bundle --overwrite ca-bundle.trust > +cat ca-bundle.trust >> ca-bundle.trust.crt > > - if [ -n "$tbits" ]; then > - targs="" > - for t in ${tbits}; do > - targs="${targs} -addtrust ${t}" > - done > +trust extract --comment --filter=ca-anchors --format=pem-bundle --overwrite --purpose server-auth ca-bundle > +cat ca-bundle >> ca-bundle.crt > > - openssl x509 -text -in "${f}" -trustout $targs >> ca-bundle.trust.crt > - fi > -done > > -exit 0 > +exit 0 > \ No newline at end of file > diff --git a/config/ca-certificates/certdata2pem.py b/config/ca-certificates/certdata2pem.py > index 44cc9e03b..a52ce9c74 100644 > --- a/config/ca-certificates/certdata2pem.py > +++ b/config/ca-certificates/certdata2pem.py > @@ -26,16 +26,17 @@ import os.path > import re > import sys > import textwrap > -import urllib > +import urllib.request, urllib.parse, urllib.error > +import subprocess > > objects = [] > > def printable_serial(obj): > - return ".".join(map(lambda x:str(ord(x)), obj['CKA_SERIAL_NUMBER'])) > + return ".".join([str(x) for x in obj['CKA_SERIAL_NUMBER']]) > > # Dirty file parser. > in_data, in_multiline, in_obj = False, False, False > -field, type, value, obj = None, None, None, dict() > +field, ftype, value, binval, obj = None, None, None, bytearray(), dict() > for line in open('certdata.txt', 'r'): > # Ignore the file header. > if not in_data: > @@ -55,33 +56,36 @@ for line in open('certdata.txt', 'r'): > continue > if in_multiline: > if not line.startswith('END'): > - if type == 'MULTILINE_OCTAL': > + if ftype == 'MULTILINE_OCTAL': > line = line.strip() > for i in re.finditer(r'\\([0-3][0-7][0-7])', line): > - value += chr(int(i.group(1), 8)) > + integ = int(i.group(1), 8) > + binval.extend((integ).to_bytes(1, sys.byteorder)) > + obj[field] = binval > else: > value += line > + obj[field] = value > continue > - obj[field] = value > in_multiline = False > continue > if line.startswith('CKA_CLASS'): > in_obj = True > line_parts = line.strip().split(' ', 2) > if len(line_parts) > 2: > - field, type = line_parts[0:2] > + field, ftype = line_parts[0:2] > value = ' '.join(line_parts[2:]) > elif len(line_parts) == 2: > - field, type = line_parts > + field, ftype = line_parts > value = None > else: > - raise NotImplementedError, 'line_parts < 2 not supported.\n' + line > - if type == 'MULTILINE_OCTAL': > + raise NotImplementedError('line_parts < 2 not supported.\n' + line) > + if ftype == 'MULTILINE_OCTAL': > in_multiline = True > value = "" > + binval = bytearray() > continue > obj[field] = value > -if len(obj.items()) > 0: > +if len(list(obj.items())) > 0: > objects.append(obj) > > # Build up trust database. > @@ -91,7 +95,7 @@ for obj in objects: > continue > key = obj['CKA_LABEL'] + printable_serial(obj) > trustmap[key] = obj > - print " added trust", key > + print(" added trust", key) > > # Build up cert database. > certmap = dict() > @@ -100,7 +104,7 @@ for obj in objects: > continue > key = obj['CKA_LABEL'] + printable_serial(obj) > certmap[key] = obj > - print " added cert", key > + print(" added cert", key) > > def obj_to_filename(obj): > label = obj['CKA_LABEL'][1:-1] > @@ -109,10 +113,32 @@ def obj_to_filename(obj): > .replace('(', '=')\ > .replace(')', '=')\ > .replace(',', '_') > - label = re.sub(r'\\x[0-9a-fA-F]{2}', lambda m:chr(int(m.group(0)[2:], 16)), label) > + labelbytes = bytearray() > + i = 0 > + imax = len(label) > + while i < imax: > + if i < imax-3 and label[i] == '\\' and label[i+1] == 'x': > + labelbytes.extend(bytes.fromhex(label[i+2:i+4])) > + i += 4 > + continue > + labelbytes.extend(str.encode(label[i])) > + i = i+1 > + continue > + label = labelbytes.decode('utf-8') > serial = printable_serial(obj) > return label + ":" + serial > > +def write_cert_ext_to_file(f, oid, value, public_key): > + f.write("[p11-kit-object-v1]\n") > + f.write("label: "); > + f.write(tobj['CKA_LABEL']) > + f.write("\n") > + f.write("class: x-certificate-extension\n"); > + f.write("object-id: " + oid + "\n") > + f.write("value: \"" + value + "\"\n") > + f.write("modifiable: false\n"); > + f.write(public_key) > + > trust_types = { > "CKA_TRUST_DIGITAL_SIGNATURE": "digital-signature", > "CKA_TRUST_NON_REPUDIATION": "non-repudiation", > @@ -151,34 +177,39 @@ openssl_trust = { > "CKA_TRUST_EMAIL_PROTECTION": "emailProtection", > } > > +cert_distrust_types = { > + "CKA_NSS_SERVER_DISTRUST_AFTER": "nss-server-distrust-after", > + "CKA_NSS_EMAIL_DISTRUST_AFTER": "nss-email-distrust-after", > +} > + > for tobj in objects: > if tobj['CKA_CLASS'] == 'CKO_NSS_TRUST': > key = tobj['CKA_LABEL'] + printable_serial(tobj) > - print "producing trust for " + key > + print("producing trust for " + key) > trustbits = [] > distrustbits = [] > openssl_trustflags = [] > openssl_distrustflags = [] > legacy_trustbits = [] > legacy_openssl_trustflags = [] > - for t in trust_types.keys(): > - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR': > + for t in list(trust_types.keys()): > + if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR': > trustbits.append(t) > if t in openssl_trust: > openssl_trustflags.append(openssl_trust[t]) > - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED': > + if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED': > distrustbits.append(t) > if t in openssl_trust: > openssl_distrustflags.append(openssl_trust[t]) > > - for t in legacy_trust_types.keys(): > - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR': > + for t in list(legacy_trust_types.keys()): > + if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR': > real_t = legacy_to_real_trust_types[t] > legacy_trustbits.append(real_t) > if real_t in openssl_trust: > legacy_openssl_trustflags.append(openssl_trust[real_t]) > - if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED': > - raise NotImplementedError, 'legacy distrust not supported.\n' + line > + if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED': > + raise NotImplementedError('legacy distrust not supported.\n' + line) > > fname = obj_to_filename(tobj) > try: > @@ -186,43 +217,181 @@ for tobj in objects: > except: > obj = None > > - if obj != None: > - fname += ".crt" > - else: > - fname += ".p11-kit" > + # optional debug code, that dumps the parsed input to files > + #fulldump = "dump-" + fname > + #dumpf = open(fulldump, 'w') > + #dumpf.write(str(obj)); > + #dumpf.write(str(tobj)); > + #dumpf.close(); > > is_legacy = 0 > - if tobj.has_key('LEGACY_CKA_TRUST_SERVER_AUTH') or tobj.has_key('LEGACY_CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('LEGACY_CKA_TRUST_CODE_SIGNING'): > + if 'LEGACY_CKA_TRUST_SERVER_AUTH' in tobj or 'LEGACY_CKA_TRUST_EMAIL_PROTECTION' in tobj or 'LEGACY_CKA_TRUST_CODE_SIGNING' in tobj: > is_legacy = 1 > if obj == None: > - raise NotImplementedError, 'found legacy trust without certificate.\n' + line > - legacy_fname = "legacy-default/" + fname > + raise NotImplementedError('found legacy trust without certificate.\n' + line) > + > + legacy_fname = "legacy-default/" + fname + ".crt" > f = open(legacy_fname, 'w') > f.write("# alias=%s\n"%tobj['CKA_LABEL']) > f.write("# trust=" + " ".join(legacy_trustbits) + "\n") > if legacy_openssl_trustflags: > f.write("# openssl-trust=" + " ".join(legacy_openssl_trustflags) + "\n") > f.write("-----BEGIN CERTIFICATE-----\n") > - f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64))) > + temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE']) > + temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64) > + f.write("\n".join(temp_wrapped)) > f.write("\n-----END CERTIFICATE-----\n") > f.close() > - if tobj.has_key('CKA_TRUST_SERVER_AUTH') or tobj.has_key('CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('CKA_TRUST_CODE_SIGNING'): > - fname = "legacy-disable/" + fname > - else: > - continue > > + if 'CKA_TRUST_SERVER_AUTH' in tobj or 'CKA_TRUST_EMAIL_PROTECTION' in tobj or 'CKA_TRUST_CODE_SIGNING' in tobj: > + legacy_fname = "legacy-disable/" + fname + ".crt" > + f = open(legacy_fname, 'w') > + f.write("# alias=%s\n"%tobj['CKA_LABEL']) > + f.write("# trust=" + " ".join(trustbits) + "\n") > + if openssl_trustflags: > + f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n") > + f.write("-----BEGIN CERTIFICATE-----\n") > + f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64))) > + f.write("\n-----END CERTIFICATE-----\n") > + f.close() > + > + # don't produce p11-kit output for legacy certificates > + continue > + > + pk = '' > + cert_comment = '' > + if obj != None: > + # must extract the public key from the cert, let's use openssl > + cert_fname = "cert-" + fname > + fc = open(cert_fname, 'w') > + fc.write("-----BEGIN CERTIFICATE-----\n") > + temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE']) > + temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64) > + fc.write("\n".join(temp_wrapped)) > + fc.write("\n-----END CERTIFICATE-----\n") > + fc.close(); > + pk_fname = "pubkey-" + fname > + fpkout = open(pk_fname, "w") > + dump_pk_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-pubkey"] > + subprocess.call(dump_pk_command, stdout=fpkout) > + fpkout.close() > + with open (pk_fname, "r") as myfile: > + pk=myfile.read() > + # obtain certificate information suitable as a comment > + comment_fname = "comment-" + fname > + fcout = open(comment_fname, "w") > + comment_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-text"] > + subprocess.call(comment_command, stdout=fcout) > + fcout.close() > + sed_command = ["sed", "--in-place", "s/^/#/", comment_fname] > + subprocess.call(sed_command) > + with open (comment_fname, "r", errors = 'replace') as myfile: > + cert_comment=myfile.read() > + > + fname += ".tmp-p11-kit" > f = open(fname, 'w') > + > if obj != None: > - f.write("# alias=%s\n"%tobj['CKA_LABEL']) > - f.write("# trust=" + " ".join(trustbits) + "\n") > - f.write("# distrust=" + " ".join(distrustbits) + "\n") > - if openssl_trustflags: > - f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n") > - if openssl_distrustflags: > - f.write("# openssl-distrust=" + " ".join(openssl_distrustflags) + "\n") > + is_distrusted = False > + has_server_trust = False > + has_email_trust = False > + has_code_trust = False > + > + if 'CKA_TRUST_SERVER_AUTH' in tobj: > + if tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED': > + is_distrusted = True > + elif tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_TRUSTED_DELEGATOR': > + has_server_trust = True > + > + if 'CKA_TRUST_EMAIL_PROTECTION' in tobj: > + if tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED': > + is_distrusted = True > + elif tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_TRUSTED_DELEGATOR': > + has_email_trust = True > + > + if 'CKA_TRUST_CODE_SIGNING' in tobj: > + if tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED': > + is_distrusted = True > + elif tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_TRUSTED_DELEGATOR': > + has_code_trust = True > + > + if is_distrusted: > + trust_ext_oid = "1.3.6.1.4.1.3319.6.10.1" > + trust_ext_value = "0.%06%0a%2b%06%01%04%01%99w%06%0a%01%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03" > + write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk) > + > + trust_ext_oid = "2.5.29.37" > + if has_server_trust: > + if has_email_trust: > + if has_code_trust: > + # server + email + code > + trust_ext_value = "0%2a%06%03U%1d%25%01%01%ff%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03" > + else: > + # server + email > + trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01" > + else: > + if has_code_trust: > + # server + code > + trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03" > + else: > + # server > + trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%01" > + else: > + if has_email_trust: > + if has_code_trust: > + # email + code > + trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%03" > + else: > + # email > + trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%04" > + else: > + if has_code_trust: > + # code > + trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%03" > + else: > + # none > + trust_ext_value = "0%18%06%03U%1d%25%01%01%ff%04%0e0%0c%06%0a%2b%06%01%04%01%99w%06%0a%10" > + > + # no 2.5.29.37 for neutral certificates > + if (is_distrusted or has_server_trust or has_email_trust or has_code_trust): > + write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk) > + > + pk = '' > + f.write("\n") > + > + f.write("[p11-kit-object-v1]\n") > + f.write("label: "); > + f.write(tobj['CKA_LABEL']) > + f.write("\n") > + if is_distrusted: > + f.write("x-distrusted: true\n") > + elif has_server_trust or has_email_trust or has_code_trust: > + f.write("trusted: true\n") > + else: > + f.write("trusted: false\n") > + > + # requires p11-kit >= 0.23.4 > + f.write("nss-mozilla-ca-policy: true\n") > + f.write("modifiable: false\n"); > + > + # requires p11-kit >= 0.23.19 > + for t in list(cert_distrust_types.keys()): > + if t in obj: > + value = obj[t] > + if value == 'CK_FALSE': > + value = bytearray(1) > + f.write(cert_distrust_types[t] + ": \"") > + f.write(urllib.parse.quote(value)); > + f.write("\"\n") > + > f.write("-----BEGIN CERTIFICATE-----\n") > - f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64))) > + temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE']) > + temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64) > + f.write("\n".join(temp_wrapped)) > f.write("\n-----END CERTIFICATE-----\n") > + f.write(cert_comment) > + f.write("\n") > + > else: > f.write("[p11-kit-object-v1]\n") > f.write("label: "); > @@ -230,14 +399,15 @@ for tobj in objects: > f.write("\n") > f.write("class: certificate\n") > f.write("certificate-type: x-509\n") > + f.write("modifiable: false\n"); > f.write("issuer: \""); > - f.write(urllib.quote(tobj['CKA_ISSUER'])); > + f.write(urllib.parse.quote(tobj['CKA_ISSUER'])); > f.write("\"\n") > f.write("serial-number: \""); > - f.write(urllib.quote(tobj['CKA_SERIAL_NUMBER'])); > + f.write(urllib.parse.quote(tobj['CKA_SERIAL_NUMBER'])); > f.write("\"\n") > if (tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED'): > f.write("x-distrusted: true\n") > f.write("\n\n") > f.close() > - print " -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags) > + print(" -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags)) > diff --git a/config/rootfiles/common/ca-certificates b/config/rootfiles/common/ca-certificates > index 087c3e450..06eb66f3b 100644 > --- a/config/rootfiles/common/ca-certificates > +++ b/config/rootfiles/common/ca-certificates > @@ -1,4 +1,7 @@ > +#etc/pki > +#etc/pki/ca-trust > +#etc/pki/ca-trust/source > +etc/pki/ca-trust/source/ca-bundle.trust.p11-kit > etc/ssl/cert.pem > -#etc/ssl/certs > etc/ssl/certs/ca-bundle.crt > etc/ssl/certs/ca-bundle.trust.crt > diff --git a/lfs/ca-certificates b/lfs/ca-certificates > index f3c68a7c0..9e37687da 100644 > --- a/lfs/ca-certificates > +++ b/lfs/ca-certificates > @@ -24,7 +24,7 @@ > > include Config > > -VER = 20210611 > +VER = 20210819 > > THISAPP = ca-certificates > DIR_APP = $(DIR_SRC)/$(THISAPP) > -- > 2.33.0 >