Re: [PATCH 1/4] ca-certificates: Update to work with python3 version of certdata2pem.py

Reviewed-by: Michael Tremer michael.tremer@ipfire.org

On 20 Aug 2021, at 21:04, Adolf Belka adolf.belka@ipfire.org wrote:

• Implement python3 version of certdata2pem.py script from fedora
• Modify build.sh to work with python3 script that uses p11-kit based on fedora approach - https://src.fedoraproject.org/rpms/ca-certificates/tree/rawhide
• Extraction of cert files now uses p11-kit which requires libtasn1 as a build dependency
• Updated rootfile
• Updated ca-certificates installed into a vm and confirmed to download a file from an https site with the same results as with existing ca-certfictaes system

Tested-by: Adolf Belka adolf.belka@ipfire.org Signed-off-by: Adolf Belka adolf.belka@ipfire.org

---

config/ca-certificates/build.sh | 48 +++-- config/ca-certificates/certdata2pem.py | 260 ++++++++++++++++++++---- config/rootfiles/common/ca-certificates | 5 +- lfs/ca-certificates | 2 +- 4 files changed, 248 insertions(+), 67 deletions(-)

diff --git a/config/ca-certificates/build.sh b/config/ca-certificates/build.sh index c868ed94a..8e64f9e9f 100644 --- a/config/ca-certificates/build.sh +++ b/config/ca-certificates/build.sh @@ -3,13 +3,34 @@ set -e

# Create file layout. -mkdir -pv certs certs/legacy-default certs/legacy-disable +mkdir -pv certs +mkdir -pv /etc/pki/ca-trust/source cp certdata.txt certs cd certs

-python ../certdata2pem.py +python3 ../certdata2pem.py

cd ..

+cat <<EOF > ca-bundle.trust.p11-kit +# This is a bundle of X.509 certificates of public Certificate +# Authorities. It was generated from the Mozilla root CA list. +# These certificates and trust/distrust attributes use the file format accepted +# by the p11-kit-trust module. +# +# Source: mozilla/security/nss/lib/ckfw/builtins/certdata.txt +# +EOF

+P11FILES=`find certs -name *.tmp-p11-kit | wc -l` +if [ $P11FILES -ne 0 ]; then

• for p in certs/*.tmp-p11-kit; do
• cat "$p" >> /etc/pki/ca-trust/source/ca-bundle.trust.p11-kit
• done

+fi

cat <<EOF > ca-bundle.crt # This is a bundle of X.509 certificates of public Certificate # Authorities. It was generated from the Mozilla root CA list. @@ -28,24 +49,11 @@ cat <<EOF > ca-bundle.trust.crt # EOF

-for f in certs/*.crt; do

• [ -z "${f}" ] && continue
• tbits=$(sed -n '/^# openssl-trust/{s/^.*=//;p;}' ${f})
• case "${tbits}" in

• *serverAuth*)
  

• 	openssl x509 -text -in "${f}" >> ca-bundle.crt
  

• 	;;
  

• esac

+trust extract --comment --filter=certificates --format=openssl-bundle --overwrite ca-bundle.trust +cat ca-bundle.trust >> ca-bundle.trust.crt

• if [ -n "$tbits" ]; then

• targs=""
  

• for t in ${tbits}; do
  

• 	targs="${targs} -addtrust ${t}"
  

• done
  

+trust extract --comment --filter=ca-anchors --format=pem-bundle --overwrite --purpose server-auth ca-bundle +cat ca-bundle >> ca-bundle.crt

• openssl x509 -text -in "${f}" -trustout $targs >> ca-bundle.trust.crt
  

• fi

-done

-exit 0 +exit 0 \ No newline at end of file diff --git a/config/ca-certificates/certdata2pem.py b/config/ca-certificates/certdata2pem.py index 44cc9e03b..a52ce9c74 100644 --- a/config/ca-certificates/certdata2pem.py +++ b/config/ca-certificates/certdata2pem.py @@ -26,16 +26,17 @@ import os.path import re import sys import textwrap -import urllib +import urllib.request, urllib.parse, urllib.error +import subprocess

objects = []

def printable_serial(obj):

• return ".".join(map(lambda x:str(ord(x)), obj['CKA_SERIAL_NUMBER']))

• return ".".join([str(x) for x in obj['CKA_SERIAL_NUMBER']])

# Dirty file parser. in_data, in_multiline, in_obj = False, False, False -field, type, value, obj = None, None, None, dict() +field, ftype, value, binval, obj = None, None, None, bytearray(), dict() for line in open('certdata.txt', 'r'): # Ignore the file header. if not in_data: @@ -55,33 +56,36 @@ for line in open('certdata.txt', 'r'): continue if in_multiline: if not line.startswith('END'):

•        if type == 'MULTILINE_OCTAL':
  

•        if ftype == 'MULTILINE_OCTAL':
            line = line.strip()
            for i in re.finditer(r'\\([0-3][0-7][0-7])', line):
  

•                value += chr(int(i.group(1), 8))
  

•                integ = int(i.group(1), 8)
  

•                binval.extend((integ).to_bytes(1, sys.byteorder))
  

•            obj[field] = binval
        else:
            value += line
  

•            obj[field] = value
        continue
  

•    obj[field] = value
    in_multiline = False
    continue
  

  if line.startswith('CKA_CLASS'): in_obj = True line_parts = line.strip().split(' ', 2) if len(line_parts) > 2:

•    field, type = line_parts[0:2]
  

•    field, ftype = line_parts[0:2]
    value = ' '.join(line_parts[2:])
  

  elif len(line_parts) == 2:

•    field, type = line_parts
  

•    field, ftype = line_parts
    value = None
  

  else:

•    raise NotImplementedError, 'line_parts < 2 not supported.\n' + line
  

• if type == 'MULTILINE_OCTAL':

•    raise NotImplementedError('line_parts < 2 not supported.\n' + line)
  

• if ftype == 'MULTILINE_OCTAL': in_multiline = True value = ""

•    binval = bytearray()
    continue
  

  obj[field] = value

-if len(obj.items()) > 0: +if len(list(obj.items())) > 0: objects.append(obj)

# Build up trust database. @@ -91,7 +95,7 @@ for obj in objects: continue key = obj['CKA_LABEL'] + printable_serial(obj) trustmap[key] = obj

• print " added trust", key

• print(" added trust", key)

# Build up cert database. certmap = dict() @@ -100,7 +104,7 @@ for obj in objects: continue key = obj['CKA_LABEL'] + printable_serial(obj) certmap[key] = obj

• print " added cert", key

• print(" added cert", key)

def obj_to_filename(obj): label = obj['CKA_LABEL'][1:-1] @@ -109,10 +113,32 @@ def obj_to_filename(obj): .replace('(', '=')\ .replace(')', '=')\ .replace(',', '_')

• label = re.sub(r'\x[0-9a-fA-F]{2}', lambda m:chr(int(m.group(0)[2:], 16)), label)

• labelbytes = bytearray()
• i = 0
• imax = len(label)
• while i < imax:

•    if i < imax-3 and label[i] == '\\' and label[i+1] == 'x':
  

•        labelbytes.extend(bytes.fromhex(label[i+2:i+4]))
  

•        i += 4
  

•        continue
  

•    labelbytes.extend(str.encode(label[i]))
  

•    i = i+1
  

•    continue
  

• label = labelbytes.decode('utf-8') serial = printable_serial(obj) return label + ":" + serial

+def write_cert_ext_to_file(f, oid, value, public_key):

• f.write("[p11-kit-object-v1]\n")
• f.write("label: ");
• f.write(tobj['CKA_LABEL'])
• f.write("\n")
• f.write("class: x-certificate-extension\n");
• f.write("object-id: " + oid + "\n")
• f.write("value: "" + value + ""\n")
• f.write("modifiable: false\n");
• f.write(public_key)

trust_types = { "CKA_TRUST_DIGITAL_SIGNATURE": "digital-signature", "CKA_TRUST_NON_REPUDIATION": "non-repudiation", @@ -151,34 +177,39 @@ openssl_trust = { "CKA_TRUST_EMAIL_PROTECTION": "emailProtection", }

+cert_distrust_types = {

• "CKA_NSS_SERVER_DISTRUST_AFTER": "nss-server-distrust-after",
• "CKA_NSS_EMAIL_DISTRUST_AFTER": "nss-email-distrust-after",

+}

for tobj in objects: if tobj['CKA_CLASS'] == 'CKO_NSS_TRUST': key = tobj['CKA_LABEL'] + printable_serial(tobj)

•    print "producing trust for " + key
  

•    print("producing trust for " + key)
    trustbits = []
    distrustbits = []
    openssl_trustflags = []
    openssl_distrustflags = []
    legacy_trustbits = []
    legacy_openssl_trustflags = []
  

•    for t in trust_types.keys():
  

•        if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
  

•    for t in list(trust_types.keys()):
  

•        if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
            trustbits.append(t)
            if t in openssl_trust:
                openssl_trustflags.append(openssl_trust[t])
  

•        if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
  

•        if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
            distrustbits.append(t)
            if t in openssl_trust:
                openssl_distrustflags.append(openssl_trust[t])
  
  

•    for t in legacy_trust_types.keys():
  

•        if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
  

•    for t in list(legacy_trust_types.keys()):
  

•        if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
            real_t = legacy_to_real_trust_types[t]
            legacy_trustbits.append(real_t)
            if real_t in openssl_trust:
                legacy_openssl_trustflags.append(openssl_trust[real_t])
  

•        if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
  

•            raise NotImplementedError, 'legacy distrust not supported.\n' + line
  

•        if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
  

•            raise NotImplementedError('legacy distrust not supported.\n' + line)
  
    fname = obj_to_filename(tobj)
    try:
  

@@ -186,43 +217,181 @@ for tobj in objects: except: obj = None

•    if obj != None:
  

•        fname += ".crt"
  

•    else:
  

•        fname += ".p11-kit"
  

•    # optional debug code, that dumps the parsed input to files
  

•    #fulldump = "dump-" + fname
  

•    #dumpf = open(fulldump, 'w')
  

•    #dumpf.write(str(obj));
  

•    #dumpf.write(str(tobj));
  

•    #dumpf.close();
  
    is_legacy = 0
  

•    if tobj.has_key('LEGACY_CKA_TRUST_SERVER_AUTH') or tobj.has_key('LEGACY_CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('LEGACY_CKA_TRUST_CODE_SIGNING'):
  

•    if 'LEGACY_CKA_TRUST_SERVER_AUTH' in tobj or 'LEGACY_CKA_TRUST_EMAIL_PROTECTION' in tobj or 'LEGACY_CKA_TRUST_CODE_SIGNING' in tobj:
        is_legacy = 1
        if obj == None:
  

•            raise NotImplementedError, 'found legacy trust without certificate.\n' + line
  

•        legacy_fname = "legacy-default/" + fname
  

•            raise NotImplementedError('found legacy trust without certificate.\n' + line)
  

•        legacy_fname = "legacy-default/" + fname + ".crt"
        f = open(legacy_fname, 'w')
        f.write("# alias=%s\n"%tobj['CKA_LABEL'])
        f.write("# trust=" + " ".join(legacy_trustbits) + "\n")
        if legacy_openssl_trustflags:
            f.write("# openssl-trust=" + " ".join(legacy_openssl_trustflags) + "\n")
        f.write("-----BEGIN CERTIFICATE-----\n")
  

•        f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
  

•        temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
  

•        temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
  

•        f.write("\n".join(temp_wrapped))
        f.write("\n-----END CERTIFICATE-----\n")
        f.close()
  

•        if tobj.has_key('CKA_TRUST_SERVER_AUTH') or tobj.has_key('CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('CKA_TRUST_CODE_SIGNING'):
  

•            fname = "legacy-disable/" + fname
  

•        else:
  

•            continue
  
  

•        if 'CKA_TRUST_SERVER_AUTH' in tobj or 'CKA_TRUST_EMAIL_PROTECTION' in tobj or 'CKA_TRUST_CODE_SIGNING' in tobj:
  

•            legacy_fname = "legacy-disable/" + fname + ".crt"
  

•            f = open(legacy_fname, 'w')
  

•            f.write("# alias=%s\n"%tobj['CKA_LABEL'])
  

•            f.write("# trust=" + " ".join(trustbits) + "\n")
  

•            if openssl_trustflags:
  

•                f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
  

•            f.write("-----BEGIN CERTIFICATE-----\n")
  

•            f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
  

•            f.write("\n-----END CERTIFICATE-----\n")
  

•            f.close()
  

•        # don't produce p11-kit output for legacy certificates
  

•        continue
  

•    pk = ''
  

•    cert_comment = ''
  

•    if obj != None:
  

•        # must extract the public key from the cert, let's use openssl
  

•        cert_fname = "cert-" + fname
  

•        fc = open(cert_fname, 'w')
  

•        fc.write("-----BEGIN CERTIFICATE-----\n")
  

•        temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
  

•        temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
  

•        fc.write("\n".join(temp_wrapped))
  

•        fc.write("\n-----END CERTIFICATE-----\n")
  

•        fc.close();
  

•        pk_fname = "pubkey-" + fname
  

•        fpkout = open(pk_fname, "w")
  

•        dump_pk_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-pubkey"]
  

•        subprocess.call(dump_pk_command, stdout=fpkout)
  

•        fpkout.close()
  

•        with open (pk_fname, "r") as myfile:
  

•            pk=myfile.read()
  

•        # obtain certificate information suitable as a comment
  

•        comment_fname = "comment-" + fname
  

•        fcout = open(comment_fname, "w")
  

•        comment_command = ["openssl", "x509", "-in", cert_fname, "-noout", "-text"]
  

•        subprocess.call(comment_command, stdout=fcout)
  

•        fcout.close()
  

•        sed_command = ["sed", "--in-place", "s/^/#/", comment_fname]
  

•        subprocess.call(sed_command)
  

•        with open (comment_fname, "r", errors = 'replace') as myfile:
  

•            cert_comment=myfile.read()
  

•    fname += ".tmp-p11-kit"
    f = open(fname, 'w')
  

•   if obj != None:
  

•        f.write("# alias=%s\n"%tobj['CKA_LABEL'])
  

•        f.write("# trust=" + " ".join(trustbits) + "\n")
  

•        f.write("# distrust=" + " ".join(distrustbits) + "\n")
  

•        if openssl_trustflags:
  

•            f.write("# openssl-trust=" + " ".join(openssl_trustflags) + "\n")
  

•        if openssl_distrustflags:
  

•            f.write("# openssl-distrust=" + " ".join(openssl_distrustflags) + "\n")
  

•        is_distrusted = False
  

•        has_server_trust = False
  

•        has_email_trust = False
  

•        has_code_trust = False
  

•        if 'CKA_TRUST_SERVER_AUTH' in tobj:
  

•            if tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED':
  

•                is_distrusted = True
  

•            elif tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_TRUSTED_DELEGATOR':
  

•                has_server_trust = True
  

•        if 'CKA_TRUST_EMAIL_PROTECTION' in tobj:
  

•            if tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED':
  

•                is_distrusted = True
  

•            elif tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_TRUSTED_DELEGATOR':
  

•                has_email_trust = True
  

•        if 'CKA_TRUST_CODE_SIGNING' in tobj:
  

•            if tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED':
  

•                is_distrusted = True
  

•            elif tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_TRUSTED_DELEGATOR':
  

•                has_code_trust = True
  

•        if is_distrusted:
  

•            trust_ext_oid = "1.3.6.1.4.1.3319.6.10.1"
  

•            trust_ext_value = "0.%06%0a%2b%06%01%04%01%99w%06%0a%01%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
  

•            write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
  

•        trust_ext_oid = "2.5.29.37"
  

•        if has_server_trust:
  

•            if has_email_trust:
  

•                if has_code_trust:
  

•                    # server + email + code
  

•                    trust_ext_value = "0%2a%06%03U%1d%25%01%01%ff%04 0%1e%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
  

•                else:
  

•                    # server + email
  

•                    trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%01"
  

•            else:
  

•                if has_code_trust:
  

•                    # server + code
  

•                    trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%01%06%08%2b%06%01%05%05%07%03%03"
  

•                else:
  

•                    # server
  

•                    trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%01"
  

•        else:
  

•            if has_email_trust:
  

•                if has_code_trust:
  

•                    # email + code
  

•                    trust_ext_value = "0 %06%03U%1d%25%01%01%ff%04%160%14%06%08%2b%06%01%05%05%07%03%04%06%08%2b%06%01%05%05%07%03%03"
  

•                else:
  

•                    # email
  

•                    trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%04"
  

•            else:
  

•                if has_code_trust:
  

•                    # code
  

•                    trust_ext_value = "0%16%06%03U%1d%25%01%01%ff%04%0c0%0a%06%08%2b%06%01%05%05%07%03%03"
  

•                else:
  

•                    # none
  

•                    trust_ext_value = "0%18%06%03U%1d%25%01%01%ff%04%0e0%0c%06%0a%2b%06%01%04%01%99w%06%0a%10"
  

•        # no 2.5.29.37 for neutral certificates
  

•        if (is_distrusted or has_server_trust or has_email_trust or has_code_trust):
  

•            write_cert_ext_to_file(f, trust_ext_oid, trust_ext_value, pk)
  

•        pk = ''
  

•        f.write("\n")
  

•        f.write("[p11-kit-object-v1]\n")
  

•        f.write("label: ");
  

•        f.write(tobj['CKA_LABEL'])
  

•        f.write("\n")
  

•        if is_distrusted:
  

•            f.write("x-distrusted: true\n")
  

•        elif has_server_trust or has_email_trust or has_code_trust:
  

•            f.write("trusted: true\n")
  

•        else:
  

•            f.write("trusted: false\n")
  

•        # requires p11-kit >= 0.23.4
  

•        f.write("nss-mozilla-ca-policy: true\n")
  

•        f.write("modifiable: false\n");
  

•        # requires p11-kit >= 0.23.19
  

•        for t in list(cert_distrust_types.keys()):
  

•            if t in obj:
  

•                value = obj[t]
  

•                if value == 'CK_FALSE':
  

•                    value = bytearray(1)
  

•                f.write(cert_distrust_types[t] + ": \"")
  

•                f.write(urllib.parse.quote(value));
  

•                f.write("\"\n")
  

•       f.write("-----BEGIN CERTIFICATE-----\n")
  

•        f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
  

•        temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
  

•        temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
  

•        f.write("\n".join(temp_wrapped))
        f.write("\n-----END CERTIFICATE-----\n")
  

•        f.write(cert_comment)
  

•        f.write("\n")
  

•   else:
        f.write("[p11-kit-object-v1]\n")
        f.write("label: ");
  

@@ -230,14 +399,15 @@ for tobj in objects: f.write("\n") f.write("class: certificate\n") f.write("certificate-type: x-509\n")

•        f.write("modifiable: false\n");
        f.write("issuer: \"");
  

•        f.write(urllib.quote(tobj['CKA_ISSUER']));
  

•        f.write(urllib.parse.quote(tobj['CKA_ISSUER']));
        f.write("\"\n")
        f.write("serial-number: \"");
  

•        f.write(urllib.quote(tobj['CKA_SERIAL_NUMBER']));
  

•        f.write(urllib.parse.quote(tobj['CKA_SERIAL_NUMBER']));
        f.write("\"\n")
        if (tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED'):
          f.write("x-distrusted: true\n")
        f.write("\n\n")
    f.close()
  

•    print " -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags)
  

•    print(" -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags))
  

diff --git a/config/rootfiles/common/ca-certificates b/config/rootfiles/common/ca-certificates index 087c3e450..06eb66f3b 100644 --- a/config/rootfiles/common/ca-certificates +++ b/config/rootfiles/common/ca-certificates @@ -1,4 +1,7 @@ +#etc/pki +#etc/pki/ca-trust +#etc/pki/ca-trust/source +etc/pki/ca-trust/source/ca-bundle.trust.p11-kit etc/ssl/cert.pem -#etc/ssl/certs etc/ssl/certs/ca-bundle.crt etc/ssl/certs/ca-bundle.trust.crt diff --git a/lfs/ca-certificates b/lfs/ca-certificates index f3c68a7c0..9e37687da 100644 --- a/lfs/ca-certificates +++ b/lfs/ca-certificates @@ -24,7 +24,7 @@

include Config

-VER = 20210611 +VER = 20210819

THISAPP = ca-certificates DIR_APP = $(DIR_SRC)/$(THISAPP) -- 2.33.0