From 098374438a0443dba12b9b28a87b03a5af793a55 Mon Sep 17 00:00:00 2001 From: tb Date: Thu, 17 Oct 2024 06:19:06 +0000 Subject: [PATCH] libssl: rework cert signature security level This switches to using the X509_get_signature_info() API instead of hand rolling a part of it. This is slightly tangly since the security level API is strange. In particular, some failures are passed to the security level callback so that applications can override them. This makes the security level API handle RSA-PSS and EdDSA certificates correctly and the handshake with such can progress a bit further. Of note, we check that the certs are actually suitable for use in TLS per RFC 8446 contrary to what OpenSSL does. ok beck jsing --- lib/libssl/ssl_seclevel.c | 52 +++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/lib/libssl/ssl_seclevel.c b/lib/libssl/ssl_seclevel.c index 6a5d16bfaa2..1448368e719 100644 --- a/lib/libssl/ssl_seclevel.c +++ b/lib/libssl/ssl_seclevel.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ssl_seclevel.c,v 1.28 2024/05/09 07:12:03 tb Exp $ */ +/* $OpenBSD: ssl_seclevel.c,v 1.29 2024/10/17 06:19:06 tb Exp $ */ /* * Copyright (c) 2020-2022 Theo Buehler * @@ -331,45 +331,49 @@ ssl_security_cert_key(const SSL_CTX *ctx, const SSL *ssl, X509 *x509, int secop) } static int -ssl_cert_signature_md_nid(X509 *x509) +ssl_security_cert_sig_security_bits(X509 *x509, int *out_md_nid) { - int md_nid, signature_nid; + int pkey_nid, security_bits; + uint32_t flags; - if ((signature_nid = X509_get_signature_nid(x509)) == NID_undef) - return NID_undef; + *out_md_nid = NID_undef; - if (!OBJ_find_sigid_algs(signature_nid, &md_nid, NULL)) - return NID_undef; - - return md_nid; -} - -static int -ssl_cert_md_nid_security_bits(int md_nid) -{ - const EVP_MD *md; - - if (md_nid == NID_undef) + /* + * Returning -1 security bits makes the default security callback fail + * to match bonkers behavior in OpenSSL. This in turn lets a security + * callback override such failures. + */ + if (!X509_get_signature_info(x509, out_md_nid, &pkey_nid, &security_bits, + &flags)) return -1; - - if ((md = EVP_get_digestbynid(md_nid)) == NULL) + /* + * OpenSSL doesn't check flags. Test RSA-PSS certs we were provided have + * a salt length distinct from hash length and thus fail this check. + */ + if ((flags & X509_SIG_INFO_TLS) == 0) return -1; - /* Assume 4 bits of collision resistance for each hash octet. */ - return EVP_MD_size(md) * 4; + /* Weird OpenSSL behavior only relevant for EdDSA certs in LibreSSL. */ + if (*out_md_nid == NID_undef) + *out_md_nid = pkey_nid; + + return security_bits; } static int ssl_security_cert_sig(const SSL_CTX *ctx, const SSL *ssl, X509 *x509, int secop) { - int md_nid, security_bits; + int md_nid = NID_undef, security_bits = -1; /* Don't check signature if self signed. */ if ((X509_get_extension_flags(x509) & EXFLAG_SS) != 0) return 1; - md_nid = ssl_cert_signature_md_nid(x509); - security_bits = ssl_cert_md_nid_security_bits(md_nid); + /* + * The default security callback fails on -1 security bits. It ignores + * the md_nid (aka version) argument we pass from here. + */ + security_bits = ssl_security_cert_sig_security_bits(x509, &md_nid); if (ssl != NULL) return ssl_security(ssl, secop, security_bits, md_nid, x509); -- 2.20.1