From b7ca3793f1686a772c74c2d204de99dda1e65cf4 Mon Sep 17 00:00:00 2001 From: tb Date: Wed, 8 Nov 2023 16:02:41 +0000 Subject: [PATCH] Split OAEP padding handling into a helper function This matches what is done for PKCS#1 1.5 and PSS. This function needs a lot of work still, but it's easier to do that without having to tiptoe around a lot of other garbage. ok jsing --- lib/libcrypto/rsa/rsa_ameth.c | 112 ++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/lib/libcrypto/rsa/rsa_ameth.c b/lib/libcrypto/rsa/rsa_ameth.c index 6fb4403231a..43e66ce02a5 100644 --- a/lib/libcrypto/rsa/rsa_ameth.c +++ b/lib/libcrypto/rsa/rsa_ameth.c @@ -1,4 +1,4 @@ -/* $OpenBSD: rsa_ameth.c,v 1.42 2023/11/07 22:35:03 tb Exp $ */ +/* $OpenBSD: rsa_ameth.c,v 1.43 2023/11/08 16:02:41 tb Exp $ */ /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL * project 2006. */ @@ -908,6 +908,60 @@ rsa_alg_set_pss_padding(X509_ALGOR *alg, EVP_PKEY_CTX *pkey_ctx) } #ifndef OPENSSL_NO_CMS +static int +rsa_alg_set_oaep_padding(X509_ALGOR *alg, EVP_PKEY_CTX *pkctx) +{ + const EVP_MD *md, *mgf1md; + RSA_OAEP_PARAMS *oaep = NULL; + ASN1_STRING *os = NULL; + int rv = 0, labellen; + unsigned char *label; + + if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0) + goto err; + if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0) + goto err; + labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label); + if (labellen < 0) + goto err; + + if ((oaep = RSA_OAEP_PARAMS_new()) == NULL) + goto err; + + if (!rsa_md_to_algor(md, &oaep->hashFunc)) + goto err; + if (!rsa_mgf1md_to_maskGenAlgorithm(mgf1md, &oaep->maskGenFunc)) + goto err; + + /* XXX - why do we not set oaep->maskHash here? */ + + if (labellen > 0) { + ASN1_OCTET_STRING *los; + oaep->pSourceFunc = X509_ALGOR_new(); + if (oaep->pSourceFunc == NULL) + goto err; + los = ASN1_OCTET_STRING_new(); + if (los == NULL) + goto err; + if (!ASN1_OCTET_STRING_set(los, label, labellen)) { + ASN1_OCTET_STRING_free(los); + goto err; + } + X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified), + V_ASN1_OCTET_STRING, los); + } + /* create string with pss parameter encoding. */ + if (!ASN1_item_pack(oaep, &RSA_OAEP_PARAMS_it, &os)) + goto err; + X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os); + os = NULL; + rv = 1; + err: + RSA_OAEP_PARAMS_free(oaep); + ASN1_STRING_free(os); + return rv; +} + static int rsa_cms_sign(CMS_SignerInfo *si) { @@ -1057,13 +1111,9 @@ rsa_cms_decrypt(CMS_RecipientInfo *ri) static int rsa_cms_encrypt(CMS_RecipientInfo *ri) { - const EVP_MD *md, *mgf1md; - RSA_OAEP_PARAMS *oaep = NULL; - ASN1_STRING *os = NULL; X509_ALGOR *alg; EVP_PKEY_CTX *pkctx; - int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen; - unsigned char *label; + int pad_mode = RSA_PKCS1_PADDING; if ((pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri)) != NULL) { if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0) @@ -1074,54 +1124,10 @@ rsa_cms_encrypt(CMS_RecipientInfo *ri) return 0; if (pad_mode == RSA_PKCS1_PADDING) return rsa_alg_set_pkcs1_padding(alg); + if (pad_mode == RSA_PKCS1_OAEP_PADDING) + return rsa_alg_set_oaep_padding(alg, pkctx); - /* Not supported */ - if (pad_mode != RSA_PKCS1_OAEP_PADDING) - return 0; - - if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0) - goto err; - if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0) - goto err; - labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label); - if (labellen < 0) - goto err; - - if ((oaep = RSA_OAEP_PARAMS_new()) == NULL) - goto err; - - if (!rsa_md_to_algor(md, &oaep->hashFunc)) - goto err; - if (!rsa_mgf1md_to_maskGenAlgorithm(mgf1md, &oaep->maskGenFunc)) - goto err; - - /* XXX - why do we not set oaep->maskHash here? */ - - if (labellen > 0) { - ASN1_OCTET_STRING *los; - oaep->pSourceFunc = X509_ALGOR_new(); - if (oaep->pSourceFunc == NULL) - goto err; - los = ASN1_OCTET_STRING_new(); - if (los == NULL) - goto err; - if (!ASN1_OCTET_STRING_set(los, label, labellen)) { - ASN1_OCTET_STRING_free(los); - goto err; - } - X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified), - V_ASN1_OCTET_STRING, los); - } - /* create string with pss parameter encoding. */ - if (!ASN1_item_pack(oaep, &RSA_OAEP_PARAMS_it, &os)) - goto err; - X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os); - os = NULL; - rv = 1; - err: - RSA_OAEP_PARAMS_free(oaep); - ASN1_STRING_free(os); - return rv; + return 0; } #endif -- 2.20.1