From: tb <tb@openbsd.org>
Date: Thu, 28 Dec 2023 21:59:07 +0000 (+0000)
Subject: Rework pkey_rsa_keygen()
X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=0f0ee19aedf3f0c3ca258deff905e047cf2e8e96;p=openbsd

Rework pkey_rsa_keygen()

As usual, make the function single exit. Initialize the pkey callback
pointer and the BN_GENCB on the stack at the top rather than relying
on the weird trans_cb() in evp_pkey_set_cb_translate() to do so.
Greatly simplify the control flow and add missing error checks.

ok jsing
---

diff --git a/lib/libcrypto/rsa/rsa_pmeth.c b/lib/libcrypto/rsa/rsa_pmeth.c
index cb82b0908fd..9be90796131 100644
--- a/lib/libcrypto/rsa/rsa_pmeth.c
+++ b/lib/libcrypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_pmeth.c,v 1.39 2023/07/08 12:26:45 beck Exp $ */
+/* $OpenBSD: rsa_pmeth.c,v 1.40 2023/12/28 21:59:07 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -756,32 +756,36 @@ pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 {
 	RSA *rsa = NULL;
 	RSA_PKEY_CTX *rctx = ctx->data;
-	BN_GENCB *pcb, cb;
-	int ret;
+	BN_GENCB *pcb = NULL;
+	BN_GENCB cb = {0};
+	int ret = 0;
 
 	if (rctx->pub_exp == NULL) {
 		if ((rctx->pub_exp = BN_new()) == NULL)
-			return 0;
+			goto err;
 		if (!BN_set_word(rctx->pub_exp, RSA_F4))
-			return 0;
+			goto err;
 	}
+
 	if ((rsa = RSA_new()) == NULL)
-		return 0;
+		goto err;
 	if (ctx->pkey_gencb != NULL) {
 		pcb = &cb;
 		evp_pkey_set_cb_translate(pcb, ctx);
-	} else {
-		pcb = NULL;
 	}
-	ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
-	if (ret > 0 && !rsa_set_pss_param(rsa, ctx)) {
-		RSA_free(rsa);
-		return 0;
-	}
-	if (ret > 0)
-		EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa);
-	else
-		RSA_free(rsa);
+	if (!RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb))
+		goto err;
+	if (!rsa_set_pss_param(rsa, ctx))
+		goto err;
+	if (!EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa))
+		goto err;
+	rsa = NULL;
+
+	ret = 1;
+
+ err:
+	RSA_free(rsa);
+
 	return ret;
 }