Rework pkey_rsa_keygen()

As usual, make the function single exit. Initialize the pkey callback
pointer and the BN_GENCB on the stack at the top rather than relying
on the weird trans_cb() in evp_pkey_set_cb_translate() to do so.
Greatly simplify the control flow and add missing error checks.

ok jsing

diff --git a/lib/libcrypto/rsa/rsa_pmeth.c b/lib/libcrypto/rsa/rsa_pmeth.c
index cb82b09..9be9079 100644 (file)
--- a/lib/libcrypto/rsa/rsa_pmeth.c
+++ b/lib/libcrypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_pmeth.c,v 1.39 2023/07/08 12:26:45 beck Exp $ */
+/* $OpenBSD: rsa_pmeth.c,v 1.40 2023/12/28 21:59:07 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -756,32 +756,36 @@ pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 {
        RSA *rsa = NULL;
        RSA_PKEY_CTX *rctx = ctx->data;
-       BN_GENCB *pcb, cb;
-       int ret;
+       BN_GENCB *pcb = NULL;
+       BN_GENCB cb = {0};
+       int ret = 0;
 
        if (rctx->pub_exp == NULL) {
                if ((rctx->pub_exp = BN_new()) == NULL)
-                       return 0;
+                       goto err;
                if (!BN_set_word(rctx->pub_exp, RSA_F4))
-                       return 0;
+                       goto err;
        }
+
        if ((rsa = RSA_new()) == NULL)
-               return 0;
+               goto err;
        if (ctx->pkey_gencb != NULL) {
                pcb = &cb;
                evp_pkey_set_cb_translate(pcb, ctx);
-       } else {
-               pcb = NULL;
        }
-       ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
-       if (ret > 0 && !rsa_set_pss_param(rsa, ctx)) {
-               RSA_free(rsa);
-               return 0;
-       }
-       if (ret > 0)
-               EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa);
-       else
-               RSA_free(rsa);
+       if (!RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb))
+               goto err;
+       if (!rsa_set_pss_param(rsa, ctx))
+               goto err;
+       if (!EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa))
+               goto err;
+       rsa = NULL;
+
+       ret = 1;
+
+ err:
+       RSA_free(rsa);
+
        return ret;
 }