Convert relayd for opaque RSA_METHOD

This is a mostly mechanical diff which will hopefully be superseded
soon by work in libtls.

ok jsing

diff --git a/usr.sbin/relayd/ca.c b/usr.sbin/relayd/ca.c
index 88bdc1e..ec24eff 100644 (file)
--- a/usr.sbin/relayd/ca.c
+++ b/usr.sbin/relayd/ca.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ca.c,v 1.37 2021/12/08 03:40:44 tb Exp $      */
+/*     $OpenBSD: ca.c,v 1.38 2022/01/11 19:06:23 tb Exp $      */
 
 /*
  * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
@@ -220,10 +220,10 @@ ca_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
                config_getreset(env, imsg);
                break;
        default:
-               return (-1);
+               return -1;
        }
 
-       return (0);
+       return 0;
 }
 
 int
@@ -290,10 +290,10 @@ ca_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg)
                RSA_free(rsa);
                break;
        default:
-               return (-1);
+               return -1;
        }
 
-       return (0);
+       return 0;
 }
 
 /*
@@ -301,23 +301,7 @@ ca_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg)
  */
 
 const RSA_METHOD *rsa_default = NULL;
-
-static RSA_METHOD rsae_method = {
-       "RSA privsep engine",
-       rsae_pub_enc,
-       rsae_pub_dec,
-       rsae_priv_enc,
-       rsae_priv_dec,
-       rsae_mod_exp,
-       rsae_bn_mod_exp,
-       rsae_init,
-       rsae_finish,
-       0,
-       NULL,
-       rsae_sign,
-       rsae_verify,
-       rsae_keygen
-};
+static RSA_METHOD *rsae_method;
 
 static int
 rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
@@ -337,7 +321,7 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
        static u_int     seq = 0;
 
        if ((hash = RSA_get_ex_data(rsa, 0)) == NULL)
-               return (0);
+               return 0;
 
        iev = proc_iev(ps, PROC_CA, ps->ps_instance);
        ibuf = &iev->ibuf;
@@ -378,7 +362,7 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
                            __func__,
                            cmd == IMSG_CA_PRIVENC ? "enc" : "dec",
                            cko.cko_cookie);
-                       return (-1);
+                       return -1;
                default:
                        break;
                }
@@ -426,44 +410,42 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
        }
        imsg_event_add(iev);
 
-       return (ret);
+       return ret;
 }
 
 int
 rsae_pub_enc(int flen,const u_char *from, u_char *to, RSA *rsa,int padding)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsa_default->rsa_pub_enc(flen, from, to, rsa, padding));
+       return RSA_meth_get_pub_enc(rsa_default)(flen, from, to, rsa, padding);
 }
 
 int
 rsae_pub_dec(int flen,const u_char *from, u_char *to, RSA *rsa,int padding)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsa_default->rsa_pub_dec(flen, from, to, rsa, padding));
+       return RSA_meth_get_pub_dec(rsa_default)(flen, from, to, rsa, padding);
 }
 
 int
 rsae_priv_enc(int flen, const u_char *from, u_char *to, RSA *rsa, int padding)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsae_send_imsg(flen, from, to, rsa, padding,
-           IMSG_CA_PRIVENC));
+       return rsae_send_imsg(flen, from, to, rsa, padding, IMSG_CA_PRIVENC);
 }
 
 int
 rsae_priv_dec(int flen, const u_char *from, u_char *to, RSA *rsa, int padding)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsae_send_imsg(flen, from, to, rsa, padding,
-           IMSG_CA_PRIVDEC));
+       return rsae_send_imsg(flen, from, to, rsa, padding, IMSG_CA_PRIVDEC);
 }
 
 int
 rsae_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsa_default->rsa_mod_exp(r0, I, rsa, ctx));
+       return RSA_meth_get_mod_exp(rsa_default)(r0, I, rsa, ctx);
 }
 
 int
@@ -471,25 +453,25 @@ rsae_bn_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
     const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsa_default->bn_mod_exp(r, a, p, m, ctx, m_ctx));
+       return RSA_meth_get_bn_mod_exp(rsa_default)(r, a, p, m, ctx, m_ctx);
 }
 
 int
 rsae_init(RSA *rsa)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       if (rsa_default->init == NULL)
-               return (1);
-       return (rsa_default->init(rsa));
+       if (RSA_meth_get_init(rsa_default) == NULL)
+               return 1;
+       return RSA_meth_get_init(rsa_default)(rsa);
 }
 
 int
 rsae_finish(RSA *rsa)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       if (rsa_default->finish == NULL)
-               return (1);
-       return (rsa_default->finish(rsa));
+       if (RSA_meth_get_finish(rsa_default) == NULL)
+               return 1;
+       return RSA_meth_get_finish(rsa_default)(rsa);
 }
 
 int
@@ -497,8 +479,8 @@ rsae_sign(int type, const u_char *m, u_int m_length, u_char *sigret,
     u_int *siglen, const RSA *rsa)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsa_default->rsa_sign(type, m, m_length,
-           sigret, siglen, rsa));
+       return RSA_meth_get_sign(rsa_default)(type, m, m_length,
+           sigret, siglen, rsa);
 }
 
 int
@@ -506,15 +488,15 @@ rsae_verify(int dtype, const u_char *m, u_int m_length, const u_char *sigbuf,
     u_int siglen, const RSA *rsa)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsa_default->rsa_verify(dtype, m, m_length,
-           sigbuf, siglen, rsa));
+       return RSA_meth_get_verify(rsa_default)(dtype, m, m_length,
+           sigbuf, siglen, rsa);
 }
 
 int
 rsae_keygen(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
 {
        DPRINTF("%s:%d", __func__, __LINE__);
-       return (rsa_default->rsa_keygen(rsa, bits, e, cb));
+       return RSA_meth_get_keygen(rsa_default)(rsa, bits, e, cb);
 }
 
 void
@@ -529,12 +511,27 @@ ca_engine_init(struct relayd *x_env)
        if (rsa_default != NULL)
                return;
 
+       if ((rsae_method = RSA_meth_new("RSA privsep engine", 0)) == NULL)
+               goto fail;
+
+       RSA_meth_set_pub_enc(rsae_method, rsae_pub_enc);
+       RSA_meth_set_pub_dec(rsae_method, rsae_pub_dec);
+       RSA_meth_set_priv_enc(rsae_method, rsae_priv_enc);
+       RSA_meth_set_priv_dec(rsae_method, rsae_priv_dec);
+       RSA_meth_set_mod_exp(rsae_method, rsae_mod_exp);
+       RSA_meth_set_bn_mod_exp(rsae_method, rsae_bn_mod_exp);
+       RSA_meth_set_init(rsae_method, rsae_init);
+       RSA_meth_set_finish(rsae_method, rsae_finish);
+       RSA_meth_set_sign(rsae_method, rsae_sign);
+       RSA_meth_set_verify(rsae_method, rsae_verify);
+       RSA_meth_set_keygen(rsae_method, rsae_keygen);
+
        if ((e = ENGINE_get_default_RSA()) == NULL) {
                if ((e = ENGINE_new()) == NULL) {
                        errstr = "ENGINE_new";
                        goto fail;
                }
-               if (!ENGINE_set_name(e, rsae_method.name)) {
+               if (!ENGINE_set_name(e, RSA_meth_get0_name(rsae_method))) {
                        errstr = "ENGINE_set_name";
                        goto fail;
                }
@@ -552,20 +549,21 @@ ca_engine_init(struct relayd *x_env)
 
        log_debug("%s: using %s", __func__, name);
 
-       if (rsa_default->flags & RSA_FLAG_SIGN_VER)
+       if (RSA_meth_get_flags(rsa_default) & RSA_FLAG_SIGN_VER)
                fatalx("unsupported RSA engine");
 
-       if (rsa_default->rsa_mod_exp == NULL)
-               rsae_method.rsa_mod_exp = NULL;
-       if (rsa_default->bn_mod_exp == NULL)
-               rsae_method.bn_mod_exp = NULL;
-       if (rsa_default->rsa_keygen == NULL)
-               rsae_method.rsa_keygen = NULL;
-       rsae_method.flags = rsa_default->flags |
-           RSA_METHOD_FLAG_NO_CHECK;
-       rsae_method.app_data = rsa_default->app_data;
-
-       if (!ENGINE_set_RSA(e, &rsae_method)) {
+       if (RSA_meth_get_mod_exp(rsa_default) == NULL)
+               RSA_meth_set_mod_exp(rsae_method, NULL);
+       if (RSA_meth_get_bn_mod_exp(rsa_default) == NULL)
+               RSA_meth_set_bn_mod_exp(rsae_method, NULL);
+       if (RSA_meth_get_keygen(rsa_default) == NULL)
+               RSA_meth_set_keygen(rsae_method, NULL);
+       RSA_meth_set_flags(rsae_method,
+           RSA_meth_get_flags(rsa_default) | RSA_METHOD_FLAG_NO_CHECK);
+       RSA_meth_set0_app_data(rsae_method,
+           RSA_meth_get0_app_data(rsa_default));
+
+       if (!ENGINE_set_RSA(e, rsae_method)) {
                errstr = "ENGINE_set_RSA";
                goto fail;
        }
@@ -577,5 +575,6 @@ ca_engine_init(struct relayd *x_env)
        return;
 
  fail:
+       RSA_meth_free(rsae_method);
        fatalx("%s: %s", __func__, errstr);
 }