Stop supporting the long-retired X9.31 standard

This isolates the three API functions from the library so they can be
easily removed and any attempt to use RSA_X931_PADDING mode will now
result in an error.

ok jsing

diff --git a/lib/libcrypto/rsa/rsa_eay.c b/lib/libcrypto/rsa/rsa_eay.c
index b307a8b..e65319b 100644 (file)
--- a/lib/libcrypto/rsa/rsa_eay.c
+++ b/lib/libcrypto/rsa/rsa_eay.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_eay.c,v 1.58 2023/04/05 11:31:38 tb Exp $ */
+/* $OpenBSD: rsa_eay.c,v 1.59 2023/04/15 18:48:52 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -382,14 +382,11 @@ RSA_eay_private_encrypt(int flen, const unsigned char *from, unsigned char *to,
        case RSA_PKCS1_PADDING:
                i = RSA_padding_add_PKCS1_type_1(buf, num, from, flen);
                break;
-       case RSA_X931_PADDING:
-               i = RSA_padding_add_X931(buf, num, from, flen);
-               break;
        case RSA_NO_PADDING:
                i = RSA_padding_add_none(buf, num, from, flen);
                break;
        default:
-               RSAerror(RSA_R_UNKNOWN_PADDING_TYPE);
+               RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
                goto err;
        }
        if (i <= 0)
@@ -449,14 +446,11 @@ RSA_eay_private_encrypt(int flen, const unsigned char *from, unsigned char *to,
                        goto err;
 
        if (padding == RSA_X931_PADDING) {
-               if (!BN_sub(f, rsa->n, ret))
-                       goto err;
-               if (BN_cmp(ret, f) > 0)
-                       res = f;
-               else
-                       res = ret;
-       } else
-               res = ret;
+               RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
+               goto err;
+       }
+
+       res = ret;
 
        /* put in leading 0 bytes if the number is less than the
         * length of the modulus */
@@ -667,9 +661,10 @@ RSA_eay_public_decrypt(int flen, const unsigned char *from, unsigned char *to,
            rsa->_method_mod_n))
                goto err;
 
-       if (padding == RSA_X931_PADDING && (ret->d[0] & 0xf) != 12)
-               if (!BN_sub(ret, rsa->n, ret))
-                       goto err;
+       if (padding == RSA_X931_PADDING) {
+               RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
+               goto err;
+       }
 
        p = buf;
        i = BN_bn2bin(ret, p);
@@ -678,9 +673,6 @@ RSA_eay_public_decrypt(int flen, const unsigned char *from, unsigned char *to,
        case RSA_PKCS1_PADDING:
                r = RSA_padding_check_PKCS1_type_1(to, num, buf, i, num);
                break;
-       case RSA_X931_PADDING:
-               r = RSA_padding_check_X931(to, num, buf, i, num);
-               break;
        case RSA_NO_PADDING:
                r = RSA_padding_check_none(to, num, buf, i, num);
                break;

diff --git a/lib/libcrypto/rsa/rsa_pmeth.c b/lib/libcrypto/rsa/rsa_pmeth.c
index 3747f1d..688c0d6 100644 (file)
--- a/lib/libcrypto/rsa/rsa_pmeth.c
+++ b/lib/libcrypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_pmeth.c,v 1.35 2023/03/06 08:31:34 tb Exp $ */
+/* $OpenBSD: rsa_pmeth.c,v 1.36 2023/04/15 18:48:52 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -187,7 +187,7 @@ static int
 pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
     const unsigned char *tbs, size_t tbslen)
 {
-       int ret;
+       int ret = -1;
        RSA_PKEY_CTX *rctx = ctx->data;
        RSA *rsa = ctx->pkey->pkey.rsa;
 
@@ -197,21 +197,11 @@ pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
                        return -1;
                }
 
-               if (rctx->pad_mode == RSA_X931_PADDING) {
-                       if ((size_t)EVP_PKEY_size(ctx->pkey) < tbslen + 1) {
-                               RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);
-                               return -1;
-                       }
-                       if (!setup_tbuf(rctx, ctx)) {
-                               RSAerror(ERR_R_MALLOC_FAILURE);
-                               return -1;
-                       }
-                       memcpy(rctx->tbuf, tbs, tbslen);
-                       rctx->tbuf[tbslen] =
-                           RSA_X931_hash_id(EVP_MD_type(rctx->md));
-                       ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
-                           rsa, RSA_X931_PADDING);
-               } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
+               if (rctx->pad_mode != RSA_PKCS1_PADDING &&
+                   rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
+                       return -1;
+
+               if (rctx->pad_mode == RSA_PKCS1_PADDING) {
                        unsigned int sltmp;
 
                        ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
@@ -227,8 +217,6 @@ pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
                                return -1;
                        ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
                            sig, rsa, RSA_NO_PADDING);
-               } else {
-                       return -1;
                }
        } else {
                ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
@@ -248,36 +236,16 @@ pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, unsigned char *rout, size_t *routlen,
        RSA_PKEY_CTX *rctx = ctx->data;
 
        if (rctx->md) {
-               if (rctx->pad_mode == RSA_X931_PADDING) {
-                       if (!setup_tbuf(rctx, ctx))
-                               return -1;
-                       ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
-                           ctx->pkey->pkey.rsa, RSA_X931_PADDING);
-                       if (ret < 1)
-                               return 0;
-                       ret--;
-                       if (rctx->tbuf[ret] !=
-                           RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
-                               RSAerror(RSA_R_ALGORITHM_MISMATCH);
-                               return 0;
-                       }
-                       if (ret != EVP_MD_size(rctx->md)) {
-                               RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
-                               return 0;
-                       }
-                       if (rout)
-                               memcpy(rout, rctx->tbuf, ret);
-               } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
-                       size_t sltmp;
+               size_t sltmp;
 
-                       ret = int_rsa_verify(EVP_MD_type(rctx->md), NULL, 0,
-                           rout, &sltmp, sig, siglen, ctx->pkey->pkey.rsa);
-                       if (ret <= 0)
-                               return 0;
-                       ret = sltmp;
-               } else {
+               if (rctx->pad_mode != RSA_PKCS1_PADDING)
                        return -1;
-               }
+
+               ret = int_rsa_verify(EVP_MD_type(rctx->md), NULL, 0,
+                   rout, &sltmp, sig, siglen, ctx->pkey->pkey.rsa);
+               if (ret <= 0)
+                       return 0;
+               ret = sltmp;
        } else {
                ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
                    rctx->pad_mode);
@@ -295,6 +263,7 @@ pkey_rsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
        RSA_PKEY_CTX *rctx = ctx->data;
        RSA *rsa = ctx->pkey->pkey.rsa;
        size_t rslen;
+       int ret;
 
        if (rctx->md) {
                if (rctx->pad_mode == RSA_PKCS1_PADDING)
@@ -304,30 +273,22 @@ pkey_rsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
                        RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
                        return -1;
                }
-               if (rctx->pad_mode == RSA_X931_PADDING) {
-                       if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig,
-                           siglen) <= 0)
-                               return 0;
-               } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
-                       int ret;
 
-                       if (!setup_tbuf(rctx, ctx))
-                               return -1;
-                       ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
-                           rsa, RSA_NO_PADDING);
-                       if (ret <= 0)
-                               return 0;
-                       ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md,
-                           rctx->mgf1md, rctx->tbuf, rctx->saltlen);
-                       if (ret <= 0)
-                               return 0;
-                       return 1;
-               } else {
+               if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
                        return -1;
-               }
-       } else {
-               int ret;
 
+               if (!setup_tbuf(rctx, ctx))
+                       return -1;
+               ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
+                   rsa, RSA_NO_PADDING);
+               if (ret <= 0)
+                       return 0;
+               ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md,
+                   rctx->mgf1md, rctx->tbuf, rctx->saltlen);
+               if (ret <= 0)
+                       return 0;
+               return 1;
+       } else {
                if (!setup_tbuf(rctx, ctx))
                        return -1;
 
@@ -404,34 +365,27 @@ check_padding_md(const EVP_MD *md, int padding)
        if (md == NULL)
                return 1;
 
-       if (padding == RSA_NO_PADDING) {
-               RSAerror(RSA_R_INVALID_PADDING_MODE);
+       if (padding == RSA_NO_PADDING || padding == RSA_X931_PADDING) {
+               RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
                return 0;
        }
 
-       if (padding == RSA_X931_PADDING) {
-               if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
-                       RSAerror(RSA_R_INVALID_X931_DIGEST);
-                       return 0;
-               }
-       } else {
-               /* List of all supported RSA digests. */
-               switch(EVP_MD_type(md)) {
-               case NID_sha1:
-               case NID_sha224:
-               case NID_sha256:
-               case NID_sha384:
-               case NID_sha512:
-               case NID_md5:
-               case NID_md5_sha1:
-               case NID_md4:
-               case NID_ripemd160:
-                       return 1;
+       /* List of all supported RSA digests. */
+       switch(EVP_MD_type(md)) {
+       case NID_sha1:
+       case NID_sha224:
+       case NID_sha256:
+       case NID_sha384:
+       case NID_sha512:
+       case NID_md5:
+       case NID_md5_sha1:
+       case NID_md4:
+       case NID_ripemd160:
+               return 1;
 
-               default:
-                       RSAerror(RSA_R_INVALID_DIGEST);
-                       return 0;
-               }
+       default:
+               RSAerror(RSA_R_INVALID_DIGEST);
+               return 0;
        }
 
        return 1;
@@ -637,8 +591,6 @@ pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
                        pm = RSA_PKCS1_OAEP_PADDING;
                else if (!strcmp(value, "oaep"))
                        pm = RSA_PKCS1_OAEP_PADDING;
-               else if (!strcmp(value, "x931"))
-                       pm = RSA_X931_PADDING;
                else if (!strcmp(value, "pss"))
                        pm = RSA_PKCS1_PSS_PADDING;
                else {