Add initial regress for RSA padding.

diff --git a/regress/lib/libcrypto/rsa/Makefile b/regress/lib/libcrypto/rsa/Makefile
index 53065e6..494d40d 100644 (file)
--- a/regress/lib/libcrypto/rsa/Makefile
+++ b/regress/lib/libcrypto/rsa/Makefile
@@ -1,6 +1,7 @@
-#      $OpenBSD: Makefile,v 1.3 2024/03/30 00:34:40 jsing Exp $
+#      $OpenBSD: Makefile,v 1.4 2024/03/30 00:36:14 jsing Exp $
 
-PROG=          rsa_test
+PROGS=         rsa_test \
+               rsa_padding_test
 LDADD=         -lcrypto
 DPADD=         ${LIBCRYPTO}
 WARNINGS=      Yes

diff --git a/regress/lib/libcrypto/rsa/rsa_padding_test.c b/regress/lib/libcrypto/rsa/rsa_padding_test.c
new file mode 100644 (file)
index 0000000..64357bf
--- /dev/null
+++ b/regress/lib/libcrypto/rsa/rsa_padding_test.c
@@ -0,0 +1,325 @@
+/*     $OpenBSD: rsa_padding_test.c,v 1.1 2024/03/30 00:36:14 jsing Exp $      */
+/*
+ * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+
+#if 0
+static void
+hexdump(const unsigned char *buf, size_t len)
+{
+       size_t i;
+
+       for (i = 1; i <= len; i++)
+               fprintf(stderr, " 0x%02hhx,%s", buf[i - 1], i % 8 ? "" : "\n");
+
+       fprintf(stderr, "\n");
+}
+#endif
+
+struct pkcs1_test {
+       uint8_t in[128];
+       size_t in_len;
+       int want;
+       int want_error;
+};
+
+static const struct pkcs1_test pkcs1_type1_tests[] = {
+       {
+               .in = {
+                       0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
+                       0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                       0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+               },
+               .in_len = 32,
+               .want = 19,
+       },
+       {
+               .in = {
+                       0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00,
+               },
+               .in_len = 11,
+               .want = 0,
+       },
+       {
+               .in = {
+                       0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00, 0xff,
+               },
+               .in_len = 12,
+               .want = 1,
+       },
+       {
+               /* Insufficient padding bytes (< 8). */
+               .in = {
+                       0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0x00, 0xff, 0xff,
+               },
+               .in_len = 12,
+               .want = -1,
+               .want_error = RSA_R_BAD_PAD_BYTE_COUNT,
+       },
+       {
+               /* Incorrect padding type (0x00). */
+               .in = {
+                       0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00, 0xff,
+               },
+               .in_len = 12,
+               .want = -1,
+               .want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
+       },
+       {
+               /* Incorrect padding type (0x02). */
+               .in = {
+                       0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00, 0xff,
+               },
+               .in_len = 12,
+               .want = -1,
+               .want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
+       },
+       {
+               /* Non-padding byte before end of padding marker. */
+               .in = {
+                       0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xfe, 0x00, 0xff,
+               },
+               .in_len = 12,
+               .want = -1,
+               .want_error = RSA_R_BAD_FIXED_HEADER_DECRYPT,
+       },
+       {
+               /* No end of padding marker. */
+               .in = {
+                       0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+               },
+               .in_len = 32,
+               .want = -1,
+               .want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
+       },
+};
+
+#define N_PKCS1_TYPE1_TESTS \
+       (sizeof(pkcs1_type1_tests) / sizeof(pkcs1_type1_tests[0]))
+
+static int
+test_pkcs1_type1(void)
+{
+       const struct pkcs1_test *pt;
+       uint8_t buf[32], in[19], out[512];
+       int pad_len;
+       long err;
+       size_t i;
+       int failed = 1;
+
+       memset(in, 0x6f, sizeof(in));
+
+       if (!RSA_padding_add_PKCS1_type_1(buf, sizeof(buf), in, sizeof(in))) {
+               fprintf(stderr, "FAIL: failed to add PKCS1 type 1 padding\n");
+               goto failed;
+       }
+
+       pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
+           buf + 1, sizeof(buf) - 1, sizeof(buf));
+       if (pad_len != sizeof(in)) {
+               fprintf(stderr, "FAIL: failed to check PKCS1 type 1 padding\n");
+               ERR_print_errors_fp(stderr);
+               goto failed;
+       }
+
+       for (i = 0; i < N_PKCS1_TYPE1_TESTS; i++) {
+               pt = &pkcs1_type1_tests[i];
+
+               ERR_clear_error();
+
+               pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
+                   pt->in + 1, pt->in_len - 1, pt->in_len);
+
+               if (pad_len != pt->want) {
+                       fprintf(stderr, "FAIL: test %zu - failed to check "
+                           "PKCS1 type 1 padding (%d != %d)\n", i, pad_len,
+                           pt->want);
+                       ERR_print_errors_fp(stderr);
+                       goto failed;
+               }
+
+               err = ERR_peek_error();
+               if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
+                       fprintf(stderr, "FAIL: test %zu - PKCS1 type 1 padding "
+                           "check failed with error reason %i, want %i\n",
+                           i, ERR_GET_REASON(err), pt->want_error);
+                       ERR_print_errors_fp(stderr);
+                       goto failed;
+               }
+       }
+
+       failed = 0;
+
+ failed:
+       return failed;
+}
+
+static const struct pkcs1_test pkcs1_type2_tests[] = {
+       {
+               .in = {
+                       0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
+                       0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                       0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+               },
+               .in_len = 32,
+               .want = 19,
+       },
+       {
+               .in = {
+                       0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00,
+               },
+               .in_len = 11,
+               .want = 0,
+       },
+       {
+               .in = {
+                       0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00, 0xff,
+               },
+               .in_len = 12,
+               .want = 1,
+       },
+       {
+               /* Insufficient padding bytes (< 8). */
+               .in = {
+                       0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0x00, 0xff, 0xff,
+               },
+               .in_len = 12,
+               .want = -1,
+               .want_error = RSA_R_BAD_PAD_BYTE_COUNT,
+       },
+       {
+               /* Incorrect padding type (0x00). */
+               .in = {
+                       0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00, 0xff,
+               },
+               .in_len = 12,
+               .want = -1,
+               .want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
+       },
+       {
+               /* Incorrect padding type (0x01). */
+               .in = {
+                       0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0x00, 0xff,
+               },
+               .in_len = 12,
+               .want = -1,
+               .want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
+       },
+       {
+               /* No end of padding marker. */
+               .in = {
+                       0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                       0xff, 0xff, 0xff, 0xff, 0xff, 0x6f, 0x6f, 0x6f,
+                       0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                       0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+               },
+               .in_len = 32,
+               .want = -1,
+               .want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
+       },
+};
+
+#define N_PKCS1_TYPE2_TESTS \
+       (sizeof(pkcs1_type2_tests) / sizeof(pkcs1_type2_tests[0]))
+
+static int
+test_pkcs1_type2(void)
+{
+       const struct pkcs1_test *pt;
+       uint8_t buf[32], in[19], out[32];
+       int pad_len;
+       long err;
+       size_t i;
+       int failed = 1;
+
+       memset(in, 0x6f, sizeof(in));
+
+       if (!RSA_padding_add_PKCS1_type_2(buf, sizeof(buf), in, sizeof(in))) {
+               fprintf(stderr, "FAIL: failed to add PKCS1 type 2 padding\n");
+               goto failed;
+       }
+
+       pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
+           buf + 1, sizeof(buf) - 1, sizeof(out));
+       if (pad_len != sizeof(in)) {
+               fprintf(stderr, "FAIL: failed to check PKCS1 type 2 padding\n");
+               ERR_print_errors_fp(stderr);
+               goto failed;
+       }
+
+       for (i = 0; i < N_PKCS1_TYPE2_TESTS; i++) {
+               pt = &pkcs1_type2_tests[i];
+
+               ERR_clear_error();
+
+               pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
+                   pt->in + 1, pt->in_len - 1, pt->in_len);
+
+               if (pad_len != pt->want) {
+                       fprintf(stderr, "FAIL: test %zu - failed to check "
+                           "PKCS1 type 2 padding (%d != %d)\n", i, pad_len,
+                           pt->want);
+                       ERR_print_errors_fp(stderr);
+                       goto failed;
+               }
+
+               err = ERR_peek_error();
+               if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
+                       fprintf(stderr, "FAIL: test %zu - PKCS1 type 2 padding "
+                           "check failed with error reason %i, want %i\n",
+                           i, ERR_GET_REASON(err), pt->want_error);
+                       ERR_print_errors_fp(stderr);
+                       goto failed;
+               }
+       }
+
+       failed = 0;
+
+ failed:
+       return failed;
+}
+
+int
+main(int argc, char **argv)
+{
+       int failed = 0;
+
+       failed |= test_pkcs1_type1();
+       failed |= test_pkcs1_type2();
+
+       return failed;
+}