Avoid polluting the error stack when printing certificates

For a certificate serial number between LONG_MAX and ULONG_MAX, the call to
ASN1_INTEGER_get() fails and leaves an error on the stack because the check
bs->length <= sizeof(long) doesn't quite do what it's supposed to do (bs is
probably for bitstring, although the more common reading would be adequate,
too.)

Fix this by checking for non-negativity and using ASN1_INTEGER_get_uint64()
and add a lengthy comment to explain the nonsense per beck's request.

discussed with jsing
ok beck

diff --git a/lib/libcrypto/asn1/t_x509.c b/lib/libcrypto/asn1/t_x509.c
index 5e753f3..75bae52 100644 (file)
--- a/lib/libcrypto/asn1/t_x509.c
+++ b/lib/libcrypto/asn1/t_x509.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: t_x509.c,v 1.45 2024/04/09 13:55:02 beck Exp $ */
+/* $OpenBSD: t_x509.c,v 1.46 2024/08/28 06:17:06 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -155,8 +156,21 @@ X509_print_ex(BIO *bp, X509 *x, unsigned long nmflags, unsigned long cflag)
 
                bs = X509_get_serialNumber(x);
                l = -1;
-               if (bs->length <= (int)sizeof(long))
-                       l = ASN1_INTEGER_get(bs);
+
+               /*
+                * For historical reasons, non-negative serial numbers are
+                * printed in decimal as long as they fit into a long. Using
+                * ASN1_INTEGER_get_uint64() avoids an error on the stack for
+                * numbers between LONG_MAX and ULONG_MAX. Otherwise fall back
+                * to hexadecimal, also for numbers that are non-conformant
+                * (negative or larger than 2^159 - 1).
+                */
+               if (bs->length <= sizeof(long) && bs->type == V_ASN1_INTEGER) {
+                       uint64_t u64;
+
+                       if (ASN1_INTEGER_get_uint64(&u64, bs) && u64 <= LONG_MAX)
+                               l = (long)u64;
+               }
                if (l >= 0) {
                        if (BIO_printf(bp, " %ld (0x%lx)\n", l, l) <= 0)
                                goto err;