rpki-client: fix seqnum upper bound

An ASN1_INTEGER doesn't contain the ASN.1 encoding in its data. It contains
a BIGNUM. In particular, there's no padding octet for integers with the top
bit of the top octet set. Do the check the dumb way and check all the parts
individually: non-negative, not larger than 20 octets and bit 159 not set.

Reported by Tom Harrison

ok claudio job

diff --git a/usr.sbin/rpki-client/x509.c b/usr.sbin/rpki-client/x509.c
index 48d1a70..89d8e13 100644 (file)
--- a/usr.sbin/rpki-client/x509.c
+++ b/usr.sbin/rpki-client/x509.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: x509.c,v 1.101 2024/09/12 10:33:25 tb Exp $ */
+/*     $OpenBSD: x509.c,v 1.102 2024/10/07 12:18:31 tb Exp $ */
 /*
  * Copyright (c) 2022 Theo Buehler <tb@openbsd.org>
  * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
@@ -1023,11 +1023,6 @@ x509_seqnum_to_bn(const char *fn, const char *descr, const ASN1_INTEGER *i)
 {
        BIGNUM *bn = NULL;
 
-       if (ASN1_STRING_length(i) > 20) {
-               warnx("%s: %s should fit in 20 octets", fn, descr);
-               goto out;
-       }
-
        if ((bn = ASN1_INTEGER_to_BN(i, NULL)) == NULL) {
                warnx("%s: %s: ASN1_INTEGER_to_BN error", fn, descr);
                goto out;
@@ -1038,6 +1033,11 @@ x509_seqnum_to_bn(const char *fn, const char *descr, const ASN1_INTEGER *i)
                goto out;
        }
 
+       if (BN_num_bytes(bn) > 20 || BN_is_bit_set(bn, 159)) {
+               warnx("%s: %s should fit in 20 octets", fn, descr);
+               goto out;
+       }
+
        return bn;
 
  out: