ASN.1 enc: check ASN1_item_ex_i2d() consistency

The i2d API design is: call a function first with a pointer to NULL, get
the length, allocate a buffer, call the function passing the buffer in.
Both calls should be checked since ther are still internal allocations.

At the heart of ASN.1 encoding, this idiom is used and the second call
is assumed to succeed after the length was determined. This is far from
guaranteed. Check that the second call returns the same length and error
otherwise.

ok jsing

diff --git a/lib/libcrypto/asn1/tasn_enc.c b/lib/libcrypto/asn1/tasn_enc.c
index 55e6589..1b9cfa1 100644 (file)
--- a/lib/libcrypto/asn1/tasn_enc.c
+++ b/lib/libcrypto/asn1/tasn_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tasn_enc.c,v 1.27 2022/11/26 16:08:50 tb Exp $ */
+/* $OpenBSD: tasn_enc.c,v 1.28 2023/03/06 08:08:31 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -108,7 +108,7 @@ asn1_item_flags_i2d(ASN1_VALUE *val, unsigned char **out, const ASN1_ITEM *it,
 {
        if (out && !*out) {
                unsigned char *p, *buf;
-               int len;
+               int len, len2;
                len = ASN1_item_ex_i2d(&val, NULL, it, -1, flags);
                if (len <= 0)
                        return len;
@@ -116,7 +116,12 @@ asn1_item_flags_i2d(ASN1_VALUE *val, unsigned char **out, const ASN1_ITEM *it,
                if (!buf)
                        return -1;
                p = buf;
-               ASN1_item_ex_i2d(&val, &p, it, -1, flags);
+               len2 = ASN1_item_ex_i2d(&val, &p, it, -1, flags);
+               if (len2 != len) {
+                       freezero(buf, len);
+                       ASN1error(ASN1_R_LENGTH_ERROR);
+                       return -1;
+               }
                *out = buf;
                return len;
        }