Do not pass input length <= 0 to the cipher handlers

Input length < 0 is an error and input length == 0 can result in
strange effects in some ciphers, except in CCM mode, which is extra
special.

Based on OpenSSL 420cb707 by Matt Caswell and Richard Levitte

found by & ok jsing

diff --git a/lib/libcrypto/evp/evp_enc.c b/lib/libcrypto/evp/evp_enc.c
index 896b9e1..d925ed7 100644 (file)
--- a/lib/libcrypto/evp/evp_enc.c
+++ b/lib/libcrypto/evp/evp_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_enc.c,v 1.44 2021/02/18 19:12:29 tb Exp $ */
+/* $OpenBSD: evp_enc.c,v 1.45 2022/07/26 19:50:06 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -300,6 +300,14 @@ EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
 {
        int i, j, bl;
 
+       *outl = 0;
+
+       if (inl < 0)
+               return 0;
+
+       if (inl == 0 && EVP_CIPHER_mode(ctx->cipher) != EVP_CIPH_CCM_MODE)
+               return 1;
+
        if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
                i = M_do_cipher(ctx, out, in, inl);
                if (i < 0)
@@ -309,11 +317,6 @@ EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                return 1;
        }
 
-       if (inl <= 0) {
-               *outl = 0;
-               return inl == 0;
-       }
-
        if (ctx->buf_len == 0 && (inl&(ctx->block_mask)) == 0) {
                if (M_do_cipher(ctx, out, in, inl)) {
                        *outl = inl;
@@ -438,6 +441,14 @@ EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
        int fix_len;
        unsigned int b;
 
+       *outl = 0;
+
+       if (inl < 0)
+               return 0;
+
+       if (inl == 0 && EVP_CIPHER_mode(ctx->cipher) != EVP_CIPH_CCM_MODE)
+               return 1;
+
        if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
                fix_len = M_do_cipher(ctx, out, in, inl);
                if (fix_len < 0) {
@@ -448,11 +459,6 @@ EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                return 1;
        }
 
-       if (inl <= 0) {
-               *outl = 0;
-               return inl == 0;
-       }
-
        if (ctx->flags & EVP_CIPH_NO_PADDING)
                return EVP_EncryptUpdate(ctx, out, outl, in, inl);