Return an int in BIO_set_cipher() to be able to report errors.

tested in a bulk by sthen
ok jsing

diff --git a/lib/libcrypto/evp/bio_enc.c b/lib/libcrypto/evp/bio_enc.c
index 712222a..7b55998 100644 (file)
--- a/lib/libcrypto/evp/bio_enc.c
+++ b/lib/libcrypto/evp/bio_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bio_enc.c,v 1.21 2018/05/02 15:51:41 tb Exp $ */
+/* $OpenBSD: bio_enc.c,v 1.22 2018/08/24 19:30:24 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -404,23 +404,32 @@ EVP_CIPHER_ctx *c;
        }
 */
 
-void
+int
 BIO_set_cipher(BIO *b, const EVP_CIPHER *c, const unsigned char *k,
     const unsigned char *i, int e)
 {
        BIO_ENC_CTX *ctx;
+       long (*cb)(BIO *, int, const char *, int, long, long);
 
        if (b == NULL)
-               return;
+               return 0;
 
-       if ((b->callback != NULL) &&
-           (b->callback(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 0L) <= 0))
-               return;
+       if ((ctx = BIO_get_data(b)) == NULL)
+               return 0;
 
-       b->init = 1;
-       ctx = (BIO_ENC_CTX *)b->ptr;
-       EVP_CipherInit_ex(&(ctx->cipher), c, NULL, k, i, e);
+       if ((cb = BIO_get_callback(b)) != NULL) {
+               if (cb(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 0L)
+                   <= 0)
+                       return 0;
+       }
 
-       if (b->callback != NULL)
-               b->callback(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 1L);
+       BIO_set_init(b, 1);
+
+       if (!EVP_CipherInit_ex(&(ctx->cipher), c, NULL, k, i, e))
+               return 0;
+
+       if (cb != NULL)
+               return cb(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 1L);
+
+       return 1;
 }

diff --git a/lib/libcrypto/evp/evp.h b/lib/libcrypto/evp/evp.h
index c557ba9..c6ec0e3 100644 (file)
--- a/lib/libcrypto/evp/evp.h
+++ b/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.64 2018/05/30 15:40:50 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.65 2018/08/24 19:30:24 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -655,7 +655,7 @@ int EVP_CIPHER_CTX_rand_key(EVP_CIPHER_CTX *ctx, unsigned char *key);
 const BIO_METHOD *BIO_f_md(void);
 const BIO_METHOD *BIO_f_base64(void);
 const BIO_METHOD *BIO_f_cipher(void);
-void BIO_set_cipher(BIO *b, const EVP_CIPHER *c, const unsigned char *k,
+int BIO_set_cipher(BIO *b, const EVP_CIPHER *c, const unsigned char *k,
     const unsigned char *i, int enc);
 #endif