Reject too small bits value in BN_generate_prime_ex(), so that it does not risk

becoming negative in probable_prime_dh_safe(). Reported by Franck Denis who
noticed `openssl gendh 0' would segfault.
Fix adapted from OpenSSL RT#2701.

ok beck@ jsing@

diff --git a/lib/libcrypto/bn/bn.h b/lib/libcrypto/bn/bn.h
index 33c6162..2c648ba 100644 (file)
--- a/lib/libcrypto/bn/bn.h
+++ b/lib/libcrypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.27 2015/10/16 12:41:29 beck Exp $ */
+/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -768,6 +768,7 @@ void ERR_load_BN_strings(void);
 #define BN_F_BN_DIV_RECP                                130
 #define BN_F_BN_EXP                                     123
 #define BN_F_BN_EXPAND2                                         108
+#define BN_F_BN_GENERATE_PRIME_EX                       140
 #define BN_F_BN_EXPAND_INTERNAL                                 120
 #define BN_F_BN_GF2M_MOD                                131
 #define BN_F_BN_GF2M_MOD_EXP                            132
@@ -797,6 +798,7 @@ void ERR_load_BN_strings(void);
 #define BN_R_ARG2_LT_ARG3                               100
 #define BN_R_BAD_RECIPROCAL                             101
 #define BN_R_BIGNUM_TOO_LONG                            114
+#define BN_R_BITS_TOO_SMALL                             117
 #define BN_R_CALLED_WITH_EVEN_MODULUS                   102
 #define BN_R_DIV_BY_ZERO                                103
 #define BN_R_ENCODING_ERROR                             104

diff --git a/lib/libcrypto/bn/bn_err.c b/lib/libcrypto/bn/bn_err.c
index 5a0f359..149e58e 100644 (file)
--- a/lib/libcrypto/bn/bn_err.c
+++ b/lib/libcrypto/bn/bn_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_err.c,v 1.12 2014/07/10 22:45:56 jsing Exp $ */
+/* $OpenBSD: bn_err.c,v 1.13 2015/10/21 19:02:22 miod Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2007 The OpenSSL Project.  All rights reserved.
  *
@@ -89,6 +89,7 @@ static ERR_STRING_DATA BN_str_functs[]= {
        {ERR_FUNC(BN_F_BN_EXP), "BN_exp"},
        {ERR_FUNC(BN_F_BN_EXPAND2),     "bn_expand2"},
        {ERR_FUNC(BN_F_BN_EXPAND_INTERNAL),     "BN_EXPAND_INTERNAL"},
+       {ERR_FUNC(BN_F_BN_GENERATE_PRIME_EX),   "BN_generate_prime_ex"},
        {ERR_FUNC(BN_F_BN_GF2M_MOD),    "BN_GF2m_mod"},
        {ERR_FUNC(BN_F_BN_GF2M_MOD_EXP),        "BN_GF2m_mod_exp"},
        {ERR_FUNC(BN_F_BN_GF2M_MOD_MUL),        "BN_GF2m_mod_mul"},
@@ -119,6 +120,7 @@ static ERR_STRING_DATA BN_str_reasons[]= {
        {ERR_REASON(BN_R_ARG2_LT_ARG3)           , "arg2 lt arg3"},
        {ERR_REASON(BN_R_BAD_RECIPROCAL)         , "bad reciprocal"},
        {ERR_REASON(BN_R_BIGNUM_TOO_LONG)        , "bignum too long"},
+       {ERR_REASON(BN_R_BITS_TOO_SMALL)         , "bits too small"},
        {ERR_REASON(BN_R_CALLED_WITH_EVEN_MODULUS), "called with even modulus"},
        {ERR_REASON(BN_R_DIV_BY_ZERO)            , "div by zero"},
        {ERR_REASON(BN_R_ENCODING_ERROR)         , "encoding error"},

diff --git a/lib/libcrypto/bn/bn_prime.c b/lib/libcrypto/bn/bn_prime.c
index 02780d3..b1aba66 100644 (file)
--- a/lib/libcrypto/bn/bn_prime.c
+++ b/lib/libcrypto/bn/bn_prime.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_prime.c,v 1.13 2015/02/09 15:49:22 jsing Exp $ */
+/* $OpenBSD: bn_prime.c,v 1.14 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -112,6 +112,8 @@
 #include <stdio.h>
 #include <time.h>
 
+#include <openssl/err.h>
+
 #include "bn_lcl.h"
 
 /* NB: these functions have been "upgraded", the deprecated versions (which are
@@ -164,7 +166,16 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
        int found = 0;
        int i, j, c1 = 0;
        BN_CTX *ctx;
-       int checks = BN_prime_checks_for_size(bits);
+       int checks;
+
+       if (bits < 2 || (bits == 2 && safe)) {
+               /*
+                * There are no prime numbers smaller than 2, and the smallest
+                * safe prime (7) spans three bits.
+                */
+               BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
+               return 0;
+       }
 
        ctx = BN_CTX_new();
        if (ctx == NULL)
@@ -172,6 +183,9 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
        BN_CTX_start(ctx);
        if ((t = BN_CTX_get(ctx)) == NULL)
                goto err;
+
+       checks = BN_prime_checks_for_size(bits);
+
 loop:
        /* make a random number and set the top and bottom bits */
        if (add == NULL) {

diff --git a/lib/libssl/src/crypto/bn/bn.h b/lib/libssl/src/crypto/bn/bn.h
index 33c6162..2c648ba 100644 (file)
--- a/lib/libssl/src/crypto/bn/bn.h
+++ b/lib/libssl/src/crypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.27 2015/10/16 12:41:29 beck Exp $ */
+/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -768,6 +768,7 @@ void ERR_load_BN_strings(void);
 #define BN_F_BN_DIV_RECP                                130
 #define BN_F_BN_EXP                                     123
 #define BN_F_BN_EXPAND2                                         108
+#define BN_F_BN_GENERATE_PRIME_EX                       140
 #define BN_F_BN_EXPAND_INTERNAL                                 120
 #define BN_F_BN_GF2M_MOD                                131
 #define BN_F_BN_GF2M_MOD_EXP                            132
@@ -797,6 +798,7 @@ void ERR_load_BN_strings(void);
 #define BN_R_ARG2_LT_ARG3                               100
 #define BN_R_BAD_RECIPROCAL                             101
 #define BN_R_BIGNUM_TOO_LONG                            114
+#define BN_R_BITS_TOO_SMALL                             117
 #define BN_R_CALLED_WITH_EVEN_MODULUS                   102
 #define BN_R_DIV_BY_ZERO                                103
 #define BN_R_ENCODING_ERROR                             104

diff --git a/lib/libssl/src/crypto/bn/bn_err.c b/lib/libssl/src/crypto/bn/bn_err.c
index 5a0f359..149e58e 100644 (file)
--- a/lib/libssl/src/crypto/bn/bn_err.c
+++ b/lib/libssl/src/crypto/bn/bn_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_err.c,v 1.12 2014/07/10 22:45:56 jsing Exp $ */
+/* $OpenBSD: bn_err.c,v 1.13 2015/10/21 19:02:22 miod Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2007 The OpenSSL Project.  All rights reserved.
  *
@@ -89,6 +89,7 @@ static ERR_STRING_DATA BN_str_functs[]= {
        {ERR_FUNC(BN_F_BN_EXP), "BN_exp"},
        {ERR_FUNC(BN_F_BN_EXPAND2),     "bn_expand2"},
        {ERR_FUNC(BN_F_BN_EXPAND_INTERNAL),     "BN_EXPAND_INTERNAL"},
+       {ERR_FUNC(BN_F_BN_GENERATE_PRIME_EX),   "BN_generate_prime_ex"},
        {ERR_FUNC(BN_F_BN_GF2M_MOD),    "BN_GF2m_mod"},
        {ERR_FUNC(BN_F_BN_GF2M_MOD_EXP),        "BN_GF2m_mod_exp"},
        {ERR_FUNC(BN_F_BN_GF2M_MOD_MUL),        "BN_GF2m_mod_mul"},
@@ -119,6 +120,7 @@ static ERR_STRING_DATA BN_str_reasons[]= {
        {ERR_REASON(BN_R_ARG2_LT_ARG3)           , "arg2 lt arg3"},
        {ERR_REASON(BN_R_BAD_RECIPROCAL)         , "bad reciprocal"},
        {ERR_REASON(BN_R_BIGNUM_TOO_LONG)        , "bignum too long"},
+       {ERR_REASON(BN_R_BITS_TOO_SMALL)         , "bits too small"},
        {ERR_REASON(BN_R_CALLED_WITH_EVEN_MODULUS), "called with even modulus"},
        {ERR_REASON(BN_R_DIV_BY_ZERO)            , "div by zero"},
        {ERR_REASON(BN_R_ENCODING_ERROR)         , "encoding error"},

diff --git a/lib/libssl/src/crypto/bn/bn_prime.c b/lib/libssl/src/crypto/bn/bn_prime.c
index 02780d3..b1aba66 100644 (file)
--- a/lib/libssl/src/crypto/bn/bn_prime.c
+++ b/lib/libssl/src/crypto/bn/bn_prime.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_prime.c,v 1.13 2015/02/09 15:49:22 jsing Exp $ */
+/* $OpenBSD: bn_prime.c,v 1.14 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -112,6 +112,8 @@
 #include <stdio.h>
 #include <time.h>
 
+#include <openssl/err.h>
+
 #include "bn_lcl.h"
 
 /* NB: these functions have been "upgraded", the deprecated versions (which are
@@ -164,7 +166,16 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
        int found = 0;
        int i, j, c1 = 0;
        BN_CTX *ctx;
-       int checks = BN_prime_checks_for_size(bits);
+       int checks;
+
+       if (bits < 2 || (bits == 2 && safe)) {
+               /*
+                * There are no prime numbers smaller than 2, and the smallest
+                * safe prime (7) spans three bits.
+                */
+               BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
+               return 0;
+       }
 
        ctx = BN_CTX_new();
        if (ctx == NULL)
@@ -172,6 +183,9 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
        BN_CTX_start(ctx);
        if ((t = BN_CTX_get(ctx)) == NULL)
                goto err;
+
+       checks = BN_prime_checks_for_size(bits);
+
 loop:
        /* make a random number and set the top and bottom bits */
        if (add == NULL) {