bignum: harmonize gcrypt, libcrypto and libmcrypt bignum

Ensure most of the abstraction around the 3 libs are consistent.

Signed-off-by: Aris Adamantiadis <aris@0xbadc0de.be>
Reviewed-by: Jakub Jelen <jjelen@redhat.com>
Reviewed-by: Andreas Schneider <asn@cryptomilk.org>

7 files changed, 126 insertions, 202 deletions

diff --git a/src/bignum.c b/src/bignum.c
index 6777f023..286c6b1e 100644
--- a/src/bignum.c
+++ b/src/bignum.c
@@ -56,13 +56,7 @@ ssh_string ssh_make_bignum_string(bignum num) {
     ptr->data[0] = 0;
   }
 
-#ifdef HAVE_LIBGCRYPT
   bignum_bn2bin(num, len, ptr->data + pad);
-#elif HAVE_LIBCRYPTO
-  bignum_bn2bin(num, ptr->data + pad);
-#elif HAVE_LIBMBEDCRYPTO
-  bignum_bn2bin(num, ptr->data + pad);
-#endif
 
   return ptr;
 }
@@ -76,50 +70,30 @@ bignum ssh_make_string_bn(ssh_string string){
       len * 8, len);
 #endif /* DEBUG_CRYPTO */
 
-#ifdef HAVE_LIBGCRYPT
-  bignum_bin2bn(string->data, len, &bn);
-#elif defined HAVE_LIBCRYPTO
-  bn = bignum_bin2bn(string->data, len, NULL);
-#elif defined HAVE_LIBMBEDCRYPTO
+#if defined HAVE_LIBMBEDCRYPTO
   bn = bignum_new();
   bignum_bin2bn(string->data, len, bn);
+#else
+  // FIXME
+  bignum_bin2bn(string->data, len, &bn);
 #endif
 
   return bn;
 }
 
-void ssh_make_string_bn_inplace(ssh_string string,
-                                UNUSED_PARAM(bignum bnout))
-{
-  UNUSED_VAR(size_t len) = ssh_string_len(string);
-#ifdef HAVE_LIBGCRYPT
-  /* XXX: FIXME as needed for LIBGCRYPT ECDSA codepaths. */
-#elif defined HAVE_LIBCRYPTO
-  bignum_bin2bn(string->data, len, bnout);
-#elif defined HAVE_LIBMBEDCRYPTO
-  bignum_bin2bn(string->data, len, bnout);
-#endif
-}
-
 /* prints the bignum on stderr */
-void ssh_print_bignum(const char *which, const bignum num) {
-#ifdef HAVE_LIBGCRYPT
-  unsigned char *hex = NULL;
-  bignum_bn2hex(num, &hex);
-#elif defined HAVE_LIBCRYPTO
-  char *hex = NULL;
-  hex = bignum_bn2hex(num);
-#elif defined HAVE_LIBMBEDCRYPTO
-  char *hex = NULL;
-  hex = bignum_bn2hex(num);
-#endif
-  fprintf(stderr, "%s value: ", which);
-  fprintf(stderr, "%s\n", (hex == NULL) ? "(null)" : (char *) hex);
+void ssh_print_bignum(const char *name, const bignum num)
+{
+    unsigned char *hex = NULL;
+    if (num != NULL) {
+        bignum_bn2hex(num, &hex);
+    }
+    fprintf(stderr, "%s value: %s\n", name, (hex == NULL) ? "(null)" : (char *) hex);
 #ifdef HAVE_LIBGCRYPT
-  SAFE_FREE(hex);
+    SAFE_FREE(hex);
 #elif defined HAVE_LIBCRYPTO
-  OPENSSL_free(hex);
+    OPENSSL_free(hex);
 #elif defined HAVE_LIBMBEDCRYPTO
-  SAFE_FREE(hex);
+    SAFE_FREE(hex);
 #endif
 }
diff --git a/src/curve25519.c b/src/curve25519.c
index 73551542..7c216ade 100644
--- a/src/curve25519.c
+++ b/src/curve25519.c
@@ -87,13 +87,7 @@ int ssh_client_curve25519_init(ssh_session session){
 static int ssh_curve25519_build_k(ssh_session session) {
   ssh_curve25519_pubkey k;
 
-#ifdef HAVE_LIBCRYPTO
-  session->next_crypto->k = bignum_new();
-
-  if (session->next_crypto->k == NULL) {
-    return SSH_ERROR;
-  }
-#elif defined HAVE_LIBMBEDCRYPTO
+#if defined HAVE_LIBMBEDCRYPTO
   session->next_crypto->k = bignum_new();
 
   if (session->next_crypto->k == NULL) {
@@ -108,13 +102,15 @@ static int ssh_curve25519_build_k(ssh_session session) {
 	  crypto_scalarmult(k, session->next_crypto->curve25519_privkey,
 			  session->next_crypto->curve25519_server_pubkey);
 
-#ifdef HAVE_LIBGCRYPT
-  bignum_bin2bn(k, CURVE25519_PUBKEY_SIZE, &session->next_crypto->k);
-#elif defined HAVE_LIBCRYPTO
-  bignum_bin2bn(k, CURVE25519_PUBKEY_SIZE, session->next_crypto->k);
-#elif defined HAVE_LIBMBEDCRYPTO
+#if defined HAVE_LIBMBEDCRYPTO
+  /* FIXME */
   bignum_bin2bn(k, CURVE25519_PUBKEY_SIZE, session->next_crypto->k);
+#else
+  bignum_bin2bn(k, CURVE25519_PUBKEY_SIZE, &session->next_crypto->k);
 #endif
+  if (session->next_crypto->k == NULL) {
+    return SSH_ERROR;
+  }
 
 #ifdef DEBUG_CRYPTO
     ssh_print_hexa("Session server cookie",
diff --git a/src/dh.c b/src/dh.c
index 3fb7e6c4..499dd7a5 100644
--- a/src/dh.c
+++ b/src/dh.c
@@ -279,101 +279,62 @@ static bignum select_p(enum ssh_key_exchange_e type) {
  */
 int ssh_dh_init(void)
 {
+    int rc;
     if (dh_crypto_initialized) {
         return SSH_OK;
     }
 
     g = bignum_new();
     if (g == NULL) {
-        return SSH_ERROR;
-    }
-    bignum_set_word(g,g_int);
-
-#if defined(HAVE_LIBGCRYPT)
-    bignum_bin2bn(p_group1_value, P_GROUP1_LEN, &p_group1);
-    if (p_group1 == NULL) {
-        bignum_safe_free(g);
-
-        return SSH_ERROR;
+        goto error;
     }
-    bignum_bin2bn(p_group14_value, P_GROUP14_LEN, &p_group14);
-    if (p_group14 == NULL) {
-        bignum_safe_free(g);
-        bignum_safe_free(p_group1);
-
-        return SSH_ERROR;
+    rc = bignum_set_word(g, g_int);
+    if (rc != 1) {
+        goto error;
     }
-    bignum_bin2bn(p_group16_value, P_GROUP16_LEN, &p_group16);
-    if (p_group16 == NULL) {
-        bignum_safe_free(g);
-        bignum_safe_free(p_group1);
-        bignum_safe_free(p_group14);
 
-        return SSH_ERROR;
-    }
-    bignum_bin2bn(p_group18_value, P_GROUP18_LEN, &p_group18);
-    if (p_group18 == NULL) {
-        bignum_safe_free(g);
-        bignum_safe_free(p_group1);
-        bignum_safe_free(p_group14);
-        bignum_safe_free(p_group16);
-
-        return SSH_ERROR;
-    }
-#elif defined(HAVE_LIBCRYPTO)
+#if defined(HAVE_LIBMBEDCRYPTO)
+    /* FIXME */
     p_group1 = bignum_new();
-    if (p_group1 == NULL) {
-        bignum_safe_free(g);
-
-        return SSH_ERROR;
-    }
     bignum_bin2bn(p_group1_value, P_GROUP1_LEN, p_group1);
 
     p_group14 = bignum_new();
-    if (p_group14 == NULL) {
-        bignum_safe_free(g);
-        bignum_safe_free(p_group1);
-
-        return SSH_ERROR;
-    }
     bignum_bin2bn(p_group14_value, P_GROUP14_LEN, p_group14);
 
     p_group16 = bignum_new();
-    if (p_group16 == NULL) {
-        bignum_safe_free(g);
-        bignum_safe_free(p_group1);
-        bignum_safe_free(p_group14);
-
-        return SSH_ERROR;
-    }
     bignum_bin2bn(p_group16_value, P_GROUP16_LEN, p_group16);
 
     p_group18 = bignum_new();
+    bignum_bin2bn(p_group18_value, P_GROUP18_LEN, p_group18);
+#else
+    bignum_bin2bn(p_group1_value, P_GROUP1_LEN, &p_group1);
+    if (p_group1 == NULL) {
+        goto error;
+    }
+    bignum_bin2bn(p_group14_value, P_GROUP14_LEN, &p_group14);
+    if (p_group14 == NULL) {
+        goto error;
+    }
+    bignum_bin2bn(p_group16_value, P_GROUP16_LEN, &p_group16);
+    if (p_group16 == NULL) {
+        goto error;
+    }
+    bignum_bin2bn(p_group18_value, P_GROUP18_LEN, &p_group18);
     if (p_group18 == NULL) {
-        bignum_safe_free(g);
-        bignum_safe_free(p_group1);
-        bignum_safe_free(p_group14);
-        bignum_safe_free(p_group16);
-
-        return SSH_ERROR;
+        goto error;
     }
-    bignum_bin2bn(p_group18_value, P_GROUP18_LEN, p_group18);
-#elif defined(HAVE_LIBMBEDCRYPTO)
-    p_group1 = bignum_new();
-    bignum_bin2bn(p_group1_value, P_GROUP1_LEN, p_group1);
-
-    p_group14 = bignum_new();
-    bignum_bin2bn(p_group14_value, P_GROUP14_LEN, p_group14);
-
-    p_group16 = bignum_new();
-    bignum_bin2bn(p_group16_value, P_GROUP16_LEN, p_group16);
 
-    p_group18 = bignum_new();
-    bignum_bin2bn(p_group18_value, P_GROUP18_LEN, p_group18);
 #endif
+
     dh_crypto_initialized = 1;
 
     return 0;
+error:
+    bignum_safe_free(g);
+    bignum_safe_free(p_group1);
+    bignum_safe_free(p_group14);
+    bignum_safe_free(p_group16);
+    return SSH_ERROR;
 }
 
 /**
@@ -470,78 +431,49 @@ int ssh_dh_generate_y(ssh_session session)
 
 /* used by server */
 int ssh_dh_generate_e(ssh_session session) {
-#ifdef HAVE_LIBCRYPTO
   bignum_CTX ctx = bignum_ctx_new();
-  if (ctx == NULL) {
+  if (bignum_ctx_invalid(ctx)) {
     return -1;
   }
-#endif
 
   session->next_crypto->e = bignum_new();
   if (session->next_crypto->e == NULL) {
-#ifdef HAVE_LIBCRYPTO
     bignum_ctx_free(ctx);
-#endif
     return -1;
   }
 
-#ifdef HAVE_LIBGCRYPT
-  bignum_mod_exp(session->next_crypto->e, g, session->next_crypto->x,
-      select_p(session->next_crypto->kex_type));
-#elif defined HAVE_LIBCRYPTO
   bignum_mod_exp(session->next_crypto->e, g, session->next_crypto->x,
       select_p(session->next_crypto->kex_type), ctx);
-#elif defined HAVE_LIBMBEDCRYPTO
-  bignum_mod_exp(session->next_crypto->e, g, session->next_crypto->x,
-      select_p(session->next_crypto->kex_type), NULL);
-#endif
 
 #ifdef DEBUG_CRYPTO
   ssh_print_bignum("e", session->next_crypto->e);
 #endif
 
-#ifdef HAVE_LIBCRYPTO
   bignum_ctx_free(ctx);
-#endif
 
   return 0;
 }
 
 int ssh_dh_generate_f(ssh_session session) {
-#ifdef HAVE_LIBCRYPTO
   bignum_CTX ctx = bignum_ctx_new();
-  if (ctx == NULL) {
+  if (bignum_ctx_invalid(ctx)) {
     return -1;
-  }
-#endif
+   }
 
   session->next_crypto->f = bignum_new();
   if (session->next_crypto->f == NULL) {
-#ifdef HAVE_LIBCRYPTO
     bignum_ctx_free(ctx);
-#endif
     return -1;
   }
 
-#ifdef HAVE_LIBGCRYPT
-  bignum_mod_exp(session->next_crypto->f, g, session->next_crypto->y,
-      select_p(session->next_crypto->kex_type));
-#elif defined HAVE_LIBCRYPTO
   bignum_mod_exp(session->next_crypto->f, g, session->next_crypto->y,
       select_p(session->next_crypto->kex_type), ctx);
-#elif defined HAVE_LIBMBEDCRYPTO
-  bignum_mod_exp(session->next_crypto->f, g, session->next_crypto->y,
-      select_p(session->next_crypto->kex_type), NULL);
-#endif
 
 #ifdef DEBUG_CRYPTO
   ssh_print_bignum("f", session->next_crypto->f);
 #endif
 
-#ifdef HAVE_LIBCRYPTO
   bignum_ctx_free(ctx);
-#endif
-
   return 0;
 }
 
@@ -595,47 +527,26 @@ int ssh_dh_import_e(ssh_session session, ssh_string e_string) {
 }
 
 int ssh_dh_build_k(ssh_session session) {
-#ifdef HAVE_LIBCRYPTO
+  int rc;
   bignum_CTX ctx = bignum_ctx_new();
-  if (ctx == NULL) {
+  if (bignum_ctx_invalid(ctx)) {
     return -1;
   }
-#endif
 
   session->next_crypto->k = bignum_new();
   if (session->next_crypto->k == NULL) {
-#ifdef HAVE_LIBCRYPTO
     bignum_ctx_free(ctx);
-#endif
     return -1;
   }
 
-    /* the server and clients don't use the same numbers */
-#ifdef HAVE_LIBGCRYPT
-  if(session->client) {
-    bignum_mod_exp(session->next_crypto->k, session->next_crypto->f,
-        session->next_crypto->x, select_p(session->next_crypto->kex_type));
-  } else {
-    bignum_mod_exp(session->next_crypto->k, session->next_crypto->e,
-        session->next_crypto->y, select_p(session->next_crypto->kex_type));
-  }
-#elif defined HAVE_LIBCRYPTO
+  /* the server and clients don't use the same numbers */
   if (session->client) {
-    bignum_mod_exp(session->next_crypto->k, session->next_crypto->f,
+    rc = bignum_mod_exp(session->next_crypto->k, session->next_crypto->f,
         session->next_crypto->x, select_p(session->next_crypto->kex_type), ctx);
   } else {
-    bignum_mod_exp(session->next_crypto->k, session->next_crypto->e,
+    rc = bignum_mod_exp(session->next_crypto->k, session->next_crypto->e,
         session->next_crypto->y, select_p(session->next_crypto->kex_type), ctx);
   }
-#elif defined HAVE_LIBMBEDCRYPTO
-  if (session->client) {
-    bignum_mod_exp(session->next_crypto->k, session->next_crypto->f,
-        session->next_crypto->x, select_p(session->next_crypto->kex_type), NULL);
-  } else {
-    bignum_mod_exp(session->next_crypto->k, session->next_crypto->e,
-        session->next_crypto->y, select_p(session->next_crypto->kex_type), NULL);
-  }
-#endif
 
 #ifdef DEBUG_CRYPTO
     ssh_print_hexa("Session server cookie",
@@ -645,11 +556,11 @@ int ssh_dh_build_k(ssh_session session) {
     ssh_print_bignum("Shared secret key", session->next_crypto->k);
 #endif
 
-#ifdef HAVE_LIBCRYPTO
   bignum_ctx_free(ctx);
-#endif
-
-  return 0;
+  if (rc != 1) {
+    return SSH_ERROR;
+  }
+  return SSH_OK;
 }
 
 static SSH_PACKET_CALLBACK(ssh_packet_client_dh_reply);
diff --git a/src/ecdh_crypto.c b/src/ecdh_crypto.c
index 950578a1..0078712e 100644
--- a/src/ecdh_crypto.c
+++ b/src/ecdh_crypto.c
@@ -128,12 +128,6 @@ int ecdh_build_k(ssh_session session) {
     return -1;
   }
 
-  session->next_crypto->k = bignum_new();
-  if (session->next_crypto->k == NULL) {
-    bignum_ctx_free(ctx);
-    return -1;
-  }
-
   pubkey = EC_POINT_new(group);
   if (pubkey == NULL) {
     bignum_ctx_free(ctx);
@@ -176,9 +170,13 @@ int ecdh_build_k(ssh_session session) {
       return -1;
   }
 
-  bignum_bin2bn(buffer, len, session->next_crypto->k);
+  bignum_bin2bn(buffer, len, &session->next_crypto->k);
   free(buffer);
-
+  if (session->next_crypto->k == NULL) {
+      EC_KEY_free(session->next_crypto->ecdh_privkey);
+      session->next_crypto->ecdh_privkey = NULL;
+      return -1;
+  }
   EC_KEY_free(session->next_crypto->ecdh_privkey);
   session->next_crypto->ecdh_privkey = NULL;
 
diff --git a/src/gcrypt_missing.c b/src/gcrypt_missing.c
index 80562317..eb281e31 100644
--- a/src/gcrypt_missing.c
+++ b/src/gcrypt_missing.c
@@ -99,4 +99,22 @@ char *ssh_gcry_bn2dec(bignum bn) {
   return ret;
 }
 
+/** @brief generates a random integer between 0 and max
+ * @returns 1 in case of success, 0 otherwise
+ */
+int ssh_gcry_rand_range(bignum dest, bignum max)
+{
+    size_t bits;
+    bignum rnd;
+
+    bits = bignum_num_bits(max) + 64;
+    rnd = bignum_new();
+    if (rnd == NULL) {
+        return 0;
+    }
+    bignum_rand(rnd, bits);
+    gcry_mpi_mod(dest, rnd, max);
+    bignum_safe_free(rnd);
+    return 1;
+}
 #endif
diff --git a/src/known_hosts.c b/src/known_hosts.c
index 407e1de5..cf883c42 100644
--- a/src/known_hosts.c
+++ b/src/known_hosts.c
@@ -39,15 +39,6 @@
 /*todo: remove this include */
 #include "libssh/string.h"
 
-#ifdef HAVE_LIBGCRYPT
-#include <gcrypt.h>
-#elif defined HAVE_LIBCRYPTO
-#include <openssl/pem.h>
-#include <openssl/dsa.h>
-#include <openssl/err.h>
-#include <openssl/rsa.h>
-#endif /* HAVE_LIBCRYPTO */
-
 #ifndef _WIN32
 # include <netinet/in.h>
 # include <arpa/inet.h>
diff --git a/src/mbedcrypto_missing.c b/src/mbedcrypto_missing.c
index 2c10d5c0..bafe0ff1 100644
--- a/src/mbedcrypto_missing.c
+++ b/src/mbedcrypto_missing.c
@@ -45,7 +45,7 @@ void ssh_mbedcry_bn_free(bignum bn)
     SAFE_FREE(bn);
 }
 
-char *ssh_mbedcry_bn2num(bignum num, int radix)
+unsigned char *ssh_mbedcry_bn2num(bignum num, int radix)
 {
     char *buf = NULL;
     size_t olen;
@@ -67,7 +67,7 @@ char *ssh_mbedcry_bn2num(bignum num, int radix)
         return NULL;
     }
 
-    return buf;
+    return (unsigned char *) buf;
 }
 
 int ssh_mbedcry_rand(bignum rnd, int bits, int top, int bottom)
@@ -127,4 +127,40 @@ int ssh_mbedcry_is_bit_set(bignum num, size_t pos)
     bit = mbedtls_mpi_get_bit(num, pos);
     return bit;
 }
+
+/** @brief generates a random integer between 0 and max
+ * @returns 1 in case of success, 0 otherwise
+ */
+int ssh_mbedcry_rand_range(bignum dest, bignum max)
+{
+    size_t bits;
+    bignum rnd;
+
+    bits = bignum_num_bits(max) + 64;
+    rnd = bignum_new();
+    if (rnd == NULL){
+        return 0;
+    }
+    bignum_rand(rnd, bits);
+    mbedtls_mpi_mod_mpi(dest, rnd, max);
+    bignum_safe_free(rnd);
+    return 1;
+}
+
+int ssh_mbedcry_hex2bn(bignum *dest, char *data)
+{
+    int rc;
+
+    *dest = bignum_new();
+    if (*dest == NULL){
+        return 0;
+    }
+    rc = mbedtls_mpi_read_string(*dest, 16, data);
+    if (rc == 0) {
+        return 1;
+    }
+
+    return 0;
+}
+
 #endif