pki: RSA signatures with SHA2 hash algorithms (RFC 8332)

 * This change introduces a new API to request signature using
   one key and different hash algorithms. This is used only with
   RSA keys, that used to have SHA1 hardcoded, but the new
   algorithsms allow to use the SHA2 hashes, if the extension
   is negotiated.

Signed-off-by: Jakub Jelen <jjelen@redhat.com>
Reviewed-by: Andreas Schneider <asn@cryptomilk.org>

5 files changed, 209 insertions, 31 deletions

diff --git a/include/libssh/pki_priv.h b/include/libssh/pki_priv.h
index b10e3b95..38c7aa4b 100644
--- a/include/libssh/pki_priv.h
+++ b/include/libssh/pki_priv.h
@@ -96,9 +96,12 @@ int pki_signature_verify(ssh_session session,
                          size_t hlen);
 
 /* SSH Signing Functions */
-ssh_signature pki_do_sign(const ssh_key privkey,
-                          const unsigned char *hash,
-                          size_t hlen);
+#define pki_do_sign(key, hash, hlen) \
+    pki_do_sign_hash(key, hash, hlen, SSH_DIGEST_AUTO)
+ssh_signature pki_do_sign_hash(const ssh_key privkey,
+                               const unsigned char *hash,
+                               size_t hlen,
+                               enum ssh_digest_e hash_type);
 ssh_signature pki_do_sign_sessionid(const ssh_key key,
                                     const unsigned char *hash,
                                     size_t hlen);
diff --git a/src/pki.c b/src/pki.c
index f0557708..5106aec5 100644
--- a/src/pki.c
+++ b/src/pki.c
@@ -272,6 +272,44 @@ static enum ssh_digest_e ssh_key_hash_from_name(const char *name)
     return SSH_DIGEST_AUTO;
 }
 /**
+ * @brief Convert a key type to a hash type. This is usually unambiguous
+ * for all the key types, unless the SHA2 extension (RFC 8332) is
+ * negotiated during key exchange.
+ *
+ * @param[in]  session  SSH Session.
+ *
+ * @param[in]  type     The type to convert.
+ *
+ * @return              A hash type to be used.
+ */
+static enum ssh_keytypes_e ssh_key_type_to_hash(ssh_session session,
+                                                enum ssh_keytypes_e type)
+{
+    /* TODO this should also reflect supported key types specified in
+     * configuration (ssh_config PubkeyAcceptedKeyTypes) */
+    switch (type) {
+    case SSH_KEYTYPE_RSA:
+        if (session->extensions & SSH_EXT_SIG_RSA_SHA512) {
+            return SSH_DIGEST_SHA512;
+        }
+
+        if (session->extensions & SSH_EXT_SIG_RSA_SHA256) {
+            return SSH_DIGEST_SHA256;
+        }
+
+        /* Default algorithm for RSA is SHA1 */
+        return SSH_DIGEST_SHA1;
+
+    default:
+        /* Other key types use the default value (not used) */
+        return SSH_DIGEST_AUTO;
+    }
+
+    /* We should never reach this */
+    return SSH_DIGEST_AUTO;
+}
+
+/**
  * @brief Convert a ssh key algorithm name to a ssh key algorithm type.
  *
  * @param[in] name      The name to convert.
@@ -1783,24 +1821,55 @@ ssh_string ssh_pki_do_sign(ssh_session session,
                           ssh_buffer_get_len(buf));
         ssh_buffer_free(buf);
     } else {
-        unsigned char hash[SHA_DIGEST_LEN] = {0};
-        SHACTX ctx;
+        unsigned char hash[SHA512_DIGEST_LEN] = {0};
+        uint32_t hlen = 0;
+        enum ssh_digest_e hash_type;
+        ssh_buffer buf;
 
-        ctx = sha1_init();
-        if (ctx == NULL) {
+        buf = ssh_buffer_new();
+        if (buf == NULL) {
             ssh_string_free(session_id);
             return NULL;
         }
 
-        sha1_update(ctx, session_id, ssh_string_len(session_id) + 4);
-        sha1_update(ctx, ssh_buffer_get(sigbuf), ssh_buffer_get_len(sigbuf));
-        sha1_final(hash, ctx);
+        ssh_buffer_set_secure(buf);
+        rc = ssh_buffer_pack(buf,
+                             "SP",
+                             session_id,
+                             ssh_buffer_get_len(sigbuf), ssh_buffer_get(sigbuf));
+        if (rc != SSH_OK) {
+            ssh_string_free(session_id);
+            ssh_buffer_free(buf);
+            return NULL;
+        }
+
+        hash_type = ssh_key_type_to_hash(session, privkey->type);
+        switch (hash_type) {
+        case SSH_DIGEST_SHA256:
+            sha256(ssh_buffer_get(buf), ssh_buffer_get_len(buf), hash);
+            hlen = SHA256_DIGEST_LEN;
+            break;
+        case SSH_DIGEST_SHA512:
+            sha512(ssh_buffer_get(buf), ssh_buffer_get_len(buf), hash);
+            hlen = SHA512_DIGEST_LEN;
+            break;
+        case SSH_DIGEST_SHA1:
+        case SSH_DIGEST_AUTO:
+            sha1(ssh_buffer_get(buf), ssh_buffer_get_len(buf), hash);
+            hlen = SHA_DIGEST_LEN;
+            break;
+        default:
+            SSH_LOG(SSH_LOG_TRACE, "Unknown hash algorithm for type: %d",
+                    sig->type);
+            ssh_string_free(session_id);
+            return NULL;
+        }
 
 #ifdef DEBUG_CRYPTO
-        ssh_print_hexa("Hash being signed", hash, SHA_DIGEST_LEN);
+        ssh_print_hexa("Hash being signed", hash, hlen);
 #endif
 
-        sig = pki_do_sign(privkey, hash, SHA_DIGEST_LEN);
+        sig = pki_do_sign_hash(privkey, hash, hlen, hash_type);
     }
     ssh_string_free(session_id);
     if (sig == NULL) {
diff --git a/src/pki_crypto.c b/src/pki_crypto.c
index b41dcb3f..524cc0ff 100644
--- a/src/pki_crypto.c
+++ b/src/pki_crypto.c
@@ -1181,23 +1181,43 @@ fail:
  *
  * @param[in]  privkey   The private rsa key to use for signing.
  *
+ * @param[in]  hash_type The hash algorithm to use.
+ *
  * @return               A newly allocated rsa sig blob or NULL on error.
  */
-static ssh_string _RSA_do_sign(const unsigned char *digest,
-                               int dlen,
-                               RSA *privkey)
+static ssh_string _RSA_do_sign_hash(const unsigned char *digest,
+                                    int dlen,
+                                    RSA *privkey,
+                                    enum ssh_digest_e hash_type)
 {
     ssh_string sig_blob;
     unsigned char *sig;
     unsigned int slen;
     int ok;
+    int nid = 0;
+
+    switch (hash_type) {
+    case SSH_DIGEST_SHA1:
+    case SSH_DIGEST_AUTO:
+        nid = NID_sha1;
+        break;
+    case SSH_DIGEST_SHA256:
+        nid = NID_sha256;
+        break;
+    case SSH_DIGEST_SHA512:
+        nid = NID_sha512;
+        break;
+    default:
+        SSH_LOG(SSH_LOG_WARN, "Incomplatible hash type");
+        return NULL;
+    }
 
     sig = malloc(RSA_size(privkey));
     if (sig == NULL) {
         return NULL;
     }
 
-    ok = RSA_sign(NID_sha1, digest, dlen, sig, &slen, privkey);
+    ok = RSA_sign(nid, digest, dlen, sig, &slen, privkey);
     if (!ok) {
         SAFE_FREE(sig);
         return NULL;
@@ -1216,6 +1236,26 @@ static ssh_string _RSA_do_sign(const unsigned char *digest,
     return sig_blob;
 }
 
+/**
+ * @internal
+ *
+ * @brief Compute a digital signature.
+ *
+ * @param[in]  digest    The message digest.
+ *
+ * @param[in]  dlen      The length of the digest.
+ *
+ * @param[in]  privkey   The private rsa key to use for signing.
+ *
+ * @return               A newly allocated rsa sig blob or NULL on error.
+ */
+static ssh_string _RSA_do_sign(const unsigned char *digest,
+                                    int dlen,
+                                    RSA *privkey)
+{
+    return _RSA_do_sign_hash(digest, dlen, privkey, SSH_DIGEST_AUTO);
+}
+
 static ssh_string pki_dsa_signature_to_blob(const ssh_signature sig)
 {
     char buffer[40] = { 0 };
@@ -1686,18 +1726,27 @@ int pki_signature_verify(ssh_session session,
     return SSH_OK;
 }
 
-ssh_signature pki_do_sign(const ssh_key privkey,
-                          const unsigned char *hash,
-                          size_t hlen) {
+ssh_signature pki_do_sign_hash(const ssh_key privkey,
+                               const unsigned char *hash,
+                               size_t hlen,
+                               enum ssh_digest_e hash_type)
+{
     ssh_signature sig;
     int rc;
 
+    /* Only RSA supports different signature algorithm types now */
+    if (privkey->type != SSH_KEYTYPE_RSA && hash_type != SSH_DIGEST_AUTO) {
+        SSH_LOG(SSH_LOG_WARN, "Incompatible signature algorithm passed");
+        return NULL;
+    }
+
     sig = ssh_signature_new();
     if (sig == NULL) {
         return NULL;
     }
 
     sig->type = privkey->type;
+    sig->hash_type = hash_type;
     sig->type_c = privkey->type_c;
 
     switch(privkey->type) {
@@ -1720,7 +1769,8 @@ ssh_signature pki_do_sign(const ssh_key privkey,
             break;
         case SSH_KEYTYPE_RSA:
         case SSH_KEYTYPE_RSA1:
-            sig->rsa_sig = _RSA_do_sign(hash, hlen, privkey->rsa);
+            sig->type_c = ssh_key_signature_to_char(privkey->type, hash_type);
+            sig->rsa_sig = _RSA_do_sign_hash(hash, hlen, privkey->rsa, hash_type);
             if (sig->rsa_sig == NULL) {
                 ssh_signature_free(sig);
                 return NULL;
diff --git a/src/pki_gcrypt.c b/src/pki_gcrypt.c
index 9a8e8622..c0395e9d 100644
--- a/src/pki_gcrypt.c
+++ b/src/pki_gcrypt.c
@@ -2079,19 +2079,29 @@ int pki_signature_verify(ssh_session session,
     return SSH_OK;
 }
 
-ssh_signature pki_do_sign(const ssh_key privkey,
-                          const unsigned char *hash,
-                          size_t hlen) {
+ssh_signature pki_do_sign_hash(const ssh_key privkey,
+                               const unsigned char *hash,
+                               size_t hlen,
+                               enum ssh_digest_e hash_type)
+{
     unsigned char ghash[hlen + 1];
+    const char *hash_c = NULL;
     ssh_signature sig;
     gcry_sexp_t sexp;
     gcry_error_t err;
 
+    /* Only RSA supports different signature algorithm types now */
+    if (privkey->type != SSH_KEYTYPE_RSA && hash_type != SSH_DIGEST_AUTO) {
+        SSH_LOG(SSH_LOG_WARN, "Incompatible signature algorithm passed");
+        return NULL;
+    }
+
     sig = ssh_signature_new();
     if (sig == NULL) {
         return NULL;
     }
     sig->type = privkey->type;
+    sig->hash_type = hash_type;
     sig->type_c = privkey->type_c;
     switch (privkey->type) {
         case SSH_KEYTYPE_DSS:
@@ -2117,9 +2127,26 @@ ssh_signature pki_do_sign(const ssh_key privkey,
             }
             break;
         case SSH_KEYTYPE_RSA:
+            sig->type_c = ssh_key_signature_to_char(privkey->type, hash_type);
+            switch (hash_type) {
+            case SSH_DIGEST_SHA1:
+            case SSH_DIGEST_AUTO:
+                hash_c = "sha1";
+                break;
+            case SSH_DIGEST_SHA256:
+                hash_c = "sha256";
+                break;
+            case SSH_DIGEST_SHA512:
+                hash_c = "sha512";
+                break;
+            default:
+                SSH_LOG(SSH_LOG_WARN, "Incomplatible key algorithm");
+                return NULL;
+            }
             err = gcry_sexp_build(&sexp,
                                   NULL,
-                                  "(data(flags pkcs1)(hash sha1 %b))",
+                                  "(data(flags pkcs1)(hash %s %b))",
+                                  hash_c,
                                   hlen,
                                   hash);
             if (err) {
diff --git a/src/pki_mbedcrypto.c b/src/pki_mbedcrypto.c
index 57465009..68a80b42 100644
--- a/src/pki_mbedcrypto.c
+++ b/src/pki_mbedcrypto.c
@@ -993,21 +993,40 @@ int pki_signature_verify(ssh_session session, const ssh_signature sig, const
     return SSH_OK;
 }
 
-static ssh_string rsa_do_sign(const unsigned char *digest, int dlen,
-        mbedtls_pk_context *privkey)
+static ssh_string rsa_do_sign_hash(const unsigned char *digest,
+                                   int dlen,
+                                   mbedtls_pk_context *privkey,
+                                   enum ssh_digest_e hash_type)
 {
     ssh_string sig_blob = NULL;
+    mbedtls_md_type_t md = 0;
     unsigned char *sig = NULL;
     size_t slen;
     int ok;
 
+    switch (hash_type) {
+    case SSH_DIGEST_SHA1:
+    case SSH_DIGEST_AUTO:
+        md = MBEDTLS_MD_SHA1;
+        break;
+    case SSH_DIGEST_SHA256:
+        md = MBEDTLS_MD_SHA256;
+        break;
+    case SSH_DIGEST_SHA512:
+        md = MBEDTLS_MD_SHA512;
+        break;
+    default:
+        SSH_LOG(SSH_LOG_WARN, "Incomplatible key algorithm");
+        return NULL;
+    }
+
     sig = malloc(mbedtls_pk_get_bitlen(privkey) / 8);
     if (sig == NULL) {
         return NULL;
     }
 
     ok = mbedtls_pk_sign(privkey,
-                         MBEDTLS_MD_SHA1,
+                         md,
                          digest,
                          dlen,
                          sig,
@@ -1034,23 +1053,33 @@ static ssh_string rsa_do_sign(const unsigned char *digest, int dlen,
 }
 
 
-ssh_signature pki_do_sign(const ssh_key privkey, const unsigned char *hash,
-        size_t hlen)
+ssh_signature pki_do_sign_hash(const ssh_key privkey,
+                               const unsigned char *hash,
+                               size_t hlen,
+                               enum ssh_digest_e hash_type)
 {
     ssh_signature sig = NULL;
     int rc;
 
+    /* Only RSA supports different signature algorithm types now */
+    if (privkey->type != SSH_KEYTYPE_RSA && hash_type != SSH_DIGEST_AUTO) {
+        SSH_LOG(SSH_LOG_WARN, "Incompatible signature algorithm passed");
+        return NULL;
+    }
+
     sig = ssh_signature_new();
     if (sig == NULL) {
         return NULL;
     }
 
     sig->type = privkey->type;
+    sig->hash_type = hash_type;
     sig->type_c = privkey->type_c;
 
     switch(privkey->type) {
         case SSH_KEYTYPE_RSA:
-            sig->rsa_sig = rsa_do_sign(hash, hlen, privkey->rsa);
+            sig->type_c = ssh_key_signature_to_char(privkey->type, hash_type);
+            sig->rsa_sig = rsa_do_sign_hash(hash, hlen, privkey->rsa, hash_type);
             if (sig->rsa_sig == NULL) {
                 ssh_signature_free(sig);
                 return NULL;
@@ -1113,7 +1142,7 @@ ssh_signature pki_do_sign_sessionid(const ssh_key key, const unsigned char
 
     switch (key->type) {
         case SSH_KEYTYPE_RSA:
-            sig->rsa_sig = rsa_do_sign(hash, hlen, key->rsa);
+            sig->rsa_sig = rsa_do_sign_hash(hash, hlen, key->rsa, SSH_DIGEST_AUTO);
             if (sig->rsa_sig == NULL) {
                 ssh_signature_free(sig);
                 return NULL;