pki: RSA signatures with SHA2 hash algorithms (RFC 8332)

 * This change introduces a new API to request signature using
   one key and different hash algorithms. This is used only with
   RSA keys, that used to have SHA1 hardcoded, but the new
   algorithsms allow to use the SHA2 hashes, if the extension
   is negotiated.

Signed-off-by: Jakub Jelen <jjelen@redhat.com>
Reviewed-by: Andreas Schneider <asn@cryptomilk.org>

1 files changed, 78 insertions, 9 deletions

diff --git a/src/pki.c b/src/pki.c
index f0557708..5106aec5 100644
--- a/src/pki.c
+++ b/src/pki.c
@@ -272,6 +272,44 @@ static enum ssh_digest_e ssh_key_hash_from_name(const char *name)
     return SSH_DIGEST_AUTO;
 }
 /**
+ * @brief Convert a key type to a hash type. This is usually unambiguous
+ * for all the key types, unless the SHA2 extension (RFC 8332) is
+ * negotiated during key exchange.
+ *
+ * @param[in]  session  SSH Session.
+ *
+ * @param[in]  type     The type to convert.
+ *
+ * @return              A hash type to be used.
+ */
+static enum ssh_keytypes_e ssh_key_type_to_hash(ssh_session session,
+                                                enum ssh_keytypes_e type)
+{
+    /* TODO this should also reflect supported key types specified in
+     * configuration (ssh_config PubkeyAcceptedKeyTypes) */
+    switch (type) {
+    case SSH_KEYTYPE_RSA:
+        if (session->extensions & SSH_EXT_SIG_RSA_SHA512) {
+            return SSH_DIGEST_SHA512;
+        }
+
+        if (session->extensions & SSH_EXT_SIG_RSA_SHA256) {
+            return SSH_DIGEST_SHA256;
+        }
+
+        /* Default algorithm for RSA is SHA1 */
+        return SSH_DIGEST_SHA1;
+
+    default:
+        /* Other key types use the default value (not used) */
+        return SSH_DIGEST_AUTO;
+    }
+
+    /* We should never reach this */
+    return SSH_DIGEST_AUTO;
+}
+
+/**
  * @brief Convert a ssh key algorithm name to a ssh key algorithm type.
  *
  * @param[in] name      The name to convert.
@@ -1783,24 +1821,55 @@ ssh_string ssh_pki_do_sign(ssh_session session,
                           ssh_buffer_get_len(buf));
         ssh_buffer_free(buf);
     } else {
-        unsigned char hash[SHA_DIGEST_LEN] = {0};
-        SHACTX ctx;
+        unsigned char hash[SHA512_DIGEST_LEN] = {0};
+        uint32_t hlen = 0;
+        enum ssh_digest_e hash_type;
+        ssh_buffer buf;
 
-        ctx = sha1_init();
-        if (ctx == NULL) {
+        buf = ssh_buffer_new();
+        if (buf == NULL) {
             ssh_string_free(session_id);
             return NULL;
         }
 
-        sha1_update(ctx, session_id, ssh_string_len(session_id) + 4);
-        sha1_update(ctx, ssh_buffer_get(sigbuf), ssh_buffer_get_len(sigbuf));
-        sha1_final(hash, ctx);
+        ssh_buffer_set_secure(buf);
+        rc = ssh_buffer_pack(buf,
+                             "SP",
+                             session_id,
+                             ssh_buffer_get_len(sigbuf), ssh_buffer_get(sigbuf));
+        if (rc != SSH_OK) {
+            ssh_string_free(session_id);
+            ssh_buffer_free(buf);
+            return NULL;
+        }
+
+        hash_type = ssh_key_type_to_hash(session, privkey->type);
+        switch (hash_type) {
+        case SSH_DIGEST_SHA256:
+            sha256(ssh_buffer_get(buf), ssh_buffer_get_len(buf), hash);
+            hlen = SHA256_DIGEST_LEN;
+            break;
+        case SSH_DIGEST_SHA512:
+            sha512(ssh_buffer_get(buf), ssh_buffer_get_len(buf), hash);
+            hlen = SHA512_DIGEST_LEN;
+            break;
+        case SSH_DIGEST_SHA1:
+        case SSH_DIGEST_AUTO:
+            sha1(ssh_buffer_get(buf), ssh_buffer_get_len(buf), hash);
+            hlen = SHA_DIGEST_LEN;
+            break;
+        default:
+            SSH_LOG(SSH_LOG_TRACE, "Unknown hash algorithm for type: %d",
+                    sig->type);
+            ssh_string_free(session_id);
+            return NULL;
+        }
 
 #ifdef DEBUG_CRYPTO
-        ssh_print_hexa("Hash being signed", hash, SHA_DIGEST_LEN);
+        ssh_print_hexa("Hash being signed", hash, hlen);
 #endif
 
-        sig = pki_do_sign(privkey, hash, SHA_DIGEST_LEN);
+        sig = pki_do_sign_hash(privkey, hash, hlen, hash_type);
     }
     ssh_string_free(session_id);
     if (sig == NULL) {