RSA: Use hashlen as the hash input size as documented

Where hashlen was previously ignored when the hash length could be
inferred from an md_alg parameter, the two must now match.

Adapt the existing tests accordingly. Adapt the sample programs accordingly.

This commit does not add any negative testing.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
diff --git a/library/rsa.c b/library/rsa.c
index a788337..57b8a6f 100644
--- a/library/rsa.c
+++ b/library/rsa.c
@@ -1799,7 +1799,8 @@
         if( md_info == NULL )
             return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
 
-        hashlen = mbedtls_md_get_size( md_info );
+        if( hashlen != mbedtls_md_get_size( md_info ) )
+            return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
     }
 
     md_info = mbedtls_md_info_from_type( (mbedtls_md_type_t) ctx->hash_id );
@@ -1934,14 +1935,13 @@
  * Parameters:
  * - md_alg:  Identifies the hash algorithm used to generate the given hash;
  *            MBEDTLS_MD_NONE if raw data is signed.
- * - hashlen: Length of hash in case hashlen is MBEDTLS_MD_NONE.
+ * - hashlen: Length of hash. Must match md_alg if that's not NONE.
  * - hash:    Buffer containing the hashed message or the raw data.
  * - dst_len: Length of the encoded message.
  * - dst:     Buffer to hold the encoded message.
  *
  * Assumptions:
- * - hash has size hashlen if md_alg == MBEDTLS_MD_NONE.
- * - hash has size corresponding to md_alg if md_alg != MBEDTLS_MD_NONE.
+ * - hash has size hashlen.
  * - dst points to a buffer of size at least dst_len.
  *
  */
@@ -1966,7 +1966,8 @@
         if( mbedtls_oid_get_oid_by_md( md_alg, &oid, &oid_size ) != 0 )
             return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
 
-        hashlen = mbedtls_md_get_size( md_info );
+        if( hashlen != mbedtls_md_get_size( md_info ) )
+            return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
 
         /* Double-check that 8 + hashlen + oid_size can be used as a
          * 1-byte ASN.1 length encoding and that there's no overflow. */
@@ -2031,6 +2032,8 @@
      *                                 TAG-NULL + LEN [ NULL ] ]
      *                 TAG-OCTET + LEN [ HASH ] ]
      */
+    if( 0x08 + oid_size + hashlen >= 0x80 )
+        return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
     *p++ = MBEDTLS_ASN1_SEQUENCE | MBEDTLS_ASN1_CONSTRUCTED;
     *p++ = (unsigned char)( 0x08 + oid_size + hashlen );
     *p++ = MBEDTLS_ASN1_SEQUENCE | MBEDTLS_ASN1_CONSTRUCTED;
@@ -2212,7 +2215,8 @@
         if( md_info == NULL )
             return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
 
-        hashlen = mbedtls_md_get_size( md_info );
+        if( hashlen != mbedtls_md_get_size( md_info ) )
+            return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
     }
 
     md_info = mbedtls_md_info_from_type( mgf1_hash_id );
@@ -2683,7 +2687,7 @@
     }
 
     if( mbedtls_rsa_pkcs1_sign( &rsa, myrand, NULL,
-                                MBEDTLS_MD_SHA1, 0,
+                                MBEDTLS_MD_SHA1, 20,
                                 sha1sum, rsa_ciphertext ) != 0 )
     {
         if( verbose != 0 )
@@ -2696,7 +2700,7 @@
     if( verbose != 0 )
         mbedtls_printf( "passed\n  PKCS#1 sig. verify: " );
 
-    if( mbedtls_rsa_pkcs1_verify( &rsa, MBEDTLS_MD_SHA1, 0,
+    if( mbedtls_rsa_pkcs1_verify( &rsa, MBEDTLS_MD_SHA1, 20,
                                   sha1sum, rsa_ciphertext ) != 0 )
     {
         if( verbose != 0 )
diff --git a/programs/pkey/dh_client.c b/programs/pkey/dh_client.c
index d68dc24..88e817e 100644
--- a/programs/pkey/dh_client.c
+++ b/programs/pkey/dh_client.c
@@ -221,7 +221,7 @@
     }
 
     if( ( ret = mbedtls_rsa_pkcs1_verify( &rsa, MBEDTLS_MD_SHA256,
-                                          0, hash, p ) ) != 0 )
+                                          32, hash, p ) ) != 0 )
     {
         mbedtls_printf( " failed\n  ! mbedtls_rsa_pkcs1_verify returned %d\n\n", ret );
         goto exit;
diff --git a/programs/pkey/dh_server.c b/programs/pkey/dh_server.c
index 9d51c14..e469bbc 100644
--- a/programs/pkey/dh_server.c
+++ b/programs/pkey/dh_server.c
@@ -229,7 +229,7 @@
     buf[n + 1] = (unsigned char)( rsa.MBEDTLS_PRIVATE(len)      );
 
     if( ( ret = mbedtls_rsa_pkcs1_sign( &rsa, NULL, NULL, MBEDTLS_MD_SHA256,
-                                0, hash, buf + n + 2 ) ) != 0 )
+                                32, hash, buf + n + 2 ) ) != 0 )
     {
         mbedtls_printf( " failed\n  ! mbedtls_rsa_pkcs1_sign returned %d\n\n", ret );
         goto exit;
diff --git a/programs/pkey/rsa_sign.c b/programs/pkey/rsa_sign.c
index d9ba3bb..ebc88e4 100644
--- a/programs/pkey/rsa_sign.c
+++ b/programs/pkey/rsa_sign.c
@@ -147,7 +147,7 @@
     }
 
     if( ( ret = mbedtls_rsa_pkcs1_sign( &rsa, NULL, NULL, MBEDTLS_MD_SHA256,
-                                20, hash, buf ) ) != 0 )
+                                32, hash, buf ) ) != 0 )
     {
         mbedtls_printf( " failed\n  ! mbedtls_rsa_pkcs1_sign returned -0x%0x\n\n", (unsigned int) -ret );
         goto exit;
diff --git a/programs/pkey/rsa_verify.c b/programs/pkey/rsa_verify.c
index fff568a..5a68246 100644
--- a/programs/pkey/rsa_verify.c
+++ b/programs/pkey/rsa_verify.c
@@ -141,7 +141,7 @@
     }
 
     if( ( ret = mbedtls_rsa_pkcs1_verify( &rsa, MBEDTLS_MD_SHA256,
-                                          20, hash, buf ) ) != 0 )
+                                          32, hash, buf ) ) != 0 )
     {
         mbedtls_printf( " failed\n  ! mbedtls_rsa_pkcs1_verify returned -0x%0x\n\n", (unsigned int) -ret );
         goto exit;
diff --git a/tests/suites/test_suite_pk.function b/tests/suites/test_suite_pk.function
index 573c9d4..c7c5f50 100644
--- a/tests/suites/test_suite_pk.function
+++ b/tests/suites/test_suite_pk.function
@@ -444,9 +444,10 @@
 
     if( digest != MBEDTLS_MD_NONE )
     {
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( digest ),
-                     message_str->x, message_str->len, hash_result ) == 0 );
-        hash_len = 0;
+        const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type( digest );
+        TEST_ASSERT( mbedtls_md( md_info, message_str->x, message_str->len,
+                                 hash_result ) == 0 );
+        hash_len = mbedtls_md_get_size( md_info );
     }
     else
     {
@@ -611,7 +612,8 @@
 {
     mbedtls_pk_context pk;
     size_t sig_len;
-    unsigned char hash[MBEDTLS_MD_MAX_SIZE];
+    unsigned char hash[32]; // Hard-coded for SHA256
+    size_t hash_len = sizeof( hash );
     unsigned char sig[MBEDTLS_PK_SIGNATURE_MAX_SIZE];
     void *rs_ctx = NULL;
 #if defined(MBEDTLS_ECDSA_C) && defined(MBEDTLS_ECP_RESTARTABLE)
@@ -635,7 +637,7 @@
     TEST_ASSERT( pk_genkey( &pk, parameter ) == 0 );
 
     TEST_ASSERT( mbedtls_pk_sign_restartable( &pk, MBEDTLS_MD_SHA256,
-                 hash, sizeof hash, sig, &sig_len,
+                 hash, hash_len, sig, &sig_len,
                  mbedtls_test_rnd_std_rand, NULL, rs_ctx ) == sign_ret );
     if( sign_ret == 0 )
         TEST_ASSERT( sig_len <= MBEDTLS_PK_SIGNATURE_MAX_SIZE );
@@ -643,22 +645,22 @@
         sig_len = MBEDTLS_PK_SIGNATURE_MAX_SIZE;
 
     TEST_ASSERT( mbedtls_pk_verify( &pk, MBEDTLS_MD_SHA256,
-                            hash, sizeof hash, sig, sig_len ) == verify_ret );
+                            hash, hash_len, sig, sig_len ) == verify_ret );
 
     if( verify_ret == 0 )
     {
         hash[0]++;
         TEST_ASSERT( mbedtls_pk_verify( &pk, MBEDTLS_MD_SHA256,
-                                hash, sizeof hash, sig, sig_len ) != 0 );
+                                hash, hash_len, sig, sig_len ) != 0 );
         hash[0]--;
 
         sig[0]++;
         TEST_ASSERT( mbedtls_pk_verify( &pk, MBEDTLS_MD_SHA256,
-                                hash, sizeof hash, sig, sig_len ) != 0 );
+                                hash, hash_len, sig, sig_len ) != 0 );
         sig[0]--;
     }
 
-    TEST_ASSERT( mbedtls_pk_sign( &pk, MBEDTLS_MD_SHA256, hash, sizeof hash,
+    TEST_ASSERT( mbedtls_pk_sign( &pk, MBEDTLS_MD_SHA256, hash, hash_len,
                                   sig, &sig_len,
                                   mbedtls_test_rnd_std_rand,
                                   NULL ) == sign_ret );
@@ -668,7 +670,7 @@
         sig_len = MBEDTLS_PK_SIGNATURE_MAX_SIZE;
 
     TEST_ASSERT( mbedtls_pk_verify_restartable( &pk, MBEDTLS_MD_SHA256,
-                 hash, sizeof hash, sig, sig_len, rs_ctx ) == verify_ret );
+                 hash, hash_len, sig, sig_len, rs_ctx ) == verify_ret );
 
     if( verify_ret == 0 )
     {
diff --git a/tests/suites/test_suite_pkcs1_v15.function b/tests/suites/test_suite_pkcs1_v15.function
index d78ee88..3d29f10 100644
--- a/tests/suites/test_suite_pkcs1_v15.function
+++ b/tests/suites/test_suite_pkcs1_v15.function
@@ -269,6 +269,7 @@
                             data_t * result_str, int result )
 {
     unsigned char hash_result[MBEDTLS_MD_MAX_SIZE];
+    const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type( digest );
     unsigned char output[128];
     mbedtls_rsa_context ctx;
     mbedtls_mpi N, P, Q, E;
@@ -298,13 +299,13 @@
     TEST_ASSERT( mbedtls_rsa_complete( &ctx ) == 0 );
     TEST_ASSERT( mbedtls_rsa_check_privkey( &ctx ) == 0 );
 
+    if( md_info != NULL )
+        TEST_ASSERT( mbedtls_md( md_info, message_str->x, message_str->len, hash_result ) == 0 );
 
-    if( mbedtls_md_info_from_type( digest ) != NULL )
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( digest ), message_str->x, message_str->len, hash_result ) == 0 );
-
-    TEST_ASSERT( mbedtls_rsa_pkcs1_sign( &ctx, &mbedtls_test_rnd_buffer_rand,
-                                         &info, digest, 0, hash_result,
-                                         output ) == result );
+    TEST_ASSERT( mbedtls_rsa_pkcs1_sign(
+                     &ctx, &mbedtls_test_rnd_buffer_rand, &info,
+                     digest, mbedtls_md_get_size( md_info ), hash_result,
+                     output ) == result );
     if( result == 0 )
     {
 
@@ -326,6 +327,7 @@
                               data_t * result_str, int result )
 {
     unsigned char hash_result[MBEDTLS_MD_MAX_SIZE];
+    const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type( digest );
     mbedtls_rsa_context ctx;
     mbedtls_mpi N, E;
     ((void) salt);
@@ -343,10 +345,10 @@
     TEST_ASSERT( mbedtls_rsa_check_pubkey( &ctx ) == 0 );
 
 
-    if( mbedtls_md_info_from_type( digest ) != NULL )
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( digest ), message_str->x, message_str->len, hash_result ) == 0 );
+    if( md_info != NULL )
+        TEST_ASSERT( mbedtls_md( md_info, message_str->x, message_str->len, hash_result ) == 0 );
 
-    TEST_ASSERT( mbedtls_rsa_pkcs1_verify( &ctx, digest, 0, hash_result, result_str->x ) == result );
+    TEST_ASSERT( mbedtls_rsa_pkcs1_verify( &ctx, digest, mbedtls_md_get_size( md_info ), hash_result, result_str->x ) == result );
 
 exit:
     mbedtls_mpi_free( &N ); mbedtls_mpi_free( &E );
diff --git a/tests/suites/test_suite_pkcs1_v21.function b/tests/suites/test_suite_pkcs1_v21.function
index ec5591f..27b0990 100644
--- a/tests/suites/test_suite_pkcs1_v21.function
+++ b/tests/suites/test_suite_pkcs1_v21.function
@@ -123,6 +123,7 @@
                             int result )
 {
     unsigned char hash_result[MBEDTLS_MD_MAX_SIZE];
+    const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type( digest );
     unsigned char output[512];
     mbedtls_rsa_context ctx;
     mbedtls_test_rnd_buf_info info;
@@ -152,14 +153,15 @@
     TEST_ASSERT( mbedtls_rsa_complete( &ctx ) == 0 );
     TEST_ASSERT( mbedtls_rsa_check_privkey( &ctx ) == 0 );
 
-    if( mbedtls_md_info_from_type( digest ) != NULL )
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( digest ), message_str->x, message_str->len, hash_result ) == 0 );
+    if( md_info != NULL )
+        TEST_ASSERT( mbedtls_md( md_info, message_str->x, message_str->len, hash_result ) == 0 );
 
     if (fixed_salt_length == MBEDTLS_RSA_SALT_LEN_ANY)
     {
-        TEST_ASSERT( mbedtls_rsa_pkcs1_sign( &ctx, &mbedtls_test_rnd_buffer_rand,
-                                             &info, digest, 0,hash_result,
-                                             output ) == result );
+        TEST_ASSERT( mbedtls_rsa_pkcs1_sign(
+                         &ctx, &mbedtls_test_rnd_buffer_rand, &info,
+                         digest, mbedtls_md_get_size( md_info ), hash_result,
+                         output ) == result );
         if( result == 0 )
         {
             ASSERT_COMPARE( output, ctx.len, result_str->x, result_str->len );
@@ -169,9 +171,10 @@
         info.length = rnd_buf->len;
     }
 
-    TEST_ASSERT( mbedtls_rsa_rsassa_pss_sign_ext( &ctx, &mbedtls_test_rnd_buffer_rand,
-                                                  &info, digest, 0, hash_result,
-                                                  fixed_salt_length, output ) == result );
+    TEST_ASSERT( mbedtls_rsa_rsassa_pss_sign_ext(
+                     &ctx, &mbedtls_test_rnd_buffer_rand, &info,
+                     digest, mbedtls_md_get_size( md_info ), hash_result,
+                     fixed_salt_length, output ) == result );
     if( result == 0 )
     {
         ASSERT_COMPARE( output, ctx.len, result_str->x, result_str->len );
@@ -190,6 +193,7 @@
                               char * salt, data_t * result_str, int result )
 {
     unsigned char hash_result[MBEDTLS_MD_MAX_SIZE];
+    const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type( digest );
     mbedtls_rsa_context ctx;
     mbedtls_mpi N, E;
     ((void) salt);
@@ -208,10 +212,10 @@
     TEST_ASSERT( mbedtls_rsa_check_pubkey( &ctx ) == 0 );
 
 
-    if( mbedtls_md_info_from_type( digest ) != NULL )
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( digest ), message_str->x, message_str->len, hash_result ) == 0 );
+    if( md_info != NULL )
+        TEST_ASSERT( mbedtls_md( md_info, message_str->x, message_str->len, hash_result ) == 0 );
 
-    TEST_ASSERT( mbedtls_rsa_pkcs1_verify( &ctx, digest, 0, hash_result, result_str->x ) == result );
+    TEST_ASSERT( mbedtls_rsa_pkcs1_verify( &ctx, digest, mbedtls_md_get_size( md_info ), hash_result, result_str->x ) == result );
 
 exit:
     mbedtls_mpi_free( &N ); mbedtls_mpi_free( &E );
@@ -248,9 +252,12 @@
 
     if( msg_digest_id != MBEDTLS_MD_NONE )
     {
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( msg_digest_id ),
-                     message_str->x, message_str->len, hash_result ) == 0 );
-        hash_len = 0;
+        const mbedtls_md_info_t *md_info =
+            mbedtls_md_info_from_type( msg_digest_id );
+        TEST_ASSERT( mbedtls_md( md_info,
+                                 message_str->x, message_str->len,
+                                 hash_result ) == 0 );
+        hash_len = mbedtls_md_get_size( md_info );
     }
     else
     {
diff --git a/tests/suites/test_suite_rsa.function b/tests/suites/test_suite_rsa.function
index 14b4afc..1bffc76 100644
--- a/tests/suites/test_suite_rsa.function
+++ b/tests/suites/test_suite_rsa.function
@@ -86,6 +86,7 @@
                              data_t * result_str, int result )
 {
     unsigned char hash_result[MBEDTLS_MD_MAX_SIZE];
+    const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type( digest );
     unsigned char output[256];
     mbedtls_rsa_context ctx;
     mbedtls_mpi N, P, Q, E;
@@ -111,13 +112,13 @@
     TEST_ASSERT( mbedtls_rsa_complete( &ctx ) == 0 );
     TEST_ASSERT( mbedtls_rsa_check_privkey( &ctx ) == 0 );
 
+    if( md_info != NULL )
+        TEST_ASSERT( mbedtls_md( md_info, message_str->x, message_str->len, hash_result ) == 0 );
 
-    if( mbedtls_md_info_from_type( digest ) != NULL )
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( digest ), message_str->x, message_str->len, hash_result ) == 0 );
-
-    TEST_ASSERT( mbedtls_rsa_pkcs1_sign( &ctx, &mbedtls_test_rnd_pseudo_rand,
-                                         &rnd_info, digest, 0, hash_result,
-                                         output ) == result );
+    TEST_ASSERT( mbedtls_rsa_pkcs1_sign(
+                     &ctx, &mbedtls_test_rnd_pseudo_rand, &rnd_info,
+                     digest, mbedtls_md_get_size( md_info ), hash_result,
+                     output ) == result );
     if( result == 0 )
     {
 
@@ -139,8 +140,8 @@
                                data_t * result_str, int result )
 {
     unsigned char hash_result[MBEDTLS_MD_MAX_SIZE];
+    const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type( digest );
     mbedtls_rsa_context ctx;
-
     mbedtls_mpi N, E;
 
     mbedtls_mpi_init( &N ); mbedtls_mpi_init( &E );
@@ -155,11 +156,10 @@
     TEST_ASSERT( mbedtls_rsa_get_len( &ctx ) == (size_t) ( mod / 8 ) );
     TEST_ASSERT( mbedtls_rsa_check_pubkey( &ctx ) == 0 );
 
+    if( md_info != NULL )
+        TEST_ASSERT( mbedtls_md( md_info, message_str->x, message_str->len, hash_result ) == 0 );
 
-    if( mbedtls_md_info_from_type( digest ) != NULL )
-        TEST_ASSERT( mbedtls_md( mbedtls_md_info_from_type( digest ), message_str->x, message_str->len, hash_result ) == 0 );
-
-    TEST_ASSERT( mbedtls_rsa_pkcs1_verify( &ctx, digest, 0, hash_result, result_str->x ) == result );
+    TEST_ASSERT( mbedtls_rsa_pkcs1_verify( &ctx, digest, mbedtls_md_get_size( md_info ), hash_result, result_str->x ) == result );
 
 exit:
     mbedtls_mpi_free( &N ); mbedtls_mpi_free( &E );