commit 5e40a7cfa033619c35f583aeb5ebd6a185c3786b
author Gilles Peskine <Gilles.Peskine@arm.com> Tue Feb 02 21:06:10 2021 +0100
committer Gilles Peskine <Gilles.Peskine@arm.com> Mon Feb 22 19:24:03 2021 +0100
tree 3b1ff0245a55d6277ba315d9ee3df973bcbec9f9
parent eb94059eddbc3c5687d61dd8684f15f7f49f490d

Fix mutex leak in RSA

mbedtls_rsa_gen_key() was not freeing the RSA object, and specifically
not freeing the mutex, in some error cases.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>

library/rsa.c

diff --git a/library/rsa.c b/library/rsa.c
index 8c6a507..68a36f2 100644
--- a/library/rsa.c
+++ b/library/rsa.c
@@ -540,9 +540,6 @@
     RSA_VALIDATE_RET( ctx != NULL );
     RSA_VALIDATE_RET( f_rng != NULL );
 
-    if( nbits < 128 || exponent < 3 || nbits % 2 != 0 )
-        return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
-
     /*
      * If the modulus is 1024 bit long or shorter, then the security strength of
      * the RSA algorithm is less than or equal to 80 bits and therefore an error
@@ -555,6 +552,12 @@
     mbedtls_mpi_init( &G );
     mbedtls_mpi_init( &L );
 
+    if( nbits < 128 || exponent < 3 || nbits % 2 != 0 )
+    {
+        ret = MBEDTLS_ERR_RSA_BAD_INPUT_DATA;
+        goto cleanup;
+    }
+
     /*
      * find primes P and Q with Q < P so that:
      * 1.  |P-Q| > 2^( nbits / 2 - 100 )
@@ -632,7 +635,9 @@
     if( ret != 0 )
     {
         mbedtls_rsa_free( ctx );
-        return( MBEDTLS_ERR_RSA_KEY_GEN_FAILED + ret );
+        if( ( -ret & ~0x7f ) == 0 )
+            ret = MBEDTLS_ERR_RSA_KEY_GEN_FAILED + ret;
+        return( ret );
     }
 
     return( 0 );