commit b1ffe0b36a185d45b474fb29102c52989ac818c5
author Steven Valdez <svaldez@google.com> Thu Apr 20 10:45:25 2017 -0400
committer Adam Langley <agl@google.com> Fri Apr 21 22:38:31 2017 +0000
tree 0d9fc5e24bd36637009d0910dadf008e4616401d
parent 7ce349ef2685bb6772fc6a0ea62e093ff537a389

Add primality checking for RSA_check_fips.

This also fixes the comments regarding BN_prime_checks to match the
security level guarantees provided by BN_prime_checks.

Change-Id: I8032e88680bf51e8876e134b4253ed26c2072617
Reviewed-on: https://boringssl-review.googlesource.com/15304
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: Adam Langley <agl@google.com>

crypto/rsa/rsa.c

diff --git a/crypto/rsa/rsa.c b/crypto/rsa/rsa.c
index cc4aa75..aec8935 100644
--- a/crypto/rsa/rsa.c
+++ b/crypto/rsa/rsa.c
@@ -667,14 +667,16 @@
   int ret = 1;
 
   /* Perform partial public key validation of RSA keys (SP 800-89 5.3.3). */
-  /* TODO(svaldez): Check that n is composite and not a power of a prime using
-   * extended Miller-Rabin. */
+  enum bn_primality_result_t primality_result;
   if (BN_num_bits(key->e) <= 16 ||
       BN_num_bits(key->e) > 256 ||
       !BN_is_odd(key->n) ||
       !BN_is_odd(key->e) ||
       !BN_gcd(&small_gcd, key->n, &kSmallFactors, ctx) ||
-      !BN_is_one(&small_gcd)) {
+      !BN_is_one(&small_gcd) ||
+      !BN_enhanced_miller_rabin_primality_test(&primality_result, key->n,
+                                               BN_prime_checks, ctx, NULL) ||
+      primality_result != bn_non_prime_power_composite) {
     OPENSSL_PUT_ERROR(RSA, RSA_R_PUBLIC_KEY_VALIDATION_FAILED);
     ret = 0;
   }