Clean up signature algorithm logic.

With public keys reliably extractable from SSL_PRIVATE_KEY_METHOD keys,
we can share the pkey/sigalg check between signing and verifying.

BUG=188

Change-Id: Ieb9382807781e48ffed720b27f450847d3fca788
Reviewed-on: https://boringssl-review.googlesource.com/14566
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/ssl/internal.h b/ssl/internal.h
index 3ebac72..99dba0f 100644
--- a/ssl/internal.h
+++ b/ssl/internal.h
@@ -565,9 +565,9 @@
                                                        size_t max_out);
 
 /* ssl_private_key_supports_signature_algorithm returns one if |hs|'s private
- * key supports |signature_algorithm| and zero otherwise. */
+ * key supports |sigalg| and zero otherwise. */
 int ssl_private_key_supports_signature_algorithm(SSL_HANDSHAKE *hs,
-                                                 uint16_t signature_algorithm);
+                                                 uint16_t sigalg);
 
 /* ssl_public_key_verify verifies that the |signature| is valid for the public
  * key |pkey| and input |in|, using the |signature_algorithm| specified. */
diff --git a/ssl/ssl_privkey.c b/ssl/ssl_privkey.c
index 82842d0..7bb8b0a 100644
--- a/ssl/ssl_privkey.c
+++ b/ssl/ssl_privkey.c
@@ -66,6 +66,7 @@
 #include <openssl/type_check.h>
 
 #include "internal.h"
+#include "../crypto/internal.h"
 
 
 int ssl_is_key_type_supported(int key_type) {
@@ -294,139 +295,115 @@
   return 1;
 }
 
+typedef struct {
+  uint16_t sigalg;
+  int pkey_type;
+  int curve;
+  const EVP_MD *(*digest_func)(void);
+  char is_rsa_pss;
+} SSL_SIGNATURE_ALGORITHM;
+
+static const SSL_SIGNATURE_ALGORITHM kSignatureAlgorithms[] = {
+    {SSL_SIGN_RSA_PKCS1_MD5_SHA1, EVP_PKEY_RSA, NID_undef, &EVP_md5_sha1, 0},
+    {SSL_SIGN_RSA_PKCS1_SHA1, EVP_PKEY_RSA, NID_undef, &EVP_sha1, 0},
+    {SSL_SIGN_RSA_PKCS1_SHA256, EVP_PKEY_RSA, NID_undef, &EVP_sha256, 0},
+    {SSL_SIGN_RSA_PKCS1_SHA384, EVP_PKEY_RSA, NID_undef, &EVP_sha384, 0},
+    {SSL_SIGN_RSA_PKCS1_SHA512, EVP_PKEY_RSA, NID_undef, &EVP_sha512, 0},
+
+    {SSL_SIGN_RSA_PSS_SHA256, EVP_PKEY_RSA, NID_undef, &EVP_sha256, 1},
+    {SSL_SIGN_RSA_PSS_SHA384, EVP_PKEY_RSA, NID_undef, &EVP_sha384, 1},
+    {SSL_SIGN_RSA_PSS_SHA512, EVP_PKEY_RSA, NID_undef, &EVP_sha512, 1},
+
+    {SSL_SIGN_ECDSA_SHA1, EVP_PKEY_EC, NID_undef, &EVP_sha1, 0},
+    {SSL_SIGN_ECDSA_SECP256R1_SHA256, EVP_PKEY_EC, NID_X9_62_prime256v1,
+     &EVP_sha256, 0},
+    {SSL_SIGN_ECDSA_SECP384R1_SHA384, EVP_PKEY_EC, NID_secp384r1, &EVP_sha384,
+     0},
+    {SSL_SIGN_ECDSA_SECP521R1_SHA512, EVP_PKEY_EC, NID_secp521r1, &EVP_sha512,
+     0},
+};
+
+static const SSL_SIGNATURE_ALGORITHM *get_signature_algorithm(uint16_t sigalg) {
+  for (size_t i = 0; i < OPENSSL_ARRAY_SIZE(kSignatureAlgorithms); i++) {
+    if (kSignatureAlgorithms[i].sigalg == sigalg) {
+      return &kSignatureAlgorithms[i];
+    }
+  }
+  return NULL;
+}
+
 int ssl_has_private_key(const SSL *ssl) {
   return ssl->cert->privatekey != NULL || ssl->cert->key_method != NULL;
 }
 
-static int is_rsa_pkcs1(const EVP_MD **out_md, uint16_t sigalg) {
-  switch (sigalg) {
-    case SSL_SIGN_RSA_PKCS1_MD5_SHA1:
-      *out_md = EVP_md5_sha1();
-      return 1;
-    case SSL_SIGN_RSA_PKCS1_SHA1:
-      *out_md = EVP_sha1();
-      return 1;
-    case SSL_SIGN_RSA_PKCS1_SHA256:
-      *out_md = EVP_sha256();
-      return 1;
-    case SSL_SIGN_RSA_PKCS1_SHA384:
-      *out_md = EVP_sha384();
-      return 1;
-    case SSL_SIGN_RSA_PKCS1_SHA512:
-      *out_md = EVP_sha512();
-      return 1;
-    default:
-      return 0;
+static int pkey_supports_algorithm(const SSL *ssl, EVP_PKEY *pkey,
+                                   uint16_t sigalg) {
+  const SSL_SIGNATURE_ALGORITHM *alg = get_signature_algorithm(sigalg);
+  if (alg == NULL ||
+      EVP_PKEY_id(pkey) != alg->pkey_type) {
+    return 0;
   }
+
+  if (ssl3_protocol_version(ssl) >= TLS1_3_VERSION) {
+    /* RSA keys may only be used with RSA-PSS. */
+    if (alg->pkey_type == EVP_PKEY_RSA && !alg->is_rsa_pss) {
+      return 0;
+    }
+
+    /* EC keys have a curve requirement. */
+    if (alg->pkey_type == EVP_PKEY_EC &&
+        (alg->curve == NID_undef ||
+         EC_GROUP_get_curve_name(
+             EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(pkey))) != alg->curve)) {
+      return 0;
+    }
+  }
+
+  return 1;
 }
 
-static int is_rsa_pss(const EVP_MD **out_md, uint16_t sigalg) {
-  switch (sigalg) {
-    case SSL_SIGN_RSA_PSS_SHA256:
-      *out_md = EVP_sha256();
-      return 1;
-    case SSL_SIGN_RSA_PSS_SHA384:
-      *out_md = EVP_sha384();
-      return 1;
-    case SSL_SIGN_RSA_PSS_SHA512:
-      *out_md = EVP_sha512();
-      return 1;
-    default:
-      return 0;
+static int setup_ctx(SSL *ssl, EVP_PKEY_CTX *ctx, uint16_t sigalg) {
+  if (!pkey_supports_algorithm(ssl, EVP_PKEY_CTX_get0_pkey(ctx), sigalg)) {
+    OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
+    return 0;
   }
-}
 
-static int is_ecdsa(int *out_curve, const EVP_MD **out_md, uint16_t sigalg) {
-  switch (sigalg) {
-    case SSL_SIGN_ECDSA_SHA1:
-      *out_curve = NID_undef;
-      *out_md = EVP_sha1();
-      return 1;
-    case SSL_SIGN_ECDSA_SECP256R1_SHA256:
-      *out_curve = NID_X9_62_prime256v1;
-      *out_md = EVP_sha256();
-      return 1;
-    case SSL_SIGN_ECDSA_SECP384R1_SHA384:
-      *out_curve = NID_secp384r1;
-      *out_md = EVP_sha384();
-      return 1;
-    case SSL_SIGN_ECDSA_SECP521R1_SHA512:
-      *out_curve = NID_secp521r1;
-      *out_md = EVP_sha512();
-      return 1;
-    default:
-      return 0;
+  const SSL_SIGNATURE_ALGORITHM *alg = get_signature_algorithm(sigalg);
+  if (alg->digest_func != NULL &&
+      !EVP_PKEY_CTX_set_signature_md(ctx, alg->digest_func())) {
+    return 0;
   }
-}
 
-static int setup_ctx(SSL *ssl, EVP_PKEY_CTX *ctx,
-                     uint16_t signature_algorithm) {
-  EVP_PKEY *pkey = EVP_PKEY_CTX_get0_pkey(ctx);
-
-  const EVP_MD *md;
-  if (is_rsa_pkcs1(&md, signature_algorithm) &&
-      ssl3_protocol_version(ssl) < TLS1_3_VERSION) {
-    if (pkey->type != EVP_PKEY_RSA) {
-      OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
+  if (alg->is_rsa_pss) {
+    if (!EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PSS_PADDING) ||
+        !EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, -1 /* salt len = hash len */)) {
       return 0;
     }
-
-    return EVP_PKEY_CTX_set_signature_md(ctx, md);
   }
 
-  if (is_rsa_pss(&md, signature_algorithm)) {
-    if (pkey->type != EVP_PKEY_RSA) {
-      OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
-      return 0;
-    }
-
-    return EVP_PKEY_CTX_set_signature_md(ctx, md) &&
-           EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PSS_PADDING) &&
-           EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, -1 /* salt len = hash len */);
-  }
-
-  int curve;
-  if (is_ecdsa(&curve, &md, signature_algorithm)) {
-    EC_KEY *ec_key = EVP_PKEY_get0_EC_KEY(pkey);
-    if (ec_key == NULL) {
-      OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
-      return 0;
-    }
-
-    /* In TLS 1.3, the curve is also specified by the signature algorithm. */
-    if (ssl3_protocol_version(ssl) >= TLS1_3_VERSION &&
-        (curve == NID_undef ||
-         EC_GROUP_get_curve_name(EC_KEY_get0_group(ec_key)) != curve)) {
-      OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
-      return 0;
-    }
-
-    return EVP_PKEY_CTX_set_signature_md(ctx, md);
-  }
-
-  OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
-  return 0;
+  return 1;
 }
 
 enum ssl_private_key_result_t ssl_private_key_sign(
     SSL *ssl, uint8_t *out, size_t *out_len, size_t max_out,
-    uint16_t signature_algorithm, const uint8_t *in, size_t in_len) {
+    uint16_t sigalg, const uint8_t *in, size_t in_len) {
   if (ssl->cert->key_method != NULL) {
     if (ssl->cert->key_method->sign != NULL) {
-      return ssl->cert->key_method->sign(ssl, out, out_len, max_out,
-                                         signature_algorithm, in, in_len);
+      return ssl->cert->key_method->sign(ssl, out, out_len, max_out, sigalg, in,
+                                         in_len);
     }
 
     /* TODO(davidben): Remove support for |sign_digest|-only
      * |SSL_PRIVATE_KEY_METHOD|s. */
-    const EVP_MD *md;
-    int curve;
-    if (!is_rsa_pkcs1(&md, signature_algorithm) &&
-        !is_ecdsa(&curve, &md, signature_algorithm)) {
+    const SSL_SIGNATURE_ALGORITHM *alg = get_signature_algorithm(sigalg);
+    if (alg == NULL ||
+        (alg->pkey_type != EVP_PKEY_EC && alg->pkey_type != EVP_PKEY_RSA) ||
+        alg->is_rsa_pss) {
       OPENSSL_PUT_ERROR(SSL, SSL_R_UNSUPPORTED_PROTOCOL_FOR_CUSTOM_KEY);
       return ssl_private_key_failure;
     }
 
+    const EVP_MD *md = alg->digest_func();
     uint8_t hash[EVP_MAX_MD_SIZE];
     unsigned hash_len;
     if (!EVP_Digest(in, in_len, hash, &hash_len, md, NULL)) {
@@ -441,7 +418,7 @@
   EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(ssl->cert->privatekey, NULL);
   int ret = ctx != NULL &&
             EVP_PKEY_sign_init(ctx) &&
-            setup_ctx(ssl, ctx, signature_algorithm) &&
+            setup_ctx(ssl, ctx, sigalg) &&
             EVP_PKEY_sign_message(ctx, out, out_len, in, in_len);
   EVP_PKEY_CTX_free(ctx);
   return ret ? ssl_private_key_success : ssl_private_key_failure;
@@ -490,53 +467,31 @@
 }
 
 int ssl_private_key_supports_signature_algorithm(SSL_HANDSHAKE *hs,
-                                                 uint16_t signature_algorithm) {
+                                                 uint16_t sigalg) {
   SSL *const ssl = hs->ssl;
-  int type = EVP_PKEY_id(hs->local_pubkey);
-  const EVP_MD *md;
-  if (is_rsa_pkcs1(&md, signature_algorithm) &&
-      ssl3_protocol_version(ssl) < TLS1_3_VERSION) {
-    return type == EVP_PKEY_RSA;
+  if (!pkey_supports_algorithm(ssl, hs->local_pubkey, sigalg)) {
+    return 0;
   }
 
-  int curve;
-  if (is_ecdsa(&curve, &md, signature_algorithm)) {
-    if (type != EVP_PKEY_EC) {
-      return 0;
-    }
-
-    /* Prior to TLS 1.3, ECDSA curves did not match the signature algorithm. */
-    if (ssl3_protocol_version(ssl) < TLS1_3_VERSION) {
-      return 1;
-    }
-
-    return curve != NID_undef &&
-           EC_GROUP_get_curve_name(EC_KEY_get0_group(
-               EVP_PKEY_get0_EC_KEY(hs->local_pubkey))) == curve;
-  }
-
-  if (is_rsa_pss(&md, signature_algorithm)) {
-    if (type != EVP_PKEY_RSA) {
-      return 0;
-    }
-
+  const SSL_SIGNATURE_ALGORITHM *alg = get_signature_algorithm(sigalg);
+  if (alg->is_rsa_pss) {
     /* Ensure the RSA key is large enough for the hash. RSASSA-PSS requires that
      * emLen be at least hLen + sLen + 2. Both hLen and sLen are the size of the
      * hash in TLS. Reasonable RSA key sizes are large enough for the largest
      * defined RSASSA-PSS algorithm, but 1024-bit RSA is slightly too large for
      * SHA-512. 1024-bit RSA is sometimes used for test credentials, so check
      * the size to fall back to another algorithm. */
-    if ((size_t)EVP_PKEY_size(hs->local_pubkey) < 2 * EVP_MD_size(md) + 2) {
+    if ((size_t)EVP_PKEY_size(hs->local_pubkey) <
+        2 * EVP_MD_size(alg->digest_func()) + 2) {
       return 0;
     }
 
-    /* RSA-PSS is only supported by message-based private keys. */
+    /* RSA-PSS is only supported by message-based private keys.
+     * TODO(davidben): Remove this check when sign_digest is gone. */
     if (ssl->cert->key_method != NULL && ssl->cert->key_method->sign == NULL) {
       return 0;
     }
-
-    return 1;
   }
 
-  return 0;
+  return 1;
 }