Improve RSA-PSS test coverage around length bounds.

One test case is commented out, to be fixed in a follow-up.

Change-Id: I543c7f54e63837c6e8088fdcbb03226e0144b2e5
Reviewed-on: https://boringssl-review.googlesource.com/14320
Reviewed-by: Steven Valdez <svaldez@google.com>
Commit-Queue: Steven Valdez <svaldez@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
diff --git a/crypto/evp/evp_test.cc b/crypto/evp/evp_test.cc
index d03c61f..6ca250e 100644
--- a/crypto/evp/evp_test.cc
+++ b/crypto/evp/evp_test.cc
@@ -224,9 +224,8 @@
   }
   EVP_PKEY *key = (*key_map)[key_name].get();
 
-  std::vector<uint8_t> input, output;
-  if (!t->GetBytes(&input, "Input") ||
-      !t->GetBytes(&output, "Output")) {
+  std::vector<uint8_t> input;
+  if (!t->GetBytes(&input, "Input")) {
     return false;
   }
 
@@ -263,7 +262,9 @@
   }
 
   if (t->GetType() == "Verify") {
-    if (!EVP_PKEY_verify(ctx.get(), output.data(), output.size(), input.data(),
+    std::vector<uint8_t> output;
+    if (!t->GetBytes(&output, "Output") ||
+        !EVP_PKEY_verify(ctx.get(), output.data(), output.size(), input.data(),
                          input.size())) {
       // ECDSA sometimes doesn't push an error code. Push one on the error queue
       // so it's distinguishable from other errors.
@@ -274,7 +275,7 @@
   }
 
   size_t len;
-  std::vector<uint8_t> actual;
+  std::vector<uint8_t> actual, output;
   if (!key_op(ctx.get(), nullptr, &len, input.data(), input.size())) {
     return false;
   }
@@ -283,7 +284,8 @@
     return false;
   }
   actual.resize(len);
-  if (!t->ExpectBytesEqual(output.data(), output.size(), actual.data(), len)) {
+  if (!t->GetBytes(&output, "Output") ||
+      !t->ExpectBytesEqual(output.data(), output.size(), actual.data(), len)) {
     return false;
   }
   return true;
diff --git a/crypto/evp/evp_tests.txt b/crypto/evp/evp_tests.txt
index 9fe143c..1d57bd5 100644
--- a/crypto/evp/evp_tests.txt
+++ b/crypto/evp/evp_tests.txt
@@ -22,6 +22,11 @@
 Input = 3083000122300d06092a864886f70d01010105000382010f003082010a0282010100cd0081ea7b2ae1ea06d59f7c73d9ffb94a09615c2e4ba7c636cef08dd3533ec3185525b015c769b99a77d6725bf9c3532a9b6e5f6627d5fb85160768d3dda9cbd35974511717dc3d309d2fc47ee41f97e32adb7f9dd864a1c4767a666ecd71bc1aacf5e7517f4b38594fea9b05e42d5ada9912008013e45316a4d9bb8ed086b88d28758bacaf922d46a868b485d239c9baeb0e2b64592710f42b2d1ea0a4b4802c0becab328f8a68b0073bdb546feea9809d2849912b390c1532bc7e29c7658f8175fae46f34332ff87bcab3e40649b98577869da0ea718353f0722754886913648760d122be676e0fc483dd20ffc31bda96a31966c9aa2e75ad03de47e1c44f0203010001
 Error = DECODE_ERROR
 
+# RSA 512 bit key.
+PrivateKey = RSA-512
+Type = RSA
+Input = 30820154020100300d06092a864886f70d01010105000482013e3082013a020100024100dd20403d976a38c9d79152d87b5c8e9f05033eadd7b7de709bf5b0c4a5182a97d18483526b02362b992e154a9f37faa396ca2685cdab8fec09877ebe705f4dd70203010001024055bebcca655d7e39de8a6eaa9d636db682161907064039544755c53eeb99ec618c03a210dbc61471eaba10c5c365c9726d6b7a96f54d455f7d168d49367270e1022100f21a05d9fd6817301ce49ce10448f9bdd44f5ef5b7557cd7d83155db46382ae7022100e9d1f7157783db2feab1936954ddc4e83aa365695868144cda1be6813b61d791022100d6001eb0040920860ce41fafdf23ca6dfbdf74e6e9f98cf3164cf5c16f9e727d02206f6f73f4b52b10517be6f9bc5f87fa0a3bb817e2e711636b651f9af1c85d4f21022063eff2e57f5b4ca20342cfe793e25526624e3692f192461f9e1ce7f13f2d72c8
+
 # EC P-256 key
 PrivateKey = P-256
 Type = EC
@@ -96,14 +101,12 @@
 Sign = RSA-2048
 Digest = SHA1
 Input = "0123456789ABCDEF12345"
-Output =
 Error = INVALID_DIGEST_LENGTH
 
 # Digest too short
 Sign = RSA-2048
 Digest = SHA1
 Input = "0123456789ABCDEF12345"
-Output =
 Error = INVALID_DIGEST_LENGTH
 
 # Mismatched digest
@@ -233,6 +236,56 @@
 # If SHA-384, this input happens fail to recover the salt length altogether.
 Error = SLEN_RECOVERY_FAILED
 
+# The salt length is too large for the modulus (signing).
+Sign = RSA-2048
+RSAPadding = PSS
+PSSSaltLength = 223
+Digest = SHA256
+Input = "0123456789ABCDEF0123456789ABCDEF"
+Error = DATA_TOO_LARGE_FOR_KEY_SIZE
+
+# The salt length is too large for the modulus (verifying).
+Verify = RSA-2048
+RSAPadding = PSS
+PSSSaltLength = 223
+Digest = SHA256
+Input = "0123456789ABCDEF0123456789ABCDEF"
+Output = 4de433d5844043ef08d354da03cb29068780d52706d7d1e4d50efb7d58c9d547d83a747ddd0635a96b28f854e50145518482cb49e963054621b53c60c498d07c16e9c2789c893cf38d4d86900de71bde463bd2761d1271e358c7480a1ac0bab930ddf39602ad1bc165b5d7436b516b7a7858e8eb7ab1c420eeb482f4d207f0e462b1724959320a084e13848d11d10fb593e66bf680bf6d3f345fc3e9c3de60abbac37e1c6ec80a268c8d9fc49626c679097aa690bc1aa662b95eb8db70390861aa0898229f9349b4b5fdd030d4928c47084708a933144be23bd3c6e661b85b2c0ef9ed36d498d5b7320e8194d363d4ad478c059bae804181965e0b81b663158a
+Error = DATA_TOO_LARGE
+
+# The hash is too large for the modulus (signing).
+Sign = RSA-512
+RSAPadding = PSS
+PSSSaltLength = 0
+Digest = SHA512
+Input = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"
+Error = DATA_TOO_LARGE_FOR_KEY_SIZE
+
+Sign = RSA-512
+RSAPadding = PSS
+PSSSaltLength = -2
+Digest = SHA512
+Input = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"
+Error = DATA_TOO_LARGE_FOR_KEY_SIZE
+
+# The hash is too large for the modulus (verifying).
+Verify = RSA-512
+RSAPadding = PSS
+PSSSaltLength = 0
+Digest = SHA512
+Input = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"
+Output = aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Error = DATA_TOO_LARGE
+
+# TODO(davidben): Add this as a regression test once upstream's fix is imported.
+# Verify = RSA-512
+# RSAPadding = PSS
+# PSSSaltLength = -2
+# Digest = SHA512
+# Input = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"
+# Output = 457001d9ca50a93385fc5ec721c9dbbe7a0f2e9e4a2f846a30a8811dde66347b83901c7492039243537c7a667fafffd69049bcbd36afd0010d9b425e2d8785c1
+# Error = DATA_TOO_LARGE
+
 
 # RSA decrypt
 
@@ -243,7 +296,6 @@
 # Corrupted ciphertext
 Decrypt = RSA-2048
 Input = 550af55a2904e7b9762352f8fb7fa235a9cb053aacb2d5fcb8ca48453cb2ee3619746c701abf2d4cc67003471a187900b05aa812bd25ed05c675dfc8c97a24a7bf49bd6214992cad766d05a9a2b57b74f26a737e0237b8b76c45f1f226a836d7cfbc75ba999bdbe48dbc09227aa46c88f21dccba7840141ad5a5d71fd122e6bd6ac3e564780dfe623fc1ca9b995a6037bf0bbd43b205a84ac5444f34202c05ce9113087176432476576de6ffff9a52ea57c08be3ec2f49676cb8e12f762ac71fa3c321e00ac988910c85ff52f93825666ce0d40ffaa0592078919d4493f46d95ccf76364c6d57760dd0b64805f9afc76a2365a5575ca301d5103f0ea76cb9a79
-Output = "Hello World"
 Error = PKCS_DECODING_ERROR
 
 # OAEP padding
@@ -256,7 +308,6 @@
 Decrypt = RSA-2048
 RSAPadding = OAEP
 Input = 458708dfbd42a1297ce7a9c86c7087ab80b1754810929b89c5107ca55368587686986fce94d86cc1595b3fb736223a656ec0f34d18ba1cc5665593610f56c58e26b272d584f3d983a5c91085700755aebd921fb280bba3eda7046ec07b43e7298e52d59edc92be4639a8ce08b2f85976ecf6d98cc469eeb9d5d8e2a32ea8a6626edafe1038b3df455668a9f3c77cad8b92fb872e00058c3d2a7ede1a1f03fc5622084ae04d9d24f6bf0995c58d35b93b699b9763595e123f2ab0863cc9229eb290e2ede7715c7a8f39e0b9a3e2e1b56ebb62f1cbfbb5986fb212ebd785b83d01d968b11d1756c7337f70c1f1a63bff03608e24f3a2fd44e67f832a8701c5d5ac
-Output = "Hello World"
 Error = OAEP_DECODING_ERROR