Parse RSAPrivateKey with CBS.

This removes the version field from RSA and instead handles versioning
as part of parsing. (As a bonus, we now correctly limit multi-prime RSA
to version 1 keys.)

Most consumers are also converted. old_rsa_priv_{de,en}code are left
alone for now. Those hooks are passed in parameters which match the old
d2i/i2d pattern (they're only used in d2i_PrivateKey and
i2d_PrivateKey).

Include a test which, among other things, checks that public keys being
serialized as private keys are handled properly.

BUG=499653

Change-Id: Icdd5f0382c4a84f9c8867024f29756e1a306ba08
Reviewed-on: https://boringssl-review.googlesource.com/5273
Reviewed-by: Adam Langley <agl@google.com>

1 files changed, 53 insertions, 1 deletions

diff --git a/crypto/rsa/rsa_test.cc b/crypto/rsa/rsa_test.cc
index e0fa77ce..9d44e2c8 100644
--- a/crypto/rsa/rsa_test.cc
+++ b/crypto/rsa/rsa_test.cc
@@ -742,6 +742,57 @@ static bool TestRecoverCRTParams() {
   return true;
 }
 
+static bool TestASN1() {
+  // Test that private keys may be decoded.
+  ScopedRSA rsa(RSA_private_key_from_bytes(kKey1, sizeof(kKey1) - 1));
+  if (!rsa) {
+    return false;
+  }
+
+  // Test that the serialization round-trips.
+  uint8_t *der;
+  size_t der_len;
+  if (!RSA_private_key_to_bytes(&der, &der_len, rsa.get())) {
+    return false;
+  }
+  ScopedOpenSSLBytes delete_der(der);
+  if (der_len != sizeof(kKey1) - 1 || memcmp(der, kKey1, der_len) != 0) {
+    return false;
+  }
+
+  // Test that serializing public keys works.
+  if (!RSA_public_key_to_bytes(&der, &der_len, rsa.get())) {
+    return false;
+  }
+  delete_der.reset(der);
+
+  // Public keys may be parsed back out.
+  rsa.reset(RSA_public_key_from_bytes(der, der_len));
+  if (!rsa || rsa->p != NULL || rsa->q != NULL) {
+    return false;
+  }
+
+  // Serializing the result round-trips.
+  uint8_t *der2;
+  size_t der2_len;
+  if (!RSA_public_key_to_bytes(&der2, &der2_len, rsa.get())) {
+    return false;
+  }
+  ScopedOpenSSLBytes delete_der2(der2);
+  if (der_len != der2_len || memcmp(der, der2, der_len) != 0) {
+    return false;
+  }
+
+  // Public keys cannot be serialized as private keys.
+  if (RSA_private_key_to_bytes(&der, &der_len, rsa.get())) {
+    OPENSSL_free(der);
+    return false;
+  }
+  ERR_clear_error();
+
+  return true;
+}
+
 int main(int argc, char *argv[]) {
   CRYPTO_library_init();
 
@@ -763,7 +814,8 @@ int main(int argc, char *argv[]) {
       !TestMultiPrimeKey(6, kSixPrimeKey, sizeof(kSixPrimeKey) - 1,
                             kSixPrimeEncryptedMessage,
                             sizeof(kSixPrimeEncryptedMessage)) ||
-      !TestMultiPrimeKeygen()) {
+      !TestMultiPrimeKeygen() ||
+      !TestASN1()) {
     return 1;
   }