Parse RSAPrivateKey with CBS.

This removes the version field from RSA and instead handles versioning
as part of parsing. (As a bonus, we now correctly limit multi-prime RSA
to version 1 keys.)

Most consumers are also converted. old_rsa_priv_{de,en}code are left
alone for now. Those hooks are passed in parameters which match the old
d2i/i2d pattern (they're only used in d2i_PrivateKey and
i2d_PrivateKey).

Include a test which, among other things, checks that public keys being
serialized as private keys are handled properly.

BUG=499653

Change-Id: Icdd5f0382c4a84f9c8867024f29756e1a306ba08
Reviewed-on: https://boringssl-review.googlesource.com/5273
Reviewed-by: Adam Langley <agl@google.com>

3 files changed, 24 insertions, 31 deletions

diff --git a/crypto/evp/evp_extra_test.cc b/crypto/evp/evp_extra_test.cc
index 674547d2..9c955fa4 100644
--- a/crypto/evp/evp_extra_test.cc
+++ b/crypto/evp/evp_extra_test.cc
@@ -322,8 +322,8 @@ static const uint8_t kExampleBadECKeyDER[] = {
 };
 
 static ScopedEVP_PKEY LoadExampleRSAKey() {
-  const uint8_t *derp = kExampleRSAKeyDER;
-  ScopedRSA rsa(d2i_RSAPrivateKey(nullptr, &derp, sizeof(kExampleRSAKeyDER)));
+  ScopedRSA rsa(RSA_private_key_from_bytes(kExampleRSAKeyDER,
+                                           sizeof(kExampleRSAKeyDER)));
   if (!rsa) {
     return nullptr;
   }
diff --git a/crypto/evp/internal.h b/crypto/evp/internal.h
index 8b755d0e..4e1e9c91 100644
--- a/crypto/evp/internal.h
+++ b/crypto/evp/internal.h
@@ -114,8 +114,8 @@ struct evp_pkey_asn1_method_st {
   int (*pkey_size)(const EVP_PKEY *pk);
   int (*pkey_bits)(const EVP_PKEY *pk);
 
-  int (*param_decode)(EVP_PKEY *pkey, const unsigned char **pder, int derlen);
-  int (*param_encode)(const EVP_PKEY *pkey, unsigned char **pder);
+  int (*param_decode)(EVP_PKEY *pkey, const uint8_t **pder, int derlen);
+  int (*param_encode)(const EVP_PKEY *pkey, uint8_t **pder);
   int (*param_missing)(const EVP_PKEY *pk);
   int (*param_copy)(EVP_PKEY *to, const EVP_PKEY *from);
   int (*param_cmp)(const EVP_PKEY *a, const EVP_PKEY *b);
@@ -129,9 +129,9 @@ struct evp_pkey_asn1_method_st {
 
   /* Legacy functions for old PEM */
 
-  int (*old_priv_decode)(EVP_PKEY *pkey, const unsigned char **pder,
+  int (*old_priv_decode)(EVP_PKEY *pkey, const uint8_t **pder,
                          int derlen);
-  int (*old_priv_encode)(const EVP_PKEY *pkey, unsigned char **pder);
+  int (*old_priv_encode)(const EVP_PKEY *pkey, uint8_t **pder);
 
   /* Converting parameters to/from AlgorithmIdentifier (X509_ALGOR). */
   int (*digest_verify_init_from_algorithm)(EVP_MD_CTX *ctx,
@@ -242,23 +242,23 @@ struct evp_pkey_method_st {
   int (*keygen)(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey);
 
   int (*sign_init)(EVP_PKEY_CTX *ctx);
-  int (*sign)(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
-              const unsigned char *tbs, size_t tbslen);
+  int (*sign)(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
+              const uint8_t *tbs, size_t tbslen);
 
   int (*verify_init)(EVP_PKEY_CTX *ctx);
-  int (*verify)(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
-                const unsigned char *tbs, size_t tbslen);
+  int (*verify)(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t siglen,
+                const uint8_t *tbs, size_t tbslen);
 
   int (*encrypt_init)(EVP_PKEY_CTX *ctx);
-  int (*encrypt)(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
-                 const unsigned char *in, size_t inlen);
+  int (*encrypt)(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
+                 const uint8_t *in, size_t inlen);
 
   int (*decrypt_init)(EVP_PKEY_CTX *ctx);
-  int (*decrypt)(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
-                 const unsigned char *in, size_t inlen);
+  int (*decrypt)(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
+                 const uint8_t *in, size_t inlen);
 
   int (*derive_init)(EVP_PKEY_CTX *ctx);
-  int (*derive)(EVP_PKEY_CTX *ctx, unsigned char *key, size_t *keylen);
+  int (*derive)(EVP_PKEY_CTX *ctx, uint8_t *key, size_t *keylen);
 
   int (*ctrl)(EVP_PKEY_CTX *ctx, int type, int p1, void *p2);
   int (*ctrl_str)(EVP_PKEY_CTX *ctx, const char *type, const char *value);
diff --git a/crypto/evp/p_rsa_asn1.c b/crypto/evp/p_rsa_asn1.c
index 6fcf2ebf..9166c5a9 100644
--- a/crypto/evp/p_rsa_asn1.c
+++ b/crypto/evp/p_rsa_asn1.c
@@ -107,20 +107,16 @@ static int rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b) {
 }
 
 static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) {
-  uint8_t *rk = NULL;
-  int rklen;
-
-  rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
-
-  if (rklen <= 0) {
-    OPENSSL_PUT_ERROR(EVP, rsa_priv_encode, ERR_R_MALLOC_FAILURE);
+  uint8_t *encoded;
+  size_t encoded_len;
+  if (!RSA_private_key_to_bytes(&encoded, &encoded_len, pkey->pkey.rsa)) {
     return 0;
   }
 
   /* TODO(fork): const correctness in next line. */
   if (!PKCS8_pkey_set0(p8, (ASN1_OBJECT *)OBJ_nid2obj(NID_rsaEncryption), 0,
-                       V_ASN1_NULL, NULL, rk, rklen)) {
-    OPENSSL_free(rk);
+                       V_ASN1_NULL, NULL, encoded, encoded_len)) {
+    OPENSSL_free(encoded);
     OPENSSL_PUT_ERROR(EVP, rsa_priv_encode, ERR_R_MALLOC_FAILURE);
     return 0;
   }
@@ -131,14 +127,12 @@ static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) {
 static int rsa_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8) {
   const uint8_t *p;
   int pklen;
-  RSA *rsa;
-
   if (!PKCS8_pkey_get0(NULL, &p, &pklen, NULL, p8)) {
     OPENSSL_PUT_ERROR(EVP, rsa_priv_decode, ERR_R_MALLOC_FAILURE);
     return 0;
   }
 
-  rsa = d2i_RSAPrivateKey(NULL, &p, pklen);
+  RSA *rsa = RSA_private_key_from_bytes(p, pklen);
   if (rsa == NULL) {
     OPENSSL_PUT_ERROR(EVP, rsa_priv_decode, ERR_R_RSA_LIB);
     return 0;
@@ -227,8 +221,7 @@ static int do_rsa_print(BIO *out, const RSA *rsa, int off,
   }
 
   if (include_private && rsa->d) {
-    if (BIO_printf(out, "Private-Key: (%d bit)\nversion: %ld\n", mod_len,
-                   rsa->version) <= 0) {
+    if (BIO_printf(out, "Private-Key: (%d bit)\n", mod_len) <= 0) {
       goto err;
     }
     str = "modulus:";
@@ -442,7 +435,7 @@ static int rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg,
   return 1;
 }
 
-static int old_rsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder,
+static int old_rsa_priv_decode(EVP_PKEY *pkey, const uint8_t **pder,
                                int derlen) {
   RSA *rsa = d2i_RSAPrivateKey(NULL, pder, derlen);
   if (rsa == NULL) {
@@ -453,7 +446,7 @@ static int old_rsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder,
   return 1;
 }
 
-static int old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder) {
+static int old_rsa_priv_encode(const EVP_PKEY *pkey, uint8_t **pder) {
   return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
 }