Align EVP return values with BoringSSL convention.

Where possible, functions should return one for success and zero for
error. The use of additional negative values to indicate an error is,
itself, error prone.

This change fixes many EVP functions to remove the possibility of
negative return values. Existing code that is testing for <= 0 will
continue to function, although there is the possibility that some code
was differentiating between negative values (error) and zero (invalid
signature) for the verify functions and will now show the wrong error
message.

Change-Id: I982512596bb18a82df65861394dbd7487783bd3d
Reviewed-on: https://boringssl-review.googlesource.com/1333
Reviewed-by: Adam Langley <agl@google.com>

3 files changed, 138 insertions, 166 deletions

diff --git a/crypto/evp/evp_ctx.c b/crypto/evp/evp_ctx.c
index 1365dae7..1416d3a2 100644
--- a/crypto/evp/evp_ctx.c
+++ b/crypto/evp/evp_ctx.c
@@ -232,7 +232,6 @@ int EVP_PKEY_CTX_ctrl(EVP_PKEY_CTX *ctx, int keytype, int optype, int cmd,
 }
 
 int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx) {
-  int ret;
   if (!ctx || !ctx->pmeth || !ctx->pmeth->sign) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_sign_init,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
@@ -244,14 +243,12 @@ int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx) {
     return 1;
   }
 
-  ret = ctx->pmeth->sign_init(ctx);
-  /* TODO(fork): normalise the return value convention. */
-  if (ret <= 0) {
+  if (!ctx->pmeth->sign_init(ctx)) {
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
     return 0;
   }
 
-  return ret;
+  return 1;
 }
 
 int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len,
@@ -269,22 +266,21 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len,
 }
 
 int EVP_PKEY_verify_init(EVP_PKEY_CTX *ctx) {
-  int ret;
-
   if (!ctx || !ctx->pmeth || !ctx->pmeth->verify) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify_init,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   ctx->operation = EVP_PKEY_OP_VERIFY;
   if (!ctx->pmeth->verify_init) {
     return 1;
   }
-  ret = ctx->pmeth->verify_init(ctx);
-  if (ret <= 0) {
+  if (!ctx->pmeth->verify_init(ctx)) {
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
+    return 0;
   }
-  return ret;
+
+  return 1;
 }
 
 int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len,
@@ -292,31 +288,30 @@ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len,
   if (!ctx || !ctx->pmeth || !ctx->pmeth->verify) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   if (ctx->operation != EVP_PKEY_OP_VERIFY) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify, EVP_R_OPERATON_NOT_INITIALIZED);
-    return -1;
+    return 0;
   }
   return ctx->pmeth->verify(ctx, sig, sig_len, data, data_len);
 }
 
 int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx) {
-  int ret;
   if (!ctx || !ctx->pmeth || !ctx->pmeth->encrypt) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt_init,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   ctx->operation = EVP_PKEY_OP_ENCRYPT;
   if (!ctx->pmeth->encrypt_init) {
     return 1;
   }
-  ret = ctx->pmeth->encrypt_init(ctx);
-  if (ret <= 0) {
+  if (!ctx->pmeth->encrypt_init(ctx)) {
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
+    return 0;
   }
-  return ret;
+  return 1;
 }
 
 int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
@@ -324,32 +319,30 @@ int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
   if (!ctx || !ctx->pmeth || !ctx->pmeth->encrypt) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   if (ctx->operation != EVP_PKEY_OP_ENCRYPT) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt, EVP_R_OPERATON_NOT_INITIALIZED);
-    return -1;
+    return 0;
   }
   return ctx->pmeth->encrypt(ctx, out, outlen, in, inlen);
 }
 
 int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx) {
-  int ret;
-
   if (!ctx || !ctx->pmeth || !ctx->pmeth->decrypt) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt_init,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   ctx->operation = EVP_PKEY_OP_DECRYPT;
   if (!ctx->pmeth->decrypt_init) {
     return 1;
   }
-  ret = ctx->pmeth->decrypt_init(ctx);
-  if (ret <= 0) {
+  if (!ctx->pmeth->decrypt_init(ctx)) {
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
+    return 0;
   }
-  return ret;
+  return 1;
 }
 
 int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
@@ -357,31 +350,30 @@ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
   if (!ctx || !ctx->pmeth || !ctx->pmeth->decrypt) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   if (ctx->operation != EVP_PKEY_OP_DECRYPT) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt, EVP_R_OPERATON_NOT_INITIALIZED);
-    return -1;
+    return 0;
   }
   return ctx->pmeth->decrypt(ctx, out, outlen, in, inlen);
 }
 
 int EVP_PKEY_derive_init(EVP_PKEY_CTX *ctx) {
-  int ret;
   if (!ctx || !ctx->pmeth || !ctx->pmeth->derive) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_init,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   ctx->operation = EVP_PKEY_OP_DERIVE;
   if (!ctx->pmeth->derive_init) {
     return 1;
   }
-  ret = ctx->pmeth->derive_init(ctx);
-  if (ret <= 0) {
+  if (!ctx->pmeth->derive_init(ctx)) {
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
+    return 0;
   }
-  return ret;
+  return 1;
 }
 
 int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
@@ -391,20 +383,20 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
       !ctx->pmeth->ctrl) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   if (ctx->operation != EVP_PKEY_OP_DERIVE &&
       ctx->operation != EVP_PKEY_OP_ENCRYPT &&
       ctx->operation != EVP_PKEY_OP_DECRYPT) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer,
                       EVP_R_OPERATON_NOT_INITIALIZED);
-    return -1;
+    return 0;
   }
 
   ret = ctx->pmeth->ctrl(ctx, EVP_PKEY_CTRL_PEER_KEY, 0, peer);
 
   if (ret <= 0) {
-    return ret;
+    return 0;
   }
 
   if (ret == 2) {
@@ -413,12 +405,12 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
 
   if (!ctx->pkey) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_NO_KEY_SET);
-    return -1;
+    return 0;
   }
 
   if (ctx->pkey->type != peer->type) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_DIFFERENT_KEY_TYPES);
-    return -1;
+    return 0;
   }
 
   /* ran@cryptocom.ru: For clarity.  The error is if parameters in peer are
@@ -430,7 +422,7 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
       !EVP_PKEY_cmp_parameters(ctx->pkey, peer)) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer,
                       EVP_R_DIFFERENT_PARAMETERS);
-    return -1;
+    return 0;
   }
 
   if (ctx->peerkey) {
@@ -442,7 +434,7 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
 
   if (ret <= 0) {
     ctx->peerkey = NULL;
-    return ret;
+    return 0;
   }
 
   CRYPTO_add(&peer->references, 1, CRYPTO_LOCK_EVP_PKEY);
@@ -453,58 +445,55 @@ int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, uint8_t *key, size_t *out_key_len) {
   if (!ctx || !ctx->pmeth || !ctx->pmeth->derive) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   if (ctx->operation != EVP_PKEY_OP_DERIVE) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive, EVP_R_OPERATON_NOT_INITIALIZED);
-    return -1;
+    return 0;
   }
   return ctx->pmeth->derive(ctx, key, out_key_len);
 }
 
 int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx) {
-  int ret;
   if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen_init,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   ctx->operation = EVP_PKEY_OP_KEYGEN;
   if (!ctx->pmeth->keygen_init) {
     return 1;
   }
-  ret = ctx->pmeth->keygen_init(ctx);
-  if (ret <= 0) {
+  if (!ctx->pmeth->keygen_init(ctx)) {
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
+    return 0;
   }
-  return ret;
+  return 1;
 }
 
 int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey) {
-  int ret;
-
   if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen,
                       EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-    return -2;
+    return 0;
   }
   if (ctx->operation != EVP_PKEY_OP_KEYGEN) {
     OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen, EVP_R_OPERATON_NOT_INITIALIZED);
-    return -1;
+    return 0;
   }
 
   if (!ppkey) {
-    return -1;
+    return 0;
   }
 
   if (!*ppkey) {
     *ppkey = EVP_PKEY_new();
   }
 
-  ret = ctx->pmeth->keygen(ctx, *ppkey);
-  if (ret <= 0) {
+  if (!ctx->pmeth->keygen(ctx, *ppkey)) {
     EVP_PKEY_free(*ppkey);
     *ppkey = NULL;
+    return 0;
   }
-  return ret;
+  return 1;
 }
diff --git a/crypto/evp/p_ec.c b/crypto/evp/p_ec.c
index 2305e6e8..e429f77a 100644
--- a/crypto/evp/p_ec.c
+++ b/crypto/evp/p_ec.c
@@ -164,7 +164,7 @@ static void pkey_ec_cleanup(EVP_PKEY_CTX *ctx) {
 
 static int pkey_ec_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
                         const uint8_t *tbs, size_t tbslen) {
-  int ret, type;
+  int type;
   unsigned int sltmp;
   EC_PKEY_CTX *dctx = ctx->data;
   EC_KEY *ec = ctx->pkey->pkey.ec;
@@ -182,10 +182,8 @@ static int pkey_ec_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
     type = EVP_MD_type(dctx->md);
   }
 
-  ret = ECDSA_sign(type, tbs, tbslen, sig, &sltmp, ec);
-
-  if (ret <= 0) {
-    return ret;
+  if (!ECDSA_sign(type, tbs, tbslen, sig, &sltmp, ec)) {
+    return 0;
   }
   *siglen = (size_t)sltmp;
   return 1;
@@ -235,7 +233,7 @@ static int pkey_ec_derive(EVP_PKEY_CTX *ctx, uint8_t *key,
 
   ret = ECDH_compute_key(key, outlen, pubkey, eckey, 0);
   if (ret < 0) {
-    return ret;
+    return 0;
   }
   *keylen = ret;
   return 1;
diff --git a/crypto/evp/p_rsa.c b/crypto/evp/p_rsa.c
index 458b606e..bcaca4b5 100644
--- a/crypto/evp/p_rsa.c
+++ b/crypto/evp/p_rsa.c
@@ -168,58 +168,57 @@ static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk) {
 
 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
                          const uint8_t *tbs, size_t tbslen) {
-  int ret;
   RSA_PKEY_CTX *rctx = ctx->data;
   RSA *rsa = ctx->pkey->pkey.rsa;
+  const size_t key_len = EVP_PKEY_size(ctx->pkey);
 
   if (!sig) {
-    *siglen = RSA_size(rsa);
+    *siglen = key_len;
     return 1;
-  } else if (*siglen < (size_t)RSA_size(rsa)) {
+  }
+
+  if (*siglen < key_len) {
     OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_BUFFER_TOO_SMALL);
     return 0;
   }
 
   if (rctx->md) {
+    unsigned int out_len;
+
     if (tbslen != EVP_MD_size(rctx->md)) {
       OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_INVALID_DIGEST_LENGTH);
-      return -1;
+      return 0;
     }
 
     if (EVP_MD_type(rctx->md) == NID_mdc2) {
       OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_NO_MDC2_SUPPORT);
-      ret = -1;
-    } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
-      unsigned int sltmp;
-      ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &sltmp, rsa);
-      if (ret <= 0) {
-        return ret;
-      }
-      ret = sltmp;
-    } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
-      if (!setup_tbuf(rctx, ctx) ||
-          !RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md,
-                                          rctx->mgf1md, rctx->saltlen)) {
-        return -1;
-      }
-      /* TODO(fork): don't use private_encrypt. */
-      ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf, sig, rsa,
-                                RSA_NO_PADDING);
-    } else {
-      return -1;
+      return 0;
     }
-  } else {
-    /* TODO(fork): don't use private_encrypt. */
-    ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
-                              rctx->pad_mode);
-  }
 
-  if (ret < 0) {
-    return ret;
+    switch (rctx->pad_mode) {
+      case RSA_PKCS1_PADDING:
+        if (!RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &out_len, rsa)) {
+          return 0;
+        }
+        *siglen = out_len;
+        return 1;
+
+      case RSA_PKCS1_PSS_PADDING:
+        if (!setup_tbuf(rctx, ctx) ||
+            !RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md,
+                                            rctx->mgf1md, rctx->saltlen) ||
+            !RSA_encrypt(rsa, siglen, sig, *siglen, rctx->tbuf, key_len,
+                         RSA_NO_PADDING)) {
+          return 0;
+        }
+        return 1;
+
+      default:
+        return 0;
+    }
   }
-  *siglen = ret;
 
-  return 1;
+  return RSA_sign_raw(rsa, siglen, sig, *siglen, tbs, tbslen, rctx->pad_mode);
 }
 
 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig,
@@ -228,42 +227,33 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig,
   RSA_PKEY_CTX *rctx = ctx->data;
   RSA *rsa = ctx->pkey->pkey.rsa;
   size_t rslen;
+  const size_t key_len = EVP_PKEY_size(ctx->pkey);
 
   if (rctx->md) {
-    if (rctx->pad_mode == RSA_PKCS1_PADDING) {
-      return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, sig, siglen, rsa);
-    }
+    switch (rctx->pad_mode) {
+      case RSA_PKCS1_PADDING:
+        return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, sig, siglen, rsa);
+
+      case RSA_PKCS1_PSS_PADDING:
+        if (!setup_tbuf(rctx, ctx) ||
+            !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
+                            RSA_NO_PADDING) ||
+            !RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md, rctx->mgf1md,
+                                       rctx->tbuf, rctx->saltlen)) {
+          return 0;
+        }
+        return 1;
 
-    if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
-      int ret;
-      if (!setup_tbuf(rctx, ctx)) {
-        return -1;
-      }
-      /* TODO(fork): don't use public_decrypt. */
-      ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa, RSA_NO_PADDING);
-      if (ret <= 0) {
-        return 0;
-      }
-      ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md, rctx->mgf1md,
-                                      rctx->tbuf, rctx->saltlen);
-      if (ret <= 0) {
+      default:
         return 0;
-      }
-      return 1;
-    } else {
-      return -1;
-    }
-  } else {
-    if (!setup_tbuf(rctx, ctx)) {
-      return -1;
-    }
-    rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa, rctx->pad_mode);
-    if (rslen == 0) {
-      return 0;
     }
   }
 
-  if (rslen != tbslen || CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
+  if (!setup_tbuf(rctx, ctx) ||
+      !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
+                      rctx->pad_mode) ||
+      rslen != tbslen ||
+      CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
     return 0;
   }
 
@@ -272,77 +262,73 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig,
 
 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
                             const uint8_t *in, size_t inlen) {
-  int ret;
   RSA_PKEY_CTX *rctx = ctx->data;
   RSA *rsa = ctx->pkey->pkey.rsa;
+  const size_t key_len = EVP_PKEY_size(ctx->pkey);
 
   if (!out) {
-    *outlen = RSA_size(rsa);
+    *outlen = key_len;
     return 1;
-  } else if (*outlen < (size_t)RSA_size(rsa)) {
+  }
+
+  if (*outlen < key_len) {
     OPENSSL_PUT_ERROR(EVP, pkey_rsa_encrypt, EVP_R_BUFFER_TOO_SMALL);
     return 0;
   }
 
   if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
-    int klen = RSA_size(rsa);
-    if (!setup_tbuf(rctx, ctx)) {
-      return -1;
-    }
-    if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen, in, inlen,
+    if (!setup_tbuf(rctx, ctx) ||
+        !RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, key_len, in, inlen,
                                          rctx->oaep_label, rctx->oaep_labellen,
-                                         rctx->md, rctx->mgf1md)) {
-      return -1;
+                                         rctx->md, rctx->mgf1md) ||
+        !RSA_encrypt(rsa, outlen, out, *outlen, rctx->tbuf, key_len,
+                     RSA_NO_PADDING)) {
+      return 0;
     }
-    ret = RSA_public_encrypt(klen, rctx->tbuf, out, rsa, RSA_NO_PADDING);
-  } else {
-    ret = RSA_public_encrypt(inlen, in, out, rsa, rctx->pad_mode);
-  }
-
-  if (ret < 0) {
-    return ret;
+    return 1;
   }
-  *outlen = ret;
 
-  return 1;
+  return RSA_encrypt(rsa, outlen, out, *outlen, in, inlen, rctx->pad_mode);
 }
 
 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out,
                             size_t *outlen, const uint8_t *in,
                             size_t inlen) {
-  int ret;
   RSA_PKEY_CTX *rctx = ctx->data;
   RSA *rsa = ctx->pkey->pkey.rsa;
+  const size_t key_len = EVP_PKEY_size(ctx->pkey);
 
   if (!out) {
-    *outlen = RSA_size(rsa);
+    *outlen = key_len;
     return 1;
-  } else if (*outlen < (size_t)RSA_size(rsa)) {
+  }
+
+  if (*outlen < key_len) {
     OPENSSL_PUT_ERROR(EVP, pkey_rsa_decrypt, EVP_R_BUFFER_TOO_SMALL);
     return 0;
   }
 
   if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
-    if (!setup_tbuf(rctx, ctx)) {
-      return -1;
-    }
-    ret = RSA_private_decrypt(inlen, in, rctx->tbuf, rsa, RSA_NO_PADDING);
-    if (ret <= 0) {
-      return ret;
+    size_t plaintext_len;
+    int message_len;
+
+    if (!setup_tbuf(rctx, ctx) ||
+        !RSA_decrypt(rsa, &plaintext_len, rctx->tbuf, key_len, in, inlen,
+                     RSA_NO_PADDING)) {
+      return 0;
     }
-    ret = RSA_padding_check_PKCS1_OAEP_mgf1(
-        out, ret, rctx->tbuf, ret, rctx->oaep_label,
-        rctx->oaep_labellen, rctx->md, rctx->mgf1md);
-  } else {
-    ret = RSA_private_decrypt(inlen, in, out, rsa, rctx->pad_mode);
-  }
 
-  if (ret < 0) {
-    return ret;
+    message_len = RSA_padding_check_PKCS1_OAEP_mgf1(
+        out, key_len, rctx->tbuf, plaintext_len, rctx->oaep_label,
+        rctx->oaep_labellen, rctx->md, rctx->mgf1md);
+    if (message_len < 0) {
+      return 0;
+    }
+    *outlen = message_len;
+    return 1;
   }
-  *outlen = ret;
 
-  return 1;
+  return RSA_decrypt(rsa, outlen, out, key_len, in, inlen, rctx->pad_mode);
 }
 
 static int check_padding_md(const EVP_MD *md, int padding) {
@@ -507,7 +493,6 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) {
   RSA *rsa = NULL;
   RSA_PKEY_CTX *rctx = ctx->data;
 
-  int ret;
   if (!rctx->pub_exp) {
     rctx->pub_exp = BN_new();
     if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
@@ -518,13 +503,13 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) {
     return 0;
   }
 
-  ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, NULL);
-  if (ret > 0) {
-    EVP_PKEY_assign_RSA(pkey, rsa);
-  } else {
+  if (!RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, NULL)) {
     RSA_free(rsa);
+    return 0;
   }
-  return ret;
+
+  EVP_PKEY_assign_RSA(pkey, rsa);
+  return 1;
 }
 
 const EVP_PKEY_METHOD rsa_pkey_meth = {