Add malloc failure tests.

This commit fixes a number of crashes caused by malloc failures. They
were found using the -malloc-test=0 option to runner.go which runs tests
many times, causing a different allocation call to fail in each case.

(This test only works on Linux and only looks for crashes caused by
allocation failures, not memory leaks or other errors.)

This is not the complete set of crashes! More can be found by collecting
core dumps from running with -malloc-test=0.

Change-Id: Ia61d19f51e373bccb7bc604642c51e043a74bd83
Reviewed-on: https://boringssl-review.googlesource.com/2320
Reviewed-by: Adam Langley <agl@google.com>

3 files changed, 21 insertions, 1 deletions

diff --git a/crypto/evp/evp_ctx.c b/crypto/evp/evp_ctx.c
index d1ed67d6..66160382 100644
--- a/crypto/evp/evp_ctx.c
+++ b/crypto/evp/evp_ctx.c
@@ -124,7 +124,10 @@ static EVP_PKEY_CTX *evp_pkey_ctx_new(EVP_PKEY *pkey, ENGINE *e, int id) {
 
   if (pmeth->init) {
     if (pmeth->init(ret) <= 0) {
-      EVP_PKEY_CTX_free(ret);
+      if (pkey) {
+        EVP_PKEY_free(ret->pkey);
+      }
+      OPENSSL_free(ret);
       return NULL;
     }
   }
@@ -176,17 +179,25 @@ EVP_PKEY_CTX *EVP_PKEY_CTX_dup(EVP_PKEY_CTX *pctx) {
 
   if (pctx->pkey) {
     rctx->pkey = EVP_PKEY_dup(pctx->pkey);
+    if (rctx->pkey == NULL) {
+      goto err;
+    }
   }
 
   if (pctx->peerkey) {
     rctx->peerkey = EVP_PKEY_dup(pctx->peerkey);
+    if (rctx->peerkey == NULL) {
+      goto err;
+    }
   }
 
   if (pctx->pmeth->copy(rctx, pctx) > 0) {
     return rctx;
   }
 
+err:
   EVP_PKEY_CTX_free(rctx);
+  OPENSSL_PUT_ERROR(EVP, EVP_PKEY_CTX_dup, ERR_LIB_EVP);
   return NULL;
 }
 
@@ -485,6 +496,10 @@ int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey) {
 
   if (!*ppkey) {
     *ppkey = EVP_PKEY_new();
+    if (!*ppkey) {
+      OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen, ERR_LIB_EVP);
+      return 0;
+    }
   }
 
   if (!ctx->pmeth->keygen(ctx, *ppkey)) {
diff --git a/crypto/evp/evp_error.c b/crypto/evp/evp_error.c
index d2d8abac..b0d311ef 100644
--- a/crypto/evp/evp_error.c
+++ b/crypto/evp/evp_error.c
@@ -20,6 +20,7 @@ const ERR_STRING_DATA EVP_error_string_data[] = {
   {ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_DigestSignAlgorithm, 0), "EVP_DigestSignAlgorithm"},
   {ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_DigestVerifyInitFromAlgorithm, 0), "EVP_DigestVerifyInitFromAlgorithm"},
   {ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_PKEY_CTX_ctrl, 0), "EVP_PKEY_CTX_ctrl"},
+  {ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_PKEY_CTX_dup, 0), "EVP_PKEY_CTX_dup"},
   {ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_PKEY_copy_parameters, 0), "EVP_PKEY_copy_parameters"},
   {ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_PKEY_decrypt, 0), "EVP_PKEY_decrypt"},
   {ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_PKEY_decrypt_init, 0), "EVP_PKEY_decrypt_init"},
diff --git a/crypto/evp/p_hmac.c b/crypto/evp/p_hmac.c
index f068f205..d7819205 100644
--- a/crypto/evp/p_hmac.c
+++ b/crypto/evp/p_hmac.c
@@ -109,6 +109,10 @@ static int pkey_hmac_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src) {
 static void pkey_hmac_cleanup(EVP_PKEY_CTX *ctx) {
   HMAC_PKEY_CTX *hctx = ctx->data;
 
+  if (hctx == NULL) {
+    return;
+  }
+
   HMAC_CTX_cleanup(&hctx->ctx);
   if (hctx->ktmp.data) {
     if (hctx->ktmp.length) {