Normalize all EVP_PKEY_CTX_ctrl return values.

No code within BoringSSL or Google (grep for EVP_PKEY_CTX_(ctrl|get|set)) is
sensitive to the various failure cases. Normalize it all to 0/1 for simplicity.

This does carry a slight risk: any new ctrl hooks we import from upstream that,
like EVP_PKEY_CTX_get_rsa_oaep_md, return something other than success/failure
cannot be called directly via EVP_PKEY_CTX_ctrl. They instead need to
internally be routed through a struct like CBS and only called through the
wrappers. To that end, unexport EVP_PKEY_CTX_ctrl and require that callers use
the wrappers. No code in Google uses it directly and, if need be, switching to
the wrapper would be an incredibly upstreamable patch.

Change-Id: I3fd4e5a1a0f3d4d1c4122c52d4c74a5105b99cd5
Reviewed-on: https://boringssl-review.googlesource.com/3874
Reviewed-by: Adam Langley <agl@google.com>

1 files changed, 20 insertions, 83 deletions

diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 1e17a465..b1497f13 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -444,18 +444,6 @@ OPENSSL_EXPORT void EVP_PKEY_CTX_set_app_data(EVP_PKEY_CTX *ctx, void *data);
  * set. */
 OPENSSL_EXPORT void *EVP_PKEY_CTX_get_app_data(EVP_PKEY_CTX *ctx);
 
-/* EVP_PKEY_CTX_ctrl performs |cmd| on |ctx|. The |keytype| and |optype|
- * arguments can be -1 to specify that any type and operation are acceptable,
- * otherwise |keytype| must match the type of |ctx| and the bits of |optype|
- * must intersect the operation flags set on |ctx|.
- *
- * The |p1| and |p2| arguments depend on the value of |cmd|.
- *
- * It returns -2 if |cmd| is not recognised, -1 on error or a |cmd| specific
- * value otherwise. */
-OPENSSL_EXPORT int EVP_PKEY_CTX_ctrl(EVP_PKEY_CTX *ctx, int keytype, int optype,
-                                     int cmd, int p1, void *p2);
-
 /* EVP_PKEY_sign_init initialises an |EVP_PKEY_CTX| for a signing operation. It
  * should be called before |EVP_PKEY_sign|.
  *
@@ -569,67 +557,28 @@ OPENSSL_EXPORT int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx);
 OPENSSL_EXPORT int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
 
 
-/* EVP_PKEY_CTX_ctrl operations.
- *
- * These values are passed as the |cmd| argument to
- * EVP_PKEY_CTX_ctrl */
-
-/* Generic. */
+/* Generic control functions. */
 
 /* EVP_PKEY_CTX_set_signature_md sets |md| as the digest to be used in a
- * signature operation. It returns one on success or otherwise on error. See
- * the return values of |EVP_PKEY_CTX_ctrl| for details. */
+ * signature operation. It returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_signature_md(EVP_PKEY_CTX *ctx,
                                                  const EVP_MD *md);
 
 /* EVP_PKEY_CTX_get_signature_md sets |*out_md| to the digest to be used in a
- * signature operation. It returns one on success or otherwise on error. See
- * the return values of |EVP_PKEY_CTX_ctrl| for details. */
+ * signature operation. It returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_get_signature_md(EVP_PKEY_CTX *ctx,
                                                  const EVP_MD **out_md);
 
-/* EVP_PKEY_CTRL_DIGESTINIT is an internal value. It's called by
- * EVP_DigestInit_ex to signal the |EVP_PKEY| that a digest operation is
- * starting.
- *
- * TODO(davidben): This is only needed to support the deprecated HMAC |EVP_PKEY|
- * types. */
-#define EVP_PKEY_CTRL_DIGESTINIT 3
-
-/* EVP_PKEY_CTRL_PEER_KEY is called with different values of |p1|:
- *   0: Is called from |EVP_PKEY_derive_set_peer| and |p2| contains a peer key.
- *      If the return value is <= 0, the key is rejected.
- *   1: Is called at the end of |EVP_PKEY_derive_set_peer| and |p2| contains a
- *      peer key. If the return value is <= 0, the key is rejected.
- *   2: Is called with |p2| == NULL to test whether the peer's key was used.
- *      (EC)DH always return one in this case.
- *   3: Is called with |p2| == NULL to set whether the peer's key was used.
- *      (EC)DH always return one in this case. This was only used for GOST. */
-#define EVP_PKEY_CTRL_PEER_KEY 4
-
-/* EVP_PKEY_CTRL_SET_MAC_KEY sets a MAC key. For example, this can be done an
- * |EVP_PKEY_CTX| prior to calling |EVP_PKEY_keygen| in order to generate an
- * HMAC |EVP_PKEY| with the given key. It returns one on success and zero on
- * error. */
-#define EVP_PKEY_CTRL_SET_MAC_KEY 5
-
-/* EVP_PKEY_ALG_CTRL is the base value from which key-type specific ctrl
- * commands are numbered. */
-#define EVP_PKEY_ALG_CTRL 0x1000
-
 
 /* RSA specific control functions. */
 
 /* EVP_PKEY_CTX_set_rsa_padding sets the padding type to use. It should be one
- * of the |RSA_*_PADDING| values. Returns one on success or another value on
- * error. See |EVP_PKEY_CTX_ctrl| for the other return values, which are
- * non-standard. */
+ * of the |RSA_*_PADDING| values. Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_padding(EVP_PKEY_CTX *ctx, int padding);
 
 /* EVP_PKEY_CTX_get_rsa_padding sets |*out_padding| to the current padding
  * value, which is one of the |RSA_*_PADDING| values. Returns one on success or
- * another value on error. See |EVP_PKEY_CTX_ctrl| for the other return values,
- * which are non-standard. */
+ * zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_get_rsa_padding(EVP_PKEY_CTX *ctx,
                                                 int *out_padding);
 
@@ -638,8 +587,7 @@ OPENSSL_EXPORT int EVP_PKEY_CTX_get_rsa_padding(EVP_PKEY_CTX *ctx,
  * in the signature. A value of -2 causes the salt to be the maximum length
  * that will fit. Otherwise the value gives the size of the salt in bytes.
  *
- * Returns one on success or another value on error. See |EVP_PKEY_CTX_ctrl|
- * for the other return values, which are non-standard. */
+ * Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx,
                                                     int salt_len);
 
@@ -648,70 +596,59 @@ OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx,
  * |EVP_PKEY_CTX_set_rsa_pss_saltlen| for details of the special values that it
  * can take.
  *
- * Returns one on success or another value on error. See |EVP_PKEY_CTX_ctrl|
- * for the other return values, which are non-standard. */
+ * Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_get_rsa_pss_saltlen(EVP_PKEY_CTX *ctx,
                                                     int *out_salt_len);
 
 /* EVP_PKEY_CTX_set_rsa_keygen_bits sets the size of the desired RSA modulus,
- * in bits, for key generation. Returns one on success or another value on
- * error. See |EVP_PKEY_CTX_ctrl| for the other return values, which are
- * non-standard. */
+ * in bits, for key generation. Returns one on success or zero on
+ * error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_keygen_bits(EVP_PKEY_CTX *ctx,
                                                     int bits);
 
 /* EVP_PKEY_CTX_set_rsa_keygen_pubexp sets |e| as the public exponent for key
- * generation. Returns one on success or another value on error. See
- * |EVP_PKEY_CTX_ctrl| for the other return values, which are non-standard. */
+ * generation. Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_keygen_pubexp(EVP_PKEY_CTX *ctx,
                                                       BIGNUM *e);
 
 /* EVP_PKEY_CTX_set_rsa_oaep_md sets |md| as the digest used in OAEP padding.
- * Returns one on success or another value on error. See |EVP_PKEY_CTX_ctrl|
- * for the other return values, which are non-standard. */
+ * Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX *ctx,
                                                 const EVP_MD *md);
 
 /* EVP_PKEY_CTX_get_rsa_oaep_md sets |*out_md| to the digest function used in
- * OAEP padding. Returns one on success or another value on error. See
- * |EVP_PKEY_CTX_ctrl| for the other return values, which are non-standard. */
+ * OAEP padding. Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_get_rsa_oaep_md(EVP_PKEY_CTX *ctx,
                                                 const EVP_MD **out_md);
 
 /* EVP_PKEY_CTX_set_rsa_mgf1_md sets |md| as the digest used in MGF1. Returns
- * one on success or another value on error. See |EVP_PKEY_CTX_ctrl| for the
- * other return values, which are non-standard. */
+ * one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_mgf1_md(EVP_PKEY_CTX *ctx,
                                                 const EVP_MD *md);
 
 /* EVP_PKEY_CTX_get_rsa_mgf1_md sets |*out_md| to the digest function used in
- * MGF1. Returns one on success or another value on error. See
- * |EVP_PKEY_CTX_ctrl| for the other return values, which are non-standard. */
+ * MGF1. Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_get_rsa_mgf1_md(EVP_PKEY_CTX *ctx,
                                                 const EVP_MD **out_md);
 
 /* EVP_PKEY_CTX_set0_rsa_oaep_label sets |label_len| bytes from |label| as the
- * label used in OAEP. DANGER: this call takes ownership of |label| and will
- * call |free| on it when |ctx| is destroyed.
+ * label used in OAEP. DANGER: On success, this call takes ownership of |label|
+ * and will call |OPENSSL_free| on it when |ctx| is destroyed.
  *
- * Returns one on success or another value on error. See |EVP_PKEY_CTX_ctrl|
- * for the other return values, which are non-standard. */
+ * Returns one on success or zero on error. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_set0_rsa_oaep_label(EVP_PKEY_CTX *ctx,
                                                     const uint8_t *label,
                                                     size_t label_len);
 
 /* EVP_PKEY_CTX_get0_rsa_oaep_label sets |*out_label| to point to the internal
  * buffer containing the OAEP label (which may be NULL) and returns the length
- * of the label or a negative value on error. */
+ * of the label or a negative value on error.
+ *
+ * WARNING: the return value differs from the usual return value convention. */
 OPENSSL_EXPORT int EVP_PKEY_CTX_get0_rsa_oaep_label(EVP_PKEY_CTX *ctx,
                                                     const uint8_t **out_label);
 
 
-/* EC specific */
-
-#define EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID		(EVP_PKEY_ALG_CTRL + 1)
-
-
 /* Private functions */
 
 /* OpenSSL_add_all_algorithms does nothing. */