Stop using EVP_MD_CTX_FLAG_NON_FIPS_ALLOW

This is a flag that has lost its relevance.  The new mechanism to do
the same thing is to fetch the needed digest explicitly with "-fips"
as property query, i.e. we remove any requirement for that property to
be set when fetching, even if the default property query string
requires its presence.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/10138)

1 files changed, 8 insertions, 4 deletions

diff --git a/crypto/x509/x509_cmp.c b/crypto/x509/x509_cmp.c
index f208cd6a04..b8a61ffe2a 100644
--- a/crypto/x509/x509_cmp.c
+++ b/crypto/x509/x509_cmp.c
@@ -13,6 +13,7 @@
 #include <openssl/objects.h>
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
+#include <openssl/core_names.h>
 #include "crypto/x509.h"
 
 int X509_issuer_and_serial_cmp(const X509 *a, const X509 *b)
@@ -205,23 +206,26 @@ unsigned long X509_NAME_hash(X509_NAME *x)
 
 unsigned long X509_NAME_hash_old(X509_NAME *x)
 {
+    EVP_MD *md5 = EVP_MD_fetch(NULL, OSSL_DIGEST_NAME_MD5, "-fips");
     EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
     unsigned long ret = 0;
     unsigned char md[16];
 
-    if (md_ctx == NULL)
-        return ret;
+    if (md5 == NULL || md_ctx == NULL)
+        goto end;
 
     /* Make sure X509_NAME structure contains valid cached encoding */
     i2d_X509_NAME(x, NULL);
-    EVP_MD_CTX_set_flags(md_ctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
-    if (EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL)
+    if (EVP_DigestInit_ex(md_ctx, md5, NULL)
         && EVP_DigestUpdate(md_ctx, x->bytes->data, x->bytes->length)
         && EVP_DigestFinal_ex(md_ctx, md, NULL))
         ret = (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) |
                ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L)
             ) & 0xffffffffL;
+
+ end:
     EVP_MD_CTX_free(md_ctx);
+    EVP_MD_free(md5);
 
     return ret;
 }