[System]: Fix `MonoBtlsProvider.CheckValidationResult ()` logic. (#10396)

See my comments in https://github.com/mono/mono/pull/10134 for details.

The problem was that
1. It is wrong to assume that `chain != null`.  In fact, it likely won't be if `wantsChain == false`.
2. Similarly, the `storeCtx` should only be used if `storeCtx != NULL && wantsChain == true`.
3. It was missing `MonoBtlsX509Error.HOSTNAME_MISMATCH`.
4. I reformatted and re-indented the method to match Mono's style.

1 files changed, 88 insertions, 81 deletions

diff --git a/mcs/class/System/Mono.Btls/MonoBtlsProvider.cs b/mcs/class/System/Mono.Btls/MonoBtlsProvider.cs
index e862476bf8b..d2323b0479b 100644
--- a/mcs/class/System/Mono.Btls/MonoBtlsProvider.cs
+++ b/mcs/class/System/Mono.Btls/MonoBtlsProvider.cs
@@ -223,93 +223,100 @@ namespace Mono.Btls
 			bool success, ref MonoSslPolicyErrors errors, ref int status11)
 		{
 			status11 = unchecked((int)0);
-			if (!success) {
-				errors = MonoSslPolicyErrors.RemoteCertificateChainErrors;
-				var error = storeCtx.GetError();
-				if (error != Mono.Btls.MonoBtlsX509Error.OK & 
-				    error != Mono.Btls.MonoBtlsX509Error.CRL_NOT_YET_VALID) {
-					chain.Impl.AddStatus(MapVerifyErrorToChainStatus(error));
-					status11 = unchecked((int)0x800B010B);
-				}
+			if (success)
+				return;
+			errors = MonoSslPolicyErrors.RemoteCertificateChainErrors;
+			if (!wantsChain || storeCtx == null || chain == null) {
+				status11 = unchecked((int)0x800B010B);
+				return;
+			}
+			var error = storeCtx.GetError();
+			if (error != Mono.Btls.MonoBtlsX509Error.OK &
+			    error != Mono.Btls.MonoBtlsX509Error.CRL_NOT_YET_VALID) {
+				chain.Impl.AddStatus(MapVerifyErrorToChainStatus(error));
+				status11 = unchecked((int)0x800B010B);
 			}
 		}
 
-		internal static X509ChainStatusFlags MapVerifyErrorToChainStatus(MonoBtlsX509Error code)
+		internal static X509ChainStatusFlags MapVerifyErrorToChainStatus (MonoBtlsX509Error code)
 		{
-		    switch (code)
-		    {
-			case Mono.Btls.MonoBtlsX509Error.OK :
-			    return X509ChainStatusFlags.NoError;
-
-			case Mono.Btls.MonoBtlsX509Error.CERT_NOT_YET_VALID :
-			case Mono.Btls.MonoBtlsX509Error.CERT_HAS_EXPIRED:
-			case Mono.Btls.MonoBtlsX509Error.ERROR_IN_CERT_NOT_BEFORE_FIELD:
-			case Mono.Btls.MonoBtlsX509Error.ERROR_IN_CERT_NOT_AFTER_FIELD:
-			    return X509ChainStatusFlags.NotTimeValid;
-
-			case Mono.Btls.MonoBtlsX509Error.CERT_REVOKED:
-			    return X509ChainStatusFlags.Revoked;
-
-			case Mono.Btls.MonoBtlsX509Error.UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY:
-			case Mono.Btls.MonoBtlsX509Error.CERT_SIGNATURE_FAILURE:
-			    return X509ChainStatusFlags.NotSignatureValid;
-
-			case Mono.Btls.MonoBtlsX509Error.CERT_UNTRUSTED:
-			case Mono.Btls.MonoBtlsX509Error.DEPTH_ZERO_SELF_SIGNED_CERT:
-			case Mono.Btls.MonoBtlsX509Error.SELF_SIGNED_CERT_IN_CHAIN:
-			    return X509ChainStatusFlags.UntrustedRoot;
-
-			case Mono.Btls.MonoBtlsX509Error.CRL_HAS_EXPIRED:
-			    return X509ChainStatusFlags.OfflineRevocation;
-
-			case Mono.Btls.MonoBtlsX509Error.CRL_NOT_YET_VALID:
-			case Mono.Btls.MonoBtlsX509Error.CRL_SIGNATURE_FAILURE:
-			case Mono.Btls.MonoBtlsX509Error.ERROR_IN_CRL_LAST_UPDATE_FIELD:
-			case Mono.Btls.MonoBtlsX509Error.ERROR_IN_CRL_NEXT_UPDATE_FIELD:
-			case Mono.Btls.MonoBtlsX509Error.KEYUSAGE_NO_CRL_SIGN:
-			case Mono.Btls.MonoBtlsX509Error.UNABLE_TO_DECRYPT_CRL_SIGNATURE:
-			case Mono.Btls.MonoBtlsX509Error.UNABLE_TO_GET_CRL:
-			case Mono.Btls.MonoBtlsX509Error.UNABLE_TO_GET_CRL_ISSUER:
-			case Mono.Btls.MonoBtlsX509Error.UNHANDLED_CRITICAL_CRL_EXTENSION:
-			    return X509ChainStatusFlags.RevocationStatusUnknown;
-
-			case Mono.Btls.MonoBtlsX509Error.INVALID_EXTENSION:
-			    return X509ChainStatusFlags.InvalidExtension;
-
-			case Mono.Btls.MonoBtlsX509Error.UNABLE_TO_GET_ISSUER_CERT:
-			case Mono.Btls.MonoBtlsX509Error.UNABLE_TO_GET_ISSUER_CERT_LOCALLY:
-			case Mono.Btls.MonoBtlsX509Error.UNABLE_TO_VERIFY_LEAF_SIGNATURE:
-			    return X509ChainStatusFlags.PartialChain;
-
-			case Mono.Btls.MonoBtlsX509Error.INVALID_PURPOSE:
-			    return X509ChainStatusFlags.NotValidForUsage;
-
-			case Mono.Btls.MonoBtlsX509Error.INVALID_CA:
-			case Mono.Btls.MonoBtlsX509Error.INVALID_NON_CA:
-			case Mono.Btls.MonoBtlsX509Error.PATH_LENGTH_EXCEEDED:
-			case Mono.Btls.MonoBtlsX509Error.KEYUSAGE_NO_CERTSIGN:
-			case Mono.Btls.MonoBtlsX509Error.KEYUSAGE_NO_DIGITAL_SIGNATURE:
-			    return X509ChainStatusFlags.InvalidBasicConstraints;
-
-			case Mono.Btls.MonoBtlsX509Error.INVALID_POLICY_EXTENSION:
-			case Mono.Btls.MonoBtlsX509Error.NO_EXPLICIT_POLICY:
-			    return X509ChainStatusFlags.InvalidPolicyConstraints;
-
-			case Mono.Btls.MonoBtlsX509Error.CERT_REJECTED:
-			    return X509ChainStatusFlags.ExplicitDistrust;
-
-			case Mono.Btls.MonoBtlsX509Error.UNHANDLED_CRITICAL_EXTENSION:
-			    return X509ChainStatusFlags.HasNotSupportedCriticalExtension;
-
-			case Mono.Btls.MonoBtlsX509Error.CERT_CHAIN_TOO_LONG:
-			    throw new CryptographicException();
-
-			case Mono.Btls.MonoBtlsX509Error.OUT_OF_MEM:
-			    throw new OutOfMemoryException();
+			switch (code) {
+			case MonoBtlsX509Error.OK :
+				return X509ChainStatusFlags.NoError;
+
+			case MonoBtlsX509Error.CERT_NOT_YET_VALID :
+			case MonoBtlsX509Error.CERT_HAS_EXPIRED:
+			case MonoBtlsX509Error.ERROR_IN_CERT_NOT_BEFORE_FIELD:
+			case MonoBtlsX509Error.ERROR_IN_CERT_NOT_AFTER_FIELD:
+				return X509ChainStatusFlags.NotTimeValid;
+
+			case MonoBtlsX509Error.CERT_REVOKED:
+				return X509ChainStatusFlags.Revoked;
+
+			case MonoBtlsX509Error.UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY:
+			case MonoBtlsX509Error.CERT_SIGNATURE_FAILURE:
+				return X509ChainStatusFlags.NotSignatureValid;
+
+			case MonoBtlsX509Error.CERT_UNTRUSTED:
+			case MonoBtlsX509Error.DEPTH_ZERO_SELF_SIGNED_CERT:
+			case MonoBtlsX509Error.SELF_SIGNED_CERT_IN_CHAIN:
+				return X509ChainStatusFlags.UntrustedRoot;
+
+			case MonoBtlsX509Error.CRL_HAS_EXPIRED:
+				return X509ChainStatusFlags.OfflineRevocation;
+
+			case MonoBtlsX509Error.CRL_NOT_YET_VALID:
+			case MonoBtlsX509Error.CRL_SIGNATURE_FAILURE:
+			case MonoBtlsX509Error.ERROR_IN_CRL_LAST_UPDATE_FIELD:
+			case MonoBtlsX509Error.ERROR_IN_CRL_NEXT_UPDATE_FIELD:
+			case MonoBtlsX509Error.KEYUSAGE_NO_CRL_SIGN:
+			case MonoBtlsX509Error.UNABLE_TO_DECRYPT_CRL_SIGNATURE:
+			case MonoBtlsX509Error.UNABLE_TO_GET_CRL:
+			case MonoBtlsX509Error.UNABLE_TO_GET_CRL_ISSUER:
+			case MonoBtlsX509Error.UNHANDLED_CRITICAL_CRL_EXTENSION:
+				return X509ChainStatusFlags.RevocationStatusUnknown;
+
+			case MonoBtlsX509Error.INVALID_EXTENSION:
+				return X509ChainStatusFlags.InvalidExtension;
+
+			case MonoBtlsX509Error.UNABLE_TO_GET_ISSUER_CERT:
+			case MonoBtlsX509Error.UNABLE_TO_GET_ISSUER_CERT_LOCALLY:
+			case MonoBtlsX509Error.UNABLE_TO_VERIFY_LEAF_SIGNATURE:
+				return X509ChainStatusFlags.PartialChain;
+
+			case MonoBtlsX509Error.INVALID_PURPOSE:
+				return X509ChainStatusFlags.NotValidForUsage;
+
+			case MonoBtlsX509Error.INVALID_CA:
+			case MonoBtlsX509Error.INVALID_NON_CA:
+			case MonoBtlsX509Error.PATH_LENGTH_EXCEEDED:
+			case MonoBtlsX509Error.KEYUSAGE_NO_CERTSIGN:
+			case MonoBtlsX509Error.KEYUSAGE_NO_DIGITAL_SIGNATURE:
+				return X509ChainStatusFlags.InvalidBasicConstraints;
+
+			case MonoBtlsX509Error.INVALID_POLICY_EXTENSION:
+			case MonoBtlsX509Error.NO_EXPLICIT_POLICY:
+				return X509ChainStatusFlags.InvalidPolicyConstraints;
+
+			case MonoBtlsX509Error.CERT_REJECTED:
+				return X509ChainStatusFlags.ExplicitDistrust;
+
+			case MonoBtlsX509Error.UNHANDLED_CRITICAL_EXTENSION:
+				return X509ChainStatusFlags.HasNotSupportedCriticalExtension;
+
+			case MonoBtlsX509Error.HOSTNAME_MISMATCH:
+				// FIXME: we should have a better error flag for this.
+				return X509ChainStatusFlags.UntrustedRoot;
+
+			case MonoBtlsX509Error.CERT_CHAIN_TOO_LONG:
+				throw new CryptographicException ();
+
+			case MonoBtlsX509Error.OUT_OF_MEM:
+				throw new OutOfMemoryException ();
 
 			default:
-			    throw new CryptographicException("Unrecognized X509VerifyStatusCode:" + code);
-		    }
+				throw new CryptographicException ("Unrecognized X509VerifyStatusCode:" + code);
+			}
 		}
 
 		internal static void SetupCertificateStore (MonoBtlsX509Store store, MonoTlsSettings settings, bool server)