Use OS-provided RSA OAEP implementation for Android

Instead of using the managed OAEP implementatation, allow the OS to handle OAEP with SHA2 algorithms.

Android also supports algorithm identifiers like "RSA/ECB/OAEPwithSHA-256andMGF1Padding" which would make the code simpler, but it makes the hash algorithm used in MGF1 ambiguous. Since the MGF1 algorithm is always the same as the OAEP algorithm, we explicitly use the `OEAPParameterSpec` so that we can control both the MGF1 digest algorithm and the OAEP digest algorithm.

4 files changed, 144 insertions, 15 deletions

diff --git a/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.c b/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.c
index fe92ce9ebe3..aac61e9bf9d 100644
--- a/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.c
+++ b/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.c
@@ -92,6 +92,21 @@ jmethodID g_sslCtxGetDefaultSslParamsMethod;
 jclass    g_GCMParameterSpecClass;
 jmethodID g_GCMParameterSpecCtor;
 
+// java/security/spec/MGF1ParameterSpec
+jclass    g_MGF1ParameterSpecClass;
+jfieldID  g_MGF1ParameterSpec_SHA1Field;
+jfieldID  g_MGF1ParameterSpec_SHA256Field;
+jfieldID  g_MGF1ParameterSpec_SHA384Field;
+jfieldID  g_MGF1ParameterSpec_SHA512Field;
+
+// javax/crypto/spec/OAEPParameterSpec
+jclass    g_OAEPParameterSpecClass;
+jmethodID g_OAEPParameterSpecCtor;
+
+// javax/crypto/spec/PSource$PSpecified
+jclass   g_PSourcePSpecifiedClass;
+jfieldID g_PSourcePSpecified_DefaultField;
+
 // java/security/interfaces/DSAKey
 jclass    g_DSAKeyClass;
 
@@ -692,6 +707,18 @@ JNI_OnLoad(JavaVM *vm, void *reserved)
     g_GCMParameterSpecClass =   GetClassGRef(env, "javax/crypto/spec/GCMParameterSpec");
     g_GCMParameterSpecCtor =    GetMethod(env, false, g_GCMParameterSpecClass, "<init>", "(I[B)V");
 
+    g_MGF1ParameterSpecClass =        GetClassGRef(env, "java/security/spec/MGF1ParameterSpec");
+    g_MGF1ParameterSpec_SHA1Field   = GetField(env, true, g_MGF1ParameterSpecClass, "SHA1", "Ljava/security/spec/MGF1ParameterSpec;");
+    g_MGF1ParameterSpec_SHA256Field = GetField(env, true, g_MGF1ParameterSpecClass, "SHA256", "Ljava/security/spec/MGF1ParameterSpec;");
+    g_MGF1ParameterSpec_SHA384Field = GetField(env, true, g_MGF1ParameterSpecClass, "SHA384", "Ljava/security/spec/MGF1ParameterSpec;");
+    g_MGF1ParameterSpec_SHA512Field = GetField(env, true, g_MGF1ParameterSpecClass, "SHA512", "Ljava/security/spec/MGF1ParameterSpec;");
+
+    g_OAEPParameterSpecClass =  GetClassGRef(env, "javax/crypto/spec/OAEPParameterSpec");
+    g_OAEPParameterSpecCtor =   GetMethod(env, false, g_OAEPParameterSpecClass, "<init>", "(Ljava/lang/String;Ljava/lang/String;Ljava/security/spec/AlgorithmParameterSpec;Ljavax/crypto/spec/PSource;)V");
+
+    g_PSourcePSpecifiedClass =         GetClassGRef(env, "javax/crypto/spec/PSource$PSpecified");
+    g_PSourcePSpecified_DefaultField = GetField(env, true, g_PSourcePSpecifiedClass, "DEFAULT", "Ljavax/crypto/spec/PSource$PSpecified;");
+
     g_bigNumClass =             GetClassGRef(env, "java/math/BigInteger");
     g_bigNumCtor =              GetMethod(env, false, g_bigNumClass, "<init>", "([B)V");
     g_bigNumCtorWithSign =      GetMethod(env, false, g_bigNumClass, "<init>", "(I[B)V");
diff --git a/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.h b/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.h
index f9e9bc7acf5..f8d89e25131 100644
--- a/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.h
+++ b/src/native/libs/System.Security.Cryptography.Native.Android/pal_jni.h
@@ -101,6 +101,21 @@ extern jmethodID g_sslCtxGetDefaultSslParamsMethod;
 extern jclass    g_GCMParameterSpecClass;
 extern jmethodID g_GCMParameterSpecCtor;
 
+// java/security/spec/MGF1ParameterSpec
+extern jclass   g_MGF1ParameterSpecClass;
+extern jfieldID g_MGF1ParameterSpec_SHA1Field;
+extern jfieldID g_MGF1ParameterSpec_SHA256Field;
+extern jfieldID g_MGF1ParameterSpec_SHA384Field;
+extern jfieldID g_MGF1ParameterSpec_SHA512Field;
+
+// javax/crypto/spec/OAEPParameterSpec
+extern jclass    g_OAEPParameterSpecClass;
+extern jmethodID g_OAEPParameterSpecCtor;
+
+// javax/crypto/spec/PSource$PSpecified
+extern jclass    g_PSourcePSpecifiedClass;
+extern jfieldID  g_PSourcePSpecified_DefaultField;
+
 // java/security/cert/CertificateFactory
 extern jclass    g_CertFactoryClass;
 extern jmethodID g_CertFactoryGetInstance;
diff --git a/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.c b/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.c
index 91fcc38ffb2..e212812b2d7 100644
--- a/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.c
+++ b/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.c
@@ -8,6 +8,52 @@
 
 #define RSA_FAIL -1
 
+static jobject GetRsaOaepPadding(JNIEnv* env, RsaPadding padding)
+{
+    assert(padding >= OaepSHA1 && padding <= OaepSHA512);
+
+    jobject oaepParameterSpec;
+    INIT_LOCALS(oaepLocals, oaepDigest, mgf1, mgf1ParameterSpec, pSource);
+
+    oaepLocals[mgf1] = make_java_string(env, "MGF1");
+    oaepLocals[pSource] = (*env)->GetStaticObjectField(env, g_PSourcePSpecifiedClass, g_PSourcePSpecified_DefaultField);
+
+    if (padding == OaepSHA1)
+    {
+        oaepLocals[mgf1ParameterSpec] = (*env)->GetStaticObjectField(env, g_MGF1ParameterSpecClass, g_MGF1ParameterSpec_SHA1Field);
+        oaepLocals[oaepDigest] = make_java_string(env, "SHA-1");
+    }
+    else if (padding == OaepSHA256)
+    {
+        oaepLocals[mgf1ParameterSpec] = (*env)->GetStaticObjectField(env, g_MGF1ParameterSpecClass, g_MGF1ParameterSpec_SHA256Field);
+        oaepLocals[oaepDigest] = make_java_string(env, "SHA-256");
+    }
+    else if (padding == OaepSHA384)
+    {
+        oaepLocals[mgf1ParameterSpec] = (*env)->GetStaticObjectField(env, g_MGF1ParameterSpecClass, g_MGF1ParameterSpec_SHA384Field);
+        oaepLocals[oaepDigest] = make_java_string(env, "SHA-384");
+    }
+    else if (padding == OaepSHA512)
+    {
+        oaepLocals[mgf1ParameterSpec] = (*env)->GetStaticObjectField(env, g_MGF1ParameterSpecClass, g_MGF1ParameterSpec_SHA512Field);
+        oaepLocals[oaepDigest] = make_java_string(env, "SHA-512");
+    }
+    else
+    {
+        RELEASE_LOCALS(oaepLocals, env);
+        return FAIL;
+    }
+
+    oaepParameterSpec = (*env)->NewObject(
+        env,
+        g_OAEPParameterSpecClass,
+        g_OAEPParameterSpecCtor,
+        oaepLocals[oaepDigest], oaepLocals[mgf1], oaepLocals[mgf1ParameterSpec], oaepLocals[pSource]);
+
+    RELEASE_LOCALS(oaepLocals, env);
+    return CheckJNIExceptions(env) ? FAIL : oaepParameterSpec;
+}
+
 PALEXPORT RSA* AndroidCryptoNative_RsaCreate()
 {
     RSA* rsa = xcalloc(1, sizeof(RSA));
@@ -50,26 +96,35 @@ PALEXPORT int32_t AndroidCryptoNative_RsaPublicEncrypt(int32_t flen, uint8_t* fr
     if ((flen > 0 && !from) || flen < 0)
         return RSA_FAIL;
 
+    if (padding < Pkcs1 || padding > OaepSHA512)
+        return RSA_FAIL;
+
     JNIEnv* env = GetJNIEnv();
 
     int32_t ret = RSA_FAIL;
     INIT_LOCALS(loc, algName, cipher, fromBytes, encryptedBytes);
+    jobject oaepParameterSpec = NULL;
 
     if (padding == Pkcs1)
     {
         loc[algName] = make_java_string(env, "RSA/ECB/PKCS1Padding");
-    }
-    else if (padding == OaepSHA1)
-    {
-        loc[algName] = make_java_string(env, "RSA/ECB/OAEPPadding");
+        loc[cipher] = (*env)->CallStaticObjectMethod(env, g_cipherClass, g_cipherGetInstanceMethod, loc[algName]);
+        (*env)->CallVoidMethod(env, loc[cipher], g_cipherInit2Method, CIPHER_ENCRYPT_MODE, rsa->publicKey);
     }
     else
     {
-        loc[algName] = make_java_string(env, "RSA/ECB/NoPadding");
+        loc[algName] = make_java_string(env, "RSA/ECB/OAEPPadding");
+        loc[cipher] = (*env)->CallStaticObjectMethod(env, g_cipherClass, g_cipherGetInstanceMethod, loc[algName]);
+        oaepParameterSpec = GetRsaOaepPadding(env, padding);
+
+        if (oaepParameterSpec == FAIL)
+        {
+            goto cleanup;
+        }
+
+        (*env)->CallVoidMethod(env, loc[cipher], g_cipherInitMethod, CIPHER_ENCRYPT_MODE, rsa->publicKey, oaepParameterSpec);
     }
 
-    loc[cipher] = (*env)->CallStaticObjectMethod(env, g_cipherClass, g_cipherGetInstanceMethod, loc[algName]);
-    (*env)->CallVoidMethod(env, loc[cipher], g_cipherInit2Method, CIPHER_ENCRYPT_MODE, rsa->publicKey);
     loc[fromBytes] = make_java_byte_array(env, flen);
     (*env)->SetByteArrayRegion(env, loc[fromBytes], 0, flen, (jbyte*)from);
     loc[encryptedBytes] = (jbyteArray)(*env)->CallObjectMethod(env, loc[cipher], g_cipherDoFinal2Method, loc[fromBytes]);
@@ -82,6 +137,12 @@ PALEXPORT int32_t AndroidCryptoNative_RsaPublicEncrypt(int32_t flen, uint8_t* fr
 
 cleanup:
     RELEASE_LOCALS(loc, env);
+
+    if (oaepParameterSpec != NULL && oaepParameterSpec != FAIL)
+    {
+        (*env)->DeleteLocalRef(env, oaepParameterSpec);
+    }
+
     return ret;
 }
 
@@ -93,21 +154,39 @@ PALEXPORT int32_t AndroidCryptoNative_RsaPrivateDecrypt(int32_t flen, uint8_t* f
     if (!rsa->privateKey)
         return RSA_FAIL;
 
+    if (padding < Pkcs1 || padding > OaepSHA512)
+        return RSA_FAIL;
+
     abort_if_invalid_pointer_argument (to);
     abort_if_invalid_pointer_argument (from);
 
     JNIEnv* env = GetJNIEnv();
-
+    jobject cipher;
     jobject algName;
+    jobject oaepParameterSpec = NULL;
+
     if (padding == Pkcs1)
-        algName = make_java_string(env, "RSA/ECB/PKCS1Padding"); // TODO: is ECB needed here?
-    else if (padding == OaepSHA1)
-        algName = make_java_string(env, "RSA/ECB/OAEPPadding");
+    {
+        algName = make_java_string(env, "RSA/ECB/PKCS1Padding");
+        cipher = (*env)->CallStaticObjectMethod(env, g_cipherClass, g_cipherGetInstanceMethod, algName);
+        (*env)->CallVoidMethod(env, cipher, g_cipherInit2Method, CIPHER_DECRYPT_MODE, rsa->privateKey);
+    }
     else
-        algName = make_java_string(env, "RSA/ECB/NoPadding");
+    {
+        algName = make_java_string(env, "RSA/ECB/OAEPPadding");
+        cipher = (*env)->CallStaticObjectMethod(env, g_cipherClass, g_cipherGetInstanceMethod, algName);
+        oaepParameterSpec = GetRsaOaepPadding(env, padding);
+
+        if (oaepParameterSpec == FAIL)
+        {
+            (*env)->DeleteLocalRef(env, algName);
+            (*env)->DeleteLocalRef(env, cipher);
+            return RSA_FAIL;
+        }
+
+        (*env)->CallVoidMethod(env, cipher, g_cipherInitMethod, CIPHER_DECRYPT_MODE, rsa->privateKey, oaepParameterSpec);
+    }
 
-    jobject cipher = (*env)->CallStaticObjectMethod(env, g_cipherClass, g_cipherGetInstanceMethod, algName);
-    (*env)->CallVoidMethod(env, cipher, g_cipherInit2Method, CIPHER_DECRYPT_MODE, rsa->privateKey);
     jbyteArray fromBytes = make_java_byte_array(env, flen);
     (*env)->SetByteArrayRegion(env, fromBytes, 0, flen, (jbyte*)from);
     jbyteArray decryptedBytes = (jbyteArray)(*env)->CallObjectMethod(env, cipher, g_cipherDoFinal2Method, fromBytes);
@@ -117,6 +196,7 @@ PALEXPORT int32_t AndroidCryptoNative_RsaPrivateDecrypt(int32_t flen, uint8_t* f
         (*env)->DeleteLocalRef(env, cipher);
         (*env)->DeleteLocalRef(env, fromBytes);
         (*env)->DeleteLocalRef(env, algName);
+        (*env)->DeleteLocalRef(env, oaepParameterSpec);
         return RSA_FAIL;
     }
 
@@ -128,6 +208,11 @@ PALEXPORT int32_t AndroidCryptoNative_RsaPrivateDecrypt(int32_t flen, uint8_t* f
     (*env)->DeleteLocalRef(env, decryptedBytes);
     (*env)->DeleteLocalRef(env, algName);
 
+    if (oaepParameterSpec != NULL && oaepParameterSpec != FAIL)
+    {
+        (*env)->DeleteLocalRef(env, oaepParameterSpec);
+    }
+
     return (int32_t)decryptedBytesLen;
 }
 
diff --git a/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.h b/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.h
index 63b98d7738c..13f6113aab4 100644
--- a/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.h
+++ b/src/native/libs/System.Security.Cryptography.Native.Android/pal_rsa.h
@@ -10,7 +10,9 @@ typedef enum
 {
     Pkcs1 = 0,
     OaepSHA1 = 1,
-    NoPadding = 2,
+    OaepSHA256 = 2,
+    OaepSHA384 = 3,
+    OaepSHA512 = 4,
 } RsaPadding;
 
 typedef struct RSA