[mono] Throw a MarshalDirectiveException when marshalling generic instances as return types from pinvoke callbacks. (#21402)

Fixes https://github.com/dotnet/runtime/issues/63962.

4 files changed, 41 insertions, 1 deletions

diff --git a/mono/metadata/marshal-ilgen.c b/mono/metadata/marshal-ilgen.c
index ea8880024fa..66f680aca59 100644
--- a/mono/metadata/marshal-ilgen.c
+++ b/mono/metadata/marshal-ilgen.c
@@ -6287,7 +6287,7 @@ emit_managed_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethodSignature *invoke_s
 	/* ret = method (...) */
 	mono_mb_emit_managed_call (mb, method, NULL);
 
-	if (MONO_TYPE_ISSTRUCT (sig->ret)) {
+	if (MONO_TYPE_ISSTRUCT (sig->ret) && sig->ret->type != MONO_TYPE_GENERICINST) {
 		MonoClass *klass = mono_class_from_mono_type_internal (sig->ret);
 		mono_class_init_internal (klass);
 		if (!(mono_class_is_explicit_layout (klass) || m_class_is_blittable (klass))) {
@@ -6331,6 +6331,10 @@ emit_managed_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethodSignature *invoke_s
 		case MONO_TYPE_SZARRAY:
 			mono_emit_marshal (m, 0, sig->ret, mspecs [0], 0, NULL, MARSHAL_ACTION_MANAGED_CONV_RESULT);
 			break;
+		case MONO_TYPE_GENERICINST: {
+			mono_mb_emit_byte (mb, CEE_POP);
+			break;
+		}
 		default:
 			g_warning ("return type 0x%02x unknown", sig->ret->type);	
 			g_assert_not_reached ();
diff --git a/mono/metadata/marshal.c b/mono/metadata/marshal.c
index e3ca84bc324..c538bab36f6 100644
--- a/mono/metadata/marshal.c
+++ b/mono/metadata/marshal.c
@@ -3801,6 +3801,11 @@ mono_marshal_get_managed_wrapper (MonoMethod *method, MonoClass *delegate_klass,
 		mono_method_get_marshal_info (invoke, mspecs);
 	}
 
+	if (invoke_sig->ret->type == MONO_TYPE_GENERICINST) {
+		mono_error_set_generic_error (error, "System.Runtime.InteropServices", "MarshalDirectiveException", "%s", "Cannot marshal 'return value': Non-blittable generic types cannot be marshaled.");
+		return NULL;
+	}
+
 	sig = mono_method_signature_internal (method);
 
 	mb = mono_mb_new (method->klass, method->name, MONO_WRAPPER_NATIVE_TO_MANAGED);
diff --git a/mono/tests/libtest.c b/mono/tests/libtest.c
index 6ae260dbbe6..a785845fd12 100644
--- a/mono/tests/libtest.c
+++ b/mono/tests/libtest.c
@@ -1155,6 +1155,13 @@ mono_test_marshal_icall_delegate (IcallDelegate del)
 	return strcmp (res, "ABC") == 0 ? 0 : 1;
 }
 
+typedef char* (STDCALL *NullableReturnDelegate) (void);
+LIBTEST_API void STDCALL
+mono_test_marshal_nullable_ret_delegate (NullableReturnDelegate del)
+{
+	del ();
+}
+
 LIBTEST_API int STDCALL  
 mono_test_marshal_stringbuilder (char *s, int n)
 {
diff --git a/mono/tests/pinvoke3.cs b/mono/tests/pinvoke3.cs
index e838c1fc591..ad92b3b6259 100644
--- a/mono/tests/pinvoke3.cs
+++ b/mono/tests/pinvoke3.cs
@@ -197,6 +197,9 @@ public class Tests {
 	[DllImport ("libtest", EntryPoint="mono_test_marshal_icall_delegate")]
 	public static extern int mono_test_marshal_icall_delegate (IcallDelegate del);
 
+	[DllImport ("libtest", EntryPoint="mono_test_marshal_nullable_ret_delegate")]
+	public static extern int mono_test_marshal_nullable_ret_delegate (NullableReturnDelegate del);
+
 	public delegate string IcallDelegate (IntPtr p);
 
 	public delegate int TestDelegate (int a, ref SimpleStruct ss, int b);
@@ -225,6 +228,8 @@ public class Tests {
 
 	public delegate int VirtualDelegate (int i);
 
+	public delegate Nullable<int> NullableReturnDelegate ();
+
 	public static int Main () {
 		return TestDriver.RunTests (typeof (Tests));
 	}
@@ -1268,4 +1273,23 @@ public class Tests {
 
 		return mono_test_marshal_icall_delegate ((IcallDelegate)Delegate.CreateDelegate (typeof (IcallDelegate), m));
 	}
+
+	private static Nullable<int> nullable_ret_cb () {
+		return 0;
+	}
+
+	public static int test_0_generic_return () {
+		try {
+			Marshal.GetFunctionPointerForDelegate<NullableReturnDelegate> (nullable_ret_cb);
+			return 1;
+		} catch (MarshalDirectiveException) {
+		}
+		try {
+			mono_test_marshal_nullable_ret_delegate (nullable_ret_cb);
+			return 2;
+		} catch (MarshalDirectiveException) {
+		}
+
+		return 0;
+	}
 }