Implement marking of custom marshaler required members (#1712)

Co-authored-by: Michal Strehovský <MichalStrehovsky@users.noreply.github.com>

2 files changed, 142 insertions, 4 deletions

diff --git a/src/linker/Linker.Steps/MarkStep.cs b/src/linker/Linker.Steps/MarkStep.cs
index 036a81061..114dee794 100644
--- a/src/linker/Linker.Steps/MarkStep.cs
+++ b/src/linker/Linker.Steps/MarkStep.cs
@@ -588,7 +588,13 @@ namespace Mono.Linker.Steps
 
 			if (spec.MarshalInfo is CustomMarshalInfo marshaler) {
 				MarkType (marshaler.ManagedType, reason, sourceLocationMember);
-				MarkGetInstanceMethod (marshaler.ManagedType.Resolve (), in reason, sourceLocationMember);
+				TypeDefinition type = marshaler.ManagedType.Resolve ();
+				if (type != null) {
+					MarkICustomMarshalerMethods (type, in reason, sourceLocationMember);
+					MarkCustomMarshalerGetInstance (type, in reason, sourceLocationMember);
+				} else {
+					HandleUnresolvedType (marshaler.ManagedType);
+				}
 			}
 		}
 
@@ -2007,16 +2013,45 @@ namespace Mono.Linker.Steps
 			return MarkMethodIf (type.Methods, MethodDefinitionExtensions.IsDefaultConstructor, reason, sourceLocationMember) != null;
 		}
 
-		void MarkGetInstanceMethod (TypeDefinition type, in DependencyInfo reason, IMemberDefinition sourceLocationMember)
+		void MarkCustomMarshalerGetInstance (TypeDefinition type, in DependencyInfo reason, IMemberDefinition sourceLocationMember)
 		{
-			if (type?.HasMethods != true)
+			if (!type.HasMethods)
 				return;
 
 			MarkMethodIf (type.Methods, m =>
-				m.Name == "GetInstance" && m.Parameters.Count == 1 && m.Parameters[0].ParameterType.MetadataType == MetadataType.String,
+				m.Name == "GetInstance" && m.IsStatic && m.Parameters.Count == 1 && m.Parameters[0].ParameterType.MetadataType == MetadataType.String,
 				reason, sourceLocationMember);
 		}
 
+		void MarkICustomMarshalerMethods (TypeDefinition type, in DependencyInfo reason, IMemberDefinition sourceLocationMember)
+		{
+			do {
+				if (!type.HasInterfaces)
+					continue;
+
+				foreach (var iface in type.Interfaces) {
+					var iface_type = iface.InterfaceType;
+					if (!iface_type.IsTypeOf ("System.Runtime.InteropServices", "ICustomMarshaler"))
+						continue;
+
+					//
+					// Instead of trying to guess where to find the interface declaration linker walks
+					// the list of implemented interfaces and resolve the declaration from there
+					//
+					var tdef = iface_type.Resolve ();
+					if (tdef == null) {
+						HandleUnresolvedType (iface_type);
+						return;
+					}
+
+					MarkMethodsIf (tdef.Methods, m => !m.IsStatic, reason, sourceLocationMember);
+
+					MarkInterfaceImplementation (iface, type);
+					return;
+				}
+			} while ((type = type.BaseType?.Resolve ()) != null);
+		}
+
 		static bool IsNonEmptyStaticConstructor (MethodDefinition method)
 		{
 			if (!method.IsStaticConstructor ())
diff --git a/test/Mono.Linker.Tests.Cases/Attributes/MarshalAsCustomMarshalerInterface.cs b/test/Mono.Linker.Tests.Cases/Attributes/MarshalAsCustomMarshalerInterface.cs
new file mode 100644
index 000000000..ee278e46b
--- /dev/null
+++ b/test/Mono.Linker.Tests.Cases/Attributes/MarshalAsCustomMarshalerInterface.cs
@@ -0,0 +1,103 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Runtime.Serialization;
+using Mono.Linker.Tests.Cases.Expectations.Assertions;
+using Mono.Linker.Tests.Cases.Expectations.Metadata;
+
+namespace Mono.Linker.Tests.Cases.Attributes
+{
+	[SetupLinkerCoreAction ("link")]
+	[SkipPeVerify]
+
+	[KeptInterface (typeof (IUserData))]
+	public class MarshalAsCustomMarshalerInterface : IUserData
+	{
+		[Kept]
+		public MarshalAsCustomMarshalerInterface ()
+		{
+		}
+
+		public static void Main ()
+		{
+			IUserData icm = new MarshalAsCustomMarshalerInterface ();
+			icm.TestM1 (null);
+
+			CustomMarhaler2.StaticMethod ();
+		}
+
+		[Kept]
+		public void TestM1 (object o)
+		{
+		}
+	}
+
+	[Kept]
+	interface IUserData
+	{
+		[Kept]
+		void TestM1 ([MarshalAs (UnmanagedType.CustomMarshaler, MarshalType = "Mono.Linker.Tests.Cases.Attributes.CustomMarhaler1")] object o);
+	}
+
+	[Kept]
+	[KeptInterface (typeof (ICustomMarshaler))]
+	class CustomMarhaler1 : ICustomMarshaler
+	{
+		[Kept]
+		private CustomMarhaler1 ()
+		{
+		}
+
+		[Kept]
+		public static ICustomMarshaler GetInstance (string cookie)
+		{
+			return new CustomMarhaler1 ();
+		}
+
+		[Kept]
+		public Object MarshalNativeToManaged (IntPtr pNativeData) => throw new NotImplementedException ();
+
+		[Kept]
+		public IntPtr MarshalManagedToNative (Object ManagedObj) => throw new NotImplementedException ();
+
+		[Kept]
+		public void CleanUpNativeData (IntPtr pNativeData) => throw new NotImplementedException ();
+
+		[Kept]
+		void ICustomMarshaler.CleanUpManagedData (Object ManagedObj) => throw new NotImplementedException ();
+
+		[Kept]
+		public int GetNativeDataSize () => throw new NotImplementedException ();
+
+		public void ExtraMethod ()
+		{
+		}
+	}
+
+	[Kept]
+	class CustomMarhaler2 : ICustomMarshaler
+	{
+		public CustomMarhaler2 ()
+		{
+		}
+
+		public static ICustomMarshaler GetInstance (string cookie)
+		{
+			return new CustomMarhaler2 ();
+		}
+
+		public Object MarshalNativeToManaged (IntPtr pNativeData) => throw new NotImplementedException ();
+
+		public IntPtr MarshalManagedToNative (Object ManagedObj) => throw new NotImplementedException ();
+
+		public void CleanUpNativeData (IntPtr pNativeData) => throw new NotImplementedException ();
+
+		void ICustomMarshaler.CleanUpManagedData (Object ManagedObj) => throw new NotImplementedException ();
+
+		public int GetNativeDataSize () => throw new NotImplementedException ();
+
+		[Kept]
+		public static void StaticMethod ()
+		{
+		}
+	}
+}