[runtime] Add Dedup Support to MkBundle (#5484)

* [runtime] Fix mkbundle compilation on OSX

* [runtime] Add aot arguments to mkbundle

* [runtime] Enable compiling aot with mkbundle

* [runtime] Add new dedup interface to mkbundle

* [runtime] Fix path resolving for mkbundle

* [runtime] Use mkbundle in BCL tests

* [runtime] Skip building System.Security, System.IdentityModel tests on testing_aot_full

* [runtime] Fix in-tree building for mkbundle

* [runtime] Move testing dll into profile

* [runtime] Add target to mkbundle all tests before CI

* [runtime] Build stripper and use with mkbundle

Preliminary linker support ran into dependency-finding bug
with the monolinker.exe binary. Fix postponed until later.

* [runtime] Skip not supported tests with mkbundle

* [runtime] Don't re-aot when running BCL tests

* [runtime] Provide config to mkbundle

* [runtime] Fix mkbundle internationalization test results, build all

* [runtime] Clean up temp mkbundle aot directory

* [runtime] Add mkbundle support for dedup

* [runtime] Fix tracking of dedup module in mkbundle

* [runtime] Document mkbundle AOT options in man pages

* [runtime] Fix CADMessage generic method argument marshalling

4 files changed, 52 insertions, 1 deletions

diff --git a/mcs/class/System/Makefile b/mcs/class/System/Makefile
index 7a7647521db..125abbda448 100644
--- a/mcs/class/System/Makefile
+++ b/mcs/class/System/Makefile
@@ -177,7 +177,7 @@ $(the_libdir_base)System.Configuration.dll:
 
 $(build_lib): $(CYCLIC_DEP_FILES)
 
-$(test_lib): $(TEST_RESOURCES)
+$(test_lib_output): $(TEST_RESOURCES) $(test_lib_dir)
 
 CLEAN_FILES = $(test_lib).config $(bare_libdir)/System.dll $(secxml_libdir)/System.dll $(bare_libdir)/System.dll.mdb $(secxml_libdir)/System.dll.mdb
 
diff --git a/mcs/class/System/Test/System.IO.Compression/DeflateStreamTest.cs b/mcs/class/System/Test/System.IO.Compression/DeflateStreamTest.cs
index 1aa78a9d993..3681b5cc25c 100644
--- a/mcs/class/System/Test/System.IO.Compression/DeflateStreamTest.cs
+++ b/mcs/class/System/Test/System.IO.Compression/DeflateStreamTest.cs
@@ -46,6 +46,7 @@ namespace MonoTests.System.IO.Compression
 
 		[Test]
 		[ExpectedException (typeof (ArgumentNullException))]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void Constructor_Null ()
 		{
 			DeflateStream ds = new DeflateStream (null, CompressionMode.Compress);
@@ -53,12 +54,14 @@ namespace MonoTests.System.IO.Compression
 
 		[Test]
 		[ExpectedException (typeof (ArgumentException))]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void Constructor_InvalidCompressionMode ()
 		{
 			DeflateStream ds = new DeflateStream (new MemoryStream (), (CompressionMode)Int32.MinValue);
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckCompressDecompress ()
 		{
 			byte [] data = new byte[100000];
@@ -81,6 +84,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckDecompress ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -92,6 +96,7 @@ namespace MonoTests.System.IO.Compression
 
 		// https://bugzilla.xamarin.com/show_bug.cgi?id=22346
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckEmptyRead ()
 		{
 			byte [] dummy = new byte[1];
@@ -102,6 +107,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ArgumentNullException))]
 		public void CheckNullRead ()
 		{
@@ -111,6 +117,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (InvalidOperationException))]
 		public void CheckCompressingRead ()
 		{
@@ -121,6 +128,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ArgumentException))]
 		public void CheckRangeRead ()
 		{
@@ -132,6 +140,7 @@ namespace MonoTests.System.IO.Compression
 
 #if !MOBILE
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[Category("NotWorking")]
 		[ExpectedException (typeof (InvalidDataException))]
 		public void CheckInvalidDataRead ()
@@ -145,6 +154,7 @@ namespace MonoTests.System.IO.Compression
 #endif
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ObjectDisposedException))]
 		public void CheckClosedRead ()
 		{
@@ -156,6 +166,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ObjectDisposedException))]
 		public void CheckClosedFlush ()
 		{
@@ -166,6 +177,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckSeek ()
 		{
@@ -175,6 +187,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckSetLength ()
 		{
@@ -184,6 +197,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckGetCanSeekProp ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -204,6 +218,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckGetCanReadProp ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -224,6 +239,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckGetCanWriteProp ()
 		{
 			MemoryStream backing = new MemoryStream ();
@@ -244,6 +260,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckSetLengthProp ()
 		{
@@ -253,6 +270,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckGetLengthProp ()
 		{
@@ -262,6 +280,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckGetPositionProp ()
 		{
@@ -271,6 +290,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void DisposeTest ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -284,6 +304,7 @@ namespace MonoTests.System.IO.Compression
 
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void JunkAtTheEnd ()
 		{
 			// Write a deflated stream, then some additional data...
@@ -330,6 +351,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void Bug19313 ()
 		{
 			byte [] buffer  = new byte [512];
@@ -344,6 +366,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckNet45Overloads () // Xambug #21982
 		{
 			MemoryStream dataStream = GenerateStreamFromString("Hello");
@@ -362,6 +385,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ArgumentException))]
 		public void CheckBufferOverrun ()
 		{
@@ -374,6 +398,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void Bug28777_EmptyFlush ()
 		{
 			MemoryStream backing = new MemoryStream ();
@@ -384,6 +409,7 @@ namespace MonoTests.System.IO.Compression
 		}
 		
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void Bug28777_DoubleFlush ()
 		{
 			byte[] buffer = new byte [4096];
@@ -397,6 +423,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void Bug34916_Inflate ()
 		{
 			var base64String = @"H4sIAAAAAAAAA6yVu27bQBBF/4VtZGHeD3ZJmhTp5C5IIUiEIcCWDEUugiD/nmEQwYRNURFAsuFwd2exZ++d+farud89davT+um5aRsC1DuEO+R7lJayRV9m5gegFqBZNB83m5fjevOzadGWUPHjaXd62XYVEy3Z04wiMTKIX0dfV0G/6FO3Pu72D/+iL916W9GbOV/X58SaS6zEKKyoGUA1eNg/nLfF2jUEBBNMtT4Wzeq567Z9HkZkE1Osf93msN/+WO32m+7zsavsh30/BUU8fy+uUCC+QIHpPQW1RAXkEGWUmSnUy2iUYSMYOGpARYViiIHcqY5kExS8rg2vY8gLGEjeYsClBVE4ORQHz3kxsEF4iS01xzBIZkgYQcYQQ7C54LQaIrxWn5+4ioT1BiRQN8Fh6MrOPjOS9Eh3M8YRJJQMZioJkUODFA8RNJ9AYuYBNyGJW5D0oi3/EpZ3dWYk5X5PN81RJGJgDATMQ5X02nFS1imVlMGvu0XwBg5/K1hY1U8tecxcNDy1/FAnG+OAQSi9PliHRaNUiuoxQYFB6T8oyAUKEu9LJ6oipbr1spyZArhWX6qbi7EOUrs7SCAoDNVgzKagMlUz+q6DQ4N8/yM=";
@@ -415,6 +442,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void Bug44994_Inflate()
 		{
 			var base64String = @"7cWxCQAgDACwpeBjgqsgXiHU0fd9QzBLErX1EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADepcxcuU/atm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm3btm37zy8=";
@@ -434,6 +462,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[Category ("MobileNotWorking")]
 		public void Bug44994_InflateByteByByte()
 		{
diff --git a/mcs/class/System/Test/System.IO.Compression/GzipStreamTest.cs b/mcs/class/System/Test/System.IO.Compression/GzipStreamTest.cs
index 47a740ca14a..243c30ebe10 100644
--- a/mcs/class/System/Test/System.IO.Compression/GzipStreamTest.cs
+++ b/mcs/class/System/Test/System.IO.Compression/GzipStreamTest.cs
@@ -44,6 +44,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ArgumentNullException))]
 		public void Constructor_Null ()
 		{
@@ -51,6 +52,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ArgumentException))]
 		public void Constructor_InvalidCompressionMode ()
 		{
@@ -58,6 +60,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckCompressDecompress ()
 		{
 			byte [] data = new byte[100000];
@@ -80,6 +83,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckDecompress ()
 		{
 			byte [] data = {0x1f, 0x8b, 0x08, 0x08, 0x70, 0xbb, 0x5d, 0x41, 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0xe7, 0x02, 0x00, 0x16, 0x35, 0x96, 0x31, 0x06, 0x00, 0x00, 0x00 };
@@ -91,6 +95,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ArgumentNullException))]
 		public void CheckNullRead ()
 		{
@@ -101,6 +106,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (InvalidOperationException))]
 		public void CheckCompressingRead ()
 		{
@@ -111,6 +117,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ArgumentException))]
 		public void CheckRangeRead ()
 		{
@@ -123,6 +130,7 @@ namespace MonoTests.System.IO.Compression
 
 #if !MOBILE
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[Category("NotWorking")]
 		public void CheckInvalidDataRead ()
 		{
@@ -139,6 +147,7 @@ namespace MonoTests.System.IO.Compression
 #endif
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckClosedRead ()
 		{
 			byte [] dummy = new byte[20];
@@ -153,6 +162,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (ObjectDisposedException))]
 		public void CheckClosedFlush ()
 		{
@@ -163,6 +173,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckSeek ()
 		{
@@ -172,6 +183,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckSetLength ()
 		{
@@ -181,6 +193,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckGetCanSeekProp ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -201,6 +214,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckGetCanReadProp ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -221,6 +235,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckGetCanWriteProp ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -241,6 +256,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckSetLengthProp ()
 		{
@@ -250,6 +266,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckGetLengthProp ()
 		{
@@ -259,6 +276,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		[ExpectedException (typeof (NotSupportedException))]
 		public void CheckGetPositionProp ()
 		{
@@ -268,6 +286,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void DisposeTest ()
 		{
 			MemoryStream backing = new MemoryStream (compressed_data);
@@ -277,6 +296,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void DisposeOrderTest ()
 		{
 			var fs = new MemoryStream();
@@ -304,6 +324,7 @@ namespace MonoTests.System.IO.Compression
 		}
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 		public void CheckNet45Overloads () // Xambug #21982
 		{
 			MemoryStream dataStream = GenerateStreamFromString("Hello");
diff --git a/mcs/class/System/Test/System.Net/HttpWebResponseTest.cs b/mcs/class/System/Test/System.Net/HttpWebResponseTest.cs
index ffa7f6359e5..b35c8e7a595 100644
--- a/mcs/class/System/Test/System.Net/HttpWebResponseTest.cs
+++ b/mcs/class/System/Test/System.Net/HttpWebResponseTest.cs
@@ -1220,6 +1220,7 @@ namespace MonoTests.System.Net
 
 
 		[Test]
+		[Category ("StaticLinkedAotNotWorking")] // Native MPH loading issues
 #if FEATURE_NO_BSD_SOCKETS
 		[ExpectedException (typeof (PlatformNotSupportedException))]
 #endif