[2018-06] Crash Reporter V2 (#11162)

* [runtime] Add icall to format crashes to disk in uniform way

* [crash] More gentle error handling around stack walking

* [crash] Make self-summarizier not need managed thread state

* [crash] Remove reliance on sgen/gchandles

* [crash] Refactor dumper mechanism

* [crash] Make threads not die during thread dump

* [crash] Add silent option to dumper

* [crash] Print error when unable to dump thread

* [crash] Add icalls to directly trigger telemetry-style walks

* [crash] Increase maximum frame depth

* [crash] Add support for concurrent global dump requests

* [crash] Switch from CAS+Sleep to semaphore for dumper

Locks and GC-Safe regions are both unsafe in a signal handler
context. CAS primitives and the sleep syscall was used before.
Semaphores appear to be safe, enabling the following cleanup.

* [runtime] Make find_method use stack memory for dumper-facing entry point

* [crash] Enable install_state_summarizer on all platforms

* [crash] Dump managed stacks from controlling thread

* [crash] Fix string races with DumpTotal icall

* [crash] Fix space in crash file name

* [crash] Fix lifetime and size checks on mono-state memory

* [runtime] Make sequence point lookup lockless

* [crash] Fix managed exception dumper icall format

* [crash] Add tests for concurrent dumping

* [crash] Print half-completed dump when space runs out

* [crash] Bump static memory for crash string to 500k

Note:
- This memory isn't accessed until we're crashing
- On any modern machine, unused static memory is lazily allocated. When
we access it, the fault will trigger the MMU to map memory for us.
- This is guaranteed to be reentrant and safe. mmap (the alternative) is
surely fairly stable on many systems, but this is not guaranteed. The
documentation pages for mmap specifically claim it as not safe in these async
contexts.

* [crash] Make whitelist lookup lockless

* [crash] Split unmanaged ptrs into method ip + offset

* [crash] Copy safer strings with crash privacy disabled

* [crash] Fix BindingFlags undefined with mcs+linux x64

* [crash] Fix icall declarations for new style

* [crash] Squashed support for  RegisterReportingForNativeLib

* [crash] Disable tests on linux / non-osx platforms

3 files changed, 235 insertions, 2 deletions

diff --git a/mcs/build/platforms/linux.make b/mcs/build/platforms/linux.make
index 02348318afe..b0a7ebb9af0 100644
--- a/mcs/build/platforms/linux.make
+++ b/mcs/build/platforms/linux.make
@@ -6,7 +6,7 @@
 PLATFORM_MCS_FLAGS =
 PLATFORM_RUNTIME = $(RUNTIME)
 PLATFORM_CORLIB = mscorlib.dll
-PLATFORM_TEST_HARNESS_EXCLUDES =
+PLATFORM_TEST_HARNESS_EXCLUDES = NotOnLinux,
 
 EXTERNAL_MCS = mcs
 EXTERNAL_MBAS = mbas
diff --git a/mcs/class/corlib/Mono/Runtime.cs b/mcs/class/corlib/Mono/Runtime.cs
index 1addbac49ac..1c8cdc4a1ab 100644
--- a/mcs/class/corlib/Mono/Runtime.cs
+++ b/mcs/class/corlib/Mono/Runtime.cs
@@ -63,7 +63,7 @@ namespace Mono {
 
 		// Should not be removed intended for external use
 		// Safe to be called using reflection
-		// Format is undefined only for use as a string for reporting
+		// Format is undefined only for use as a string for 
 		[MethodImplAttribute (MethodImplOptions.InternalCall)]
 #if MOBILE || XAMMAC_4_5
 		public
@@ -105,6 +105,21 @@ namespace Mono {
 		[MethodImplAttribute (MethodImplOptions.InternalCall)]
 		static extern void SendMicrosoftTelemetry_internal (IntPtr payload, ulong portable_hash, ulong unportable_hash);
 
+		[MethodImplAttribute (MethodImplOptions.InternalCall)]
+		static extern void WriteStateToFile_internal (IntPtr payload, ulong portable_hash, ulong unportable_hash);
+
+		static void
+		WriteStateToFile (Exception exc)
+		{
+			ulong portable_hash;
+			ulong unportable_hash;
+			string payload_str = ExceptionToState_internal (exc, out portable_hash, out unportable_hash);
+			using (var payload_chars = RuntimeMarshal.MarshalString (payload_str))
+			{
+				WriteStateToFile_internal (payload_chars.Value, portable_hash, unportable_hash);
+			}
+		}
+
 		static void SendMicrosoftTelemetry (string payload_str, ulong portable_hash, ulong unportable_hash)
 		{
 			if (RuntimeInformation.IsOSPlatform (OSPlatform.OSX)) {
@@ -150,5 +165,43 @@ namespace Mono {
 		}
 #endif
 
+		[MethodImplAttribute (MethodImplOptions.InternalCall)]
+		static extern string DumpStateSingle_internal (out ulong portable_hash, out ulong unportable_hash);
+
+		[MethodImplAttribute (MethodImplOptions.InternalCall)]
+		static extern string DumpStateTotal_internal (out ulong portable_hash, out ulong unportable_hash);
+
+		static Tuple<String, ulong, ulong>
+		DumpStateSingle ()
+		{
+			ulong portable_hash;
+			ulong unportable_hash;
+			string payload_str = DumpStateSingle_internal (out portable_hash, out unportable_hash);
+
+			return new Tuple<String, ulong, ulong> (payload_str, portable_hash, unportable_hash);
+		}
+
+		static Tuple<String, ulong, ulong>
+		DumpStateTotal ()
+		{
+			ulong portable_hash;
+			ulong unportable_hash;
+			string payload_str = DumpStateTotal_internal (out portable_hash, out unportable_hash);
+
+			return new Tuple<String, ulong, ulong> (payload_str, portable_hash, unportable_hash);
+		}
+
+		[MethodImplAttribute (MethodImplOptions.InternalCall)]
+		static extern void RegisterReportingForNativeLib_internal (IntPtr modulePathSuffix, IntPtr moduleName);
+
+		static void RegisterReportingForNativeLib (string modulePathSuffix_str, string moduleName_str)
+		{
+			using (var modulePathSuffix_chars = RuntimeMarshal.MarshalString (modulePathSuffix_str))
+			using (var moduleName_chars = RuntimeMarshal.MarshalString (moduleName_str))
+			{
+				RegisterReportingForNativeLib_internal (modulePathSuffix_chars.Value, moduleName_chars.Value);
+			}
+		}
+
 	}
 }
diff --git a/mcs/class/corlib/Test/System/ExceptionTest.cs b/mcs/class/corlib/Test/System/ExceptionTest.cs
index 36f37e9c66b..68a550b51eb 100644
--- a/mcs/class/corlib/Test/System/ExceptionTest.cs
+++ b/mcs/class/corlib/Test/System/ExceptionTest.cs
@@ -15,6 +15,9 @@ using System.Runtime.Serialization;
 

 using NUnit.Framework;

 

+using System.Threading.Tasks;

+using System.Reflection;

+

 namespace MonoTests.System

 {

 	[TestFixture]

@@ -400,6 +403,183 @@ namespace MonoTests.System
 			Assert.IsNull (a.InnerException.Source);

 		}

 

+#if !MOBILE

+		// Ensure that we can convert a stacktrace to a

+		// telemetry message

+		//

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void StacktraceToState ()

+		{

+			try {

+				throw new Exception ("#0");

+			} catch (Exception exc) {

+				var monoType = Type.GetType ("Mono.Runtime", false);

+				var convert = monoType.GetMethod("ExceptionToState", BindingFlags.NonPublic | BindingFlags.Static);

+				object [] convert_params = new object[] {exc};

+				var output = (Tuple<String, ulong, ulong>) convert.Invoke(null, convert_params);

+

+				var dump = output.Item1;

+				var portable_hash = output.Item2;

+				var unportable_hash = output.Item3;

+

+				Assert.IsTrue (portable_hash != 0, "#1");

+				Assert.IsTrue (unportable_hash != 0, "#2");

+				Assert.IsTrue (dump.Length > 0, "#3");

+

+				// Console.WriteLine (dump);

+				// dump should look something like:

+				// {

+				//  "protocol_version" : "0.0.1",

+				//  "configuration" : {

+				//    "version" : "5.19.0 (managed_telemetry_pipeline/d342c73e320 Wed Aug 15 14:40:40 EDT 2018)",

+				//    "tlc" : "normal",

+				//    "sigsgev" : "altstack",

+				//    "notifications" : "kqueue",

+				//    "architecture" : "amd64",

+				//    "disabled_features" : "none",

+				//    "smallconfig" : "disabled",

+				//    "bigarrays" : "disabled",

+				//    "softdebug" : "enabled",

+				//    "interpreter" : "enabled",

+				//    "llvm_support" : "disabled",

+				//    "suspend" : "hybrid"

+				//  },

+				//  "memory" : {

+				//    "Resident Size" : "40693760",

+				//    "Virtual Size" : "4521312256",

+				//    "minor_gc_time" : "216992",

+				//    "major_gc_time" : "0",

+				//    "minor_gc_count" : "6",

+				//    "major_gc_count" : "0",

+				//    "major_gc_time_concurrent" : "0"

+				//  },

+				//  "threads" : [

+				//    {

+				//      "is_managed" : false,

+				//      "managed_thread_ptr" : "0x0",

+				//      "thread_info_addr" : "0x0",

+				//      "native_thread_id" : "0x0",

+				//      "managed_frames" : [

+				//        {

+				//          "is_managed" : "true",

+				//          "guid" : "43A03618-E657-44B0-B9FA-F63314A3C1B2",

+				//          "token" : "0x60008da",

+				//          "native_offset" : "0xb2",

+				//          "il_offset" : "0x00000"

+				//        }

+				//      ]

+				//    }

+				//  ]

+				//  }

+			}

+		}

+

+		void DumpSingle ()

+		{

+			var monoType = Type.GetType ("Mono.Runtime", false);

+			var convert = monoType.GetMethod("DumpStateSingle", BindingFlags.NonPublic | BindingFlags.Static);

+			var output = (Tuple<String, ulong, ulong>) convert.Invoke(null, Array.Empty<object> ());

+

+			var dump = output.Item1;

+			var portable_hash = output.Item2;

+			var unportable_hash = output.Item3;

+

+			Assert.IsTrue (portable_hash != 0, "#1");

+			Assert.IsTrue (unportable_hash != 0, "#2");

+			Assert.IsTrue (dump.Length > 0, "#3");

+		}

+

+		void DumpTotal ()

+		{

+			var monoType = Type.GetType ("Mono.Runtime", false);

+			var convert = monoType.GetMethod("DumpStateTotal", BindingFlags.NonPublic | BindingFlags.Static);

+			var output = (Tuple<String, ulong, ulong>) convert.Invoke(null, Array.Empty<object> ());

+

+			var dump = output.Item1;

+			var portable_hash = output.Item2;

+			var unportable_hash = output.Item3;

+

+			Assert.IsTrue (portable_hash != 0, "#1");

+			Assert.IsTrue (unportable_hash != 0, "#2");

+			Assert.IsTrue (dump.Length > 0, "#3");

+		}

+

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void DumpICallSingleOnce ()

+		{

+			DumpSingle ();

+		}

+

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void DumpICallSingleConcurrent ()

+		{

+			// checks that self-dumping works in parallel, locklessly

+			int amt = 20;

+			var tasks = new Task [amt];

+			for (int i=0; i < amt; i++)

+				tasks [i] = Task.Run(() => DumpSingle ());

+			Task.WaitAll (tasks);

+		}

+

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void DumpICallSingleAsync ()

+		{

+			// checks that dumping works in an async context

+			var t = Task.Run(() => DumpSingle ());

+			t.Wait ();

+		}

+

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void DumpICallTotalOnce ()

+		{

+			DumpTotal ();

+		}

+

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void DumpICallTotalRepeated ()

+		{

+			// checks that the state doesn't get broken with repeated use

+			DumpTotal ();

+			DumpTotal ();

+			DumpTotal ();

+		}

+

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void DumpICallTotalAsync ()

+		{

+			// checks that dumping works in an async context

+			var t = Task.Run(() => DumpTotal ());

+			t.Wait ();

+		}

+

+		[Test]

+		[Category("NotOnWindows")]

+		[Category("NotOnLinux")]

+		public void DumpICallTotalConcurrent ()

+		{

+			int amt = 3;

+			var tasks = new Task [amt];

+			for (int i=0; i < amt; i++)

+				tasks [i] = Task.Run(() => DumpTotal ());

+			Task.WaitAll (tasks);

+		}

+

+#endif

+

 		[Test]

 		public void StackTrace ()

 		{