Merge pull request #10905 from monojenkins/backport-pr-10899-to-2018-06mono-5.16.0.179

[2018-06] [runtime] Disable stack guard for main thread on osx

Backport of #10899.

/cc @luhenry @BrzVlad

Description of #10899:
On macOS Mojave, it seems that changing the mapping of stack pages for main thread can lead to corruption bugs.

https://github.com/mono/mono/issues/10802



<!--
Thank you for your Pull Request!

If you are new to contributing to Mono, please try to do your best at conforming to our coding guidelines http://www.mono-project.com/community/contributing/coding-guidelines/ but don't worry if you get something wrong. One of the project members will help you to get things landed.

Does your pull request fix any of the existing issues? Please use the following format: Fixes #issue-number
-->

3 files changed, 28 insertions, 8 deletions

diff --git a/mono/mini/mini-exceptions.c b/mono/mini/mini-exceptions.c
index d36a0af37b6..dd04c1edbcc 100644
--- a/mono/mini/mini-exceptions.c
+++ b/mono/mini/mini-exceptions.c
@@ -2703,6 +2703,15 @@ mono_setup_altstack (MonoJitTlsData *tls)
 	size_t stsize = 0;
 	stack_t sa;
 	guint8 *staddr = NULL;
+#ifdef TARGET_OSX
+	/*
+	 * On macOS Mojave we are encountering a bug when changing mapping for main thread
+	 * stack pages. Stack overflow on main thread will kill the app.
+	 */
+	gboolean disable_stack_guard = mono_threads_platform_is_main_thread ();
+#else
+	gboolean disable_stack_guard = FALSE;
+#endif
 
 	if (mono_running_on_valgrind ())
 		return;
@@ -2716,16 +2725,18 @@ mono_setup_altstack (MonoJitTlsData *tls)
 
 	/*g_print ("thread %p, stack_base: %p, stack_size: %d\n", (gpointer)pthread_self (), staddr, stsize);*/
 
-	tls->stack_ovf_guard_base = staddr + mono_pagesize ();
-	tls->stack_ovf_guard_size = ALIGN_TO (8 * 4096, mono_pagesize ());
+	if (!disable_stack_guard) {
+		tls->stack_ovf_guard_base = staddr + mono_pagesize ();
+		tls->stack_ovf_guard_size = ALIGN_TO (8 * 4096, mono_pagesize ());
 
-	g_assert ((guint8*)&sa >= (guint8*)tls->stack_ovf_guard_base + tls->stack_ovf_guard_size);
+		g_assert ((guint8*)&sa >= (guint8*)tls->stack_ovf_guard_base + tls->stack_ovf_guard_size);
 
-	if (mono_mprotect (tls->stack_ovf_guard_base, tls->stack_ovf_guard_size, MONO_MMAP_NONE)) {
-		/* mprotect can fail for the main thread stack */
-		gpointer gaddr = mono_valloc (tls->stack_ovf_guard_base, tls->stack_ovf_guard_size, MONO_MMAP_NONE|MONO_MMAP_PRIVATE|MONO_MMAP_ANON|MONO_MMAP_FIXED, MONO_MEM_ACCOUNT_EXCEPTIONS);
-		g_assert (gaddr == tls->stack_ovf_guard_base);
-		tls->stack_ovf_valloced = TRUE;
+		if (mono_mprotect (tls->stack_ovf_guard_base, tls->stack_ovf_guard_size, MONO_MMAP_NONE)) {
+			/* mprotect can fail for the main thread stack */
+			gpointer gaddr = mono_valloc (tls->stack_ovf_guard_base, tls->stack_ovf_guard_size, MONO_MMAP_NONE|MONO_MMAP_PRIVATE|MONO_MMAP_ANON|MONO_MMAP_FIXED, MONO_MEM_ACCOUNT_EXCEPTIONS);
+			g_assert (gaddr == tls->stack_ovf_guard_base);
+			tls->stack_ovf_valloced = TRUE;
+		}
 	}
 
 	/* Setup an alternate signal stack */
@@ -2756,6 +2767,9 @@ mono_free_altstack (MonoJitTlsData *tls)
 
 	if (tls->signal_stack)
 		mono_vfree (tls->signal_stack, MONO_ARCH_SIGNAL_STACK_SIZE, MONO_MEM_ACCOUNT_EXCEPTIONS);
+
+	if (!tls->stack_ovf_guard_base)
+		return;
 	if (tls->stack_ovf_valloced)
 		mono_vfree (tls->stack_ovf_guard_base, tls->stack_ovf_guard_size, MONO_MEM_ACCOUNT_EXCEPTIONS);
 	else
diff --git a/mono/utils/mono-threads-mach.c b/mono/utils/mono-threads-mach.c
index a9f885b5d3a..1507c97ed86 100644
--- a/mono/utils/mono-threads-mach.c
+++ b/mono/utils/mono-threads-mach.c
@@ -259,4 +259,9 @@ mono_threads_platform_get_stack_bounds (guint8 **staddr, size_t *stsize)
 	*staddr -= *stsize;
 }
 
+gboolean
+mono_threads_platform_is_main_thread (void)
+{
+	return pthread_main_np () == 1;
+}
 #endif
diff --git a/mono/utils/mono-threads.h b/mono/utils/mono-threads.h
index 9af75c5b96f..932a8f93645 100644
--- a/mono/utils/mono-threads.h
+++ b/mono/utils/mono-threads.h
@@ -568,6 +568,7 @@ mono_thread_platform_create_thread (MonoThreadStart thread_fn, gpointer thread_d
 	gsize* const stack_size, MonoNativeThreadId *tid);
 
 void mono_threads_platform_get_stack_bounds (guint8 **staddr, size_t *stsize);
+gboolean mono_threads_platform_is_main_thread (void);
 void mono_threads_platform_init (void);
 gboolean mono_threads_platform_in_critical_region (MonoNativeThreadId tid);
 gboolean mono_threads_platform_yield (void);