Run all managed code in a subthread. Re-enable GC threaded finalisation.

Fixes bugs 34263 and 31333.

(All tests still behave the same way - 4 fails)

2003-01-23  Dick Porter  <dick@ximian.com>

	* threads.c (start_wrapper): Create a Thread object if needed, so
	the Main() thread can do the class initialisation in a subthread
	that has been set up to allow managed code execution.

	Pass the thread ID instead of the MonoThread pointer to the thread
	start and attach callbacks.  This change is required, because the
	jit thread start callback must be called _before_ the Thread
	object can be created.

	(mono_thread_init): Removed much object creation code that is no
	longer needed.  No managed code is called from here now.

	* object.c (mono_runtime_exec_managed_code): Create a subthread
	for Main, and call back to the runtime to use it.
	Set the exit code when Main exits.

	* gc.c: Make sure domain finalisation happens in a subthread.
	Re-enable threaded GC, fixing bug 31333 (again).

	* environment.c: System.Environment internall calls (so far just
	ExitCode is here, the others are still in icall.c)

	* appdomain.c (mono_runtime_cleanup): All threads running managed
	code should have finished before mono_runtime_cleanup() is
	reached, so no need to clean up threads.


2003-01-23  Dick Porter  <dick@ximian.com>

	* mono.c: Use mono_runtime_exec_managed_code() to run all managed
	code in a subthread.

	* jit.c: Changed thread start and attach callbacks to pass the
	thread ID, not the MonoThread pointer.  Arrange that managed code
	execution will fail an assertion in the main thread, just to be
	sure.


2003-01-23  Dick Porter  <dick@ximian.com>

	* interp.c: Use mono_runtime_exec_managed_code() to run all
	managed code in a subthread.

svn path=/trunk/mono/; revision=10838

19 files changed, 466 insertions, 171 deletions

diff --git a/mono/interpreter/ChangeLog b/mono/interpreter/ChangeLog
index a1f6a082559..a981e2aa3bc 100644
--- a/mono/interpreter/ChangeLog
+++ b/mono/interpreter/ChangeLog
@@ -1,3 +1,8 @@
+2003-01-23  Dick Porter  <dick@ximian.com>
+
+	* interp.c: Use mono_runtime_exec_managed_code() to run all
+	managed code in a subthread.
+
 2002-11-15  Dick Porter  <dick@ximian.com>
 
 	* interp.c: mono_runtime_init() now has an extra parameter for
diff --git a/mono/interpreter/interp.c b/mono/interpreter/interp.c
index 9dff81d86c6..159aee3cdb4 100644
--- a/mono/interpreter/interp.c
+++ b/mono/interpreter/interp.c
@@ -54,6 +54,7 @@
 #include <mono/metadata/socket-io.h>
 #include <mono/metadata/mono-config.h>
 #include <mono/metadata/marshal.h>
+#include <mono/metadata/environment.h>
 #include <mono/os/util.h>
 
 /*#include <mono/cli/types.h>*/
@@ -4414,14 +4415,53 @@ ves_icall_get_trace (MonoException *exc, gint32 skip, MonoBoolean need_file_info
 	return NULL;
 }
 
+typedef struct
+{
+	MonoDomain *domain;
+	char *file;
+	int argc;
+	char **argv;
+} MainThreadArgs;
+
+static void main_thread_handler (gpointer user_data)
+{
+	MainThreadArgs *main_args=(MainThreadArgs *)user_data;
+	MonoAssembly *assembly;
+	char *error;
+
+	assembly = mono_domain_assembly_open (main_args->domain,
+					      main_args->file);
+
+	if (!assembly){
+		fprintf (stderr, "Can not open image %s\n", main_args->file);
+		exit (1);
+	}
+
+
+#ifdef RUN_TEST
+	test_load_class (assembly->image);
+#else
+	error = mono_verify_corlib ();
+	if (error) {
+		fprintf (stderr, "Corlib not in sync with this runtime: %s\n", error);
+		exit (1);
+	}
+	segv_exception = mono_get_exception_null_reference ();
+	segv_exception->message = mono_string_new (main_args->domain, "Segmentation fault");
+	signal (SIGSEGV, segv_handler);
+
+	ves_exec (main_args->domain, assembly, main_args->argc, main_args->argv);
+#endif
+}
+
 int 
 main (int argc, char *argv [])
 {
 	MonoDomain *domain;
-	MonoAssembly *assembly;
 	int retval = 0, i, ocount = 0;
-	char *file, *error, *config_file = NULL;
-
+	char *file, *config_file = NULL;
+	MainThreadArgs main_args;
+	
 	if (argc < 2)
 		usage ();
 
@@ -4489,33 +4529,22 @@ main (int argc, char *argv [])
 	domain = mono_init (file);
 	mono_runtime_init (domain, NULL, NULL);
 
-	assembly = mono_domain_assembly_open (domain, file);
-
-	if (!assembly){
-		fprintf (stderr, "Can not open image %s\n", file);
-		exit (1);
-	}
-
-
-#ifdef RUN_TEST
-	test_load_class (assembly->image);
-#else
-	error = mono_verify_corlib ();
-	if (error) {
-		fprintf (stderr, "Corlib not in sync with this runtime: %s\n", error);
-		exit (1);
-	}
-	segv_exception = mono_get_exception_null_reference ();
-	segv_exception->message = mono_string_new (domain, "Segmentation fault");
-	signal (SIGSEGV, segv_handler);
-
-	retval = ves_exec (domain, assembly, argc - i, argv + i);
-#endif
+	main_args.domain=domain;
+	main_args.file=file;
+	main_args.argc=argc-i;
+	main_args.argv=argv+i;
+	
+	mono_runtime_exec_managed_code (domain, main_thread_handler,
+					&main_args);
+	
 	mono_profiler_shutdown ();
 	
 	mono_runtime_cleanup (domain);
 	mono_domain_unload (domain, TRUE);
 
+	/* Get the return value from System.Environment.ExitCode */
+	retval=mono_environment_exitcode_get ();
+
 #if DEBUG_INTERP
 	if (ocount) {
 		fprintf (stderr, "opcode count: %ld\n", opcode_count);
diff --git a/mono/jit/ChangeLog b/mono/jit/ChangeLog
index 5540ce8c928..2206ba51fdb 100644
--- a/mono/jit/ChangeLog
+++ b/mono/jit/ChangeLog
@@ -1,3 +1,13 @@
+2003-01-23  Dick Porter  <dick@ximian.com>
+
+	* mono.c: Use mono_runtime_exec_managed_code() to run all managed
+	code in a subthread.
+
+	* jit.c: Changed thread start and attach callbacks to pass the
+	thread ID, not the MonoThread pointer.  Arrange that managed code
+	execution will fail an assertion in the main thread, just to be
+	sure.
+
 2003-01-22  Martin Baulig  <martin@ximian.com>
 
 	* debug.c: Reverted Gonzalo's last change since it is wrong.
diff --git a/mono/jit/jit.c b/mono/jit/jit.c
index 116865e57c9..1886ed4d528 100644
--- a/mono/jit/jit.c
+++ b/mono/jit/jit.c
@@ -4086,7 +4086,7 @@ mono_thread_abort (MonoObject *obj)
 }
 
 static void
-mono_thread_start_cb (MonoThread *thread, gpointer stack_start, gpointer func)
+mono_thread_start_cb (guint32 tid, gpointer stack_start, gpointer func)
 {
 	MonoJitTlsData *jit_tls;
 	MonoLMF *lmf;
@@ -4103,7 +4103,7 @@ mono_thread_start_cb (MonoThread *thread, gpointer stack_start, gpointer func)
 
 	jit_tls->lmf = lmf;
 
-	mono_debugger_event (MONO_DEBUGGER_EVENT_THREAD_CREATED, thread, func);
+	mono_debugger_event (MONO_DEBUGGER_EVENT_THREAD_CREATED, (gpointer)tid, func);
 }
 
 void (*mono_thread_attach_aborted_cb ) (MonoObject *obj) = NULL;
@@ -4118,7 +4118,7 @@ mono_thread_abort_dummy (MonoObject *obj)
 }
 
 static void
-mono_thread_attach_cb (MonoThread *thread, gpointer stack_start)
+mono_thread_attach_cb (guint32 tid, gpointer stack_start)
 {
 	MonoJitTlsData *jit_tls;
 	MonoLMF *lmf;
@@ -4212,7 +4212,13 @@ mono_jit_init (const char *file) {
 	InitializeCriticalSection (metadata_section);
 
 	mono_jit_tls_id = TlsAlloc ();
-	mono_thread_start_cb (NULL, (gpointer)-1, NULL);
+
+	/* Don't set up the main thread for managed code execution -
+	 * this will give a handy assertion fail in
+	 * mono_get_lmf_addr() if any buggy runtime code tries to run
+	 * managed code in this thread.
+	 */
+	/* mono_thread_start_cb (GetCurrentThreadId (), (gpointer)-1, NULL); */
 
 	mono_install_compile_method (mono_jit_compile_method);
 	mono_install_trampoline (arch_create_jit_trampoline);
@@ -4231,12 +4237,6 @@ mono_jit_init (const char *file) {
 void
 mono_jit_cleanup (MonoDomain *domain)
 {
-
-	/* 
-	 * mono_runtime_cleanup() needs to be called early since
-	 * it needs the execution engine still fully working (it will
-	 * wait for other threads to finish).
-	 */
 	mono_runtime_cleanup (domain);
 
 	mono_domain_finalize (domain);
diff --git a/mono/jit/mono.c b/mono/jit/mono.c
index 5a77c7920c1..44c06374aa8 100644
--- a/mono/jit/mono.c
+++ b/mono/jit/mono.c
@@ -16,6 +16,7 @@
 #include "mono/metadata/threadpool.h"
 #include "mono/metadata/mono-config.h"
 #include <mono/metadata/profiler-private.h>
+#include <mono/metadata/environment.h>
 #include <mono/os/util.h>
 #include <locale.h>
 
@@ -185,11 +186,73 @@ usage (char *name)
 	exit (1);
 }
 
+typedef struct 
+{
+	MonoDomain *domain;
+	char *file;
+	gboolean testjit;
+	char *debug_args;
+	char *compile_class;
+	int compile_times;
+	GList *precompile_classes;
+	int verbose;
+	int break_on_main;
+	int argc;
+	char **argv;
+} MainThreadArgs;
+
+static void main_thread_handler (gpointer user_data)
+{
+	MainThreadArgs *main_args=(MainThreadArgs *)user_data;
+	MonoAssembly *assembly;
+	MonoDebugHandle *debug = NULL;
+	
+	assembly = mono_domain_assembly_open (main_args->domain,
+					      main_args->file);
+	if (!assembly){
+		fprintf (stderr, "Can not open image %s\n", main_args->file);
+		exit (1);
+	}
+
+	if (mono_debug_format != MONO_DEBUG_FORMAT_NONE) {
+		gchar **args;
+
+		args = g_strsplit (main_args->debug_args ?
+				   main_args->debug_args : "", ",", -1);
+		debug = mono_debug_open (assembly, mono_debug_format, (const char **) args);
+		g_strfreev (args);
+	}
+
+	if (main_args->testjit) {
+		mono_jit_compile_image (assembly->image, TRUE);
+	} else if (main_args->compile_class) {
+		mono_jit_compile_class (assembly, main_args->compile_class, main_args->compile_times, TRUE);
+	} else {
+		GList *tmp;
+
+		for (tmp = main_args->precompile_classes; tmp; tmp = tmp->next)
+			mono_jit_compile_class (assembly, tmp->data, 1,
+						main_args->verbose);
+
+		if (main_args->break_on_main) {
+			MonoImage *image = assembly->image;
+			MonoMethodDesc *desc;
+			MonoMethod *method;
+
+			method = mono_get_method (image, mono_image_get_entry_point (image), NULL);
+			desc = mono_method_desc_from_method (method);
+			mono_insert_breakpoint_full (desc, FALSE);
+		}
+
+		mono_jit_exec (main_args->domain, assembly, main_args->argc,
+			       main_args->argv);
+	}
+}
+
 int 
 main (int argc, char *argv [])
 {
 	MonoDomain *domain;
-	MonoAssembly *assembly;
 	int retval = 0, i;
 	int compile_times = 1000;
 	char *compile_class = NULL;
@@ -199,8 +262,7 @@ main (int argc, char *argv [])
 	int verbose = FALSE;
 	GList *precompile_classes = NULL;
 	int break_on_main = FALSE;
-	MonoDebugHandle *debug = NULL;
-
+	MainThreadArgs main_args;
 	
 	setlocale(LC_ALL, "");
 	g_log_set_always_fatal (G_LOG_LEVEL_ERROR);
@@ -309,46 +371,27 @@ main (int argc, char *argv [])
 		fprintf (stderr, "Corlib not in sync with this runtime: %s\n", error);
 		exit (1);
 	}
-
-	assembly = mono_domain_assembly_open (domain, file);
-	if (!assembly){
-		fprintf (stderr, "Can not open image %s\n", file);
-		exit (1);
-	}
-
-	if (mono_debug_format != MONO_DEBUG_FORMAT_NONE) {
-		gchar **args;
-
-		args = g_strsplit (debug_args ? debug_args : "", ",", -1);
-		debug = mono_debug_open (assembly, mono_debug_format, (const char **) args);
-		g_strfreev (args);
-	}
-
-	if (testjit) {
-		mono_jit_compile_image (assembly->image, TRUE);
-	} else if (compile_class) {
-		mono_jit_compile_class (assembly, compile_class, compile_times, TRUE);
-	} else {
-		GList *tmp;
-
-		for (tmp = precompile_classes; tmp; tmp = tmp->next)
-			mono_jit_compile_class (assembly, tmp->data, 1, verbose);
-
-		if (break_on_main) {
-			MonoImage *image = assembly->image;
-			MonoMethodDesc *desc;
-			MonoMethod *method;
-
-			method = mono_get_method (image, mono_image_get_entry_point (image), NULL);
-			desc = mono_method_desc_from_method (method);
-			mono_insert_breakpoint_full (desc, FALSE);
-		}
-
-		retval = mono_jit_exec (domain, assembly, argc - i, argv + i);
-	}
+	
+	main_args.domain=domain;
+	main_args.file=file;
+	main_args.testjit=testjit;
+	main_args.debug_args=debug_args;
+	main_args.compile_class=compile_class;
+	main_args.compile_times=compile_times;
+	main_args.precompile_classes=precompile_classes;
+	main_args.verbose=verbose;
+	main_args.break_on_main=break_on_main;
+	main_args.argc=argc-i;
+	main_args.argv=argv+i;
+	
+	mono_runtime_exec_managed_code (domain, main_thread_handler,
+					&main_args);
 
 	mono_profiler_shutdown ();
 	mono_jit_cleanup (domain);
 
+	/* Look up return value from System.Environment.ExitCode */
+	retval=mono_environment_exitcode_get ();
+	
 	return retval;
 }
diff --git a/mono/metadata/ChangeLog b/mono/metadata/ChangeLog
index 0f2e85b9110..82733ece844 100644
--- a/mono/metadata/ChangeLog
+++ b/mono/metadata/ChangeLog
@@ -1,3 +1,31 @@
+2003-01-23  Dick Porter  <dick@ximian.com>
+
+	* threads.c (start_wrapper): Create a Thread object if needed, so
+	the Main() thread can do the class initialisation in a subthread
+	that has been set up to allow managed code execution.
+
+	Pass the thread ID instead of the MonoThread pointer to the thread
+	start and attach callbacks.  This change is required, because the
+	jit thread start callback must be called _before_ the Thread
+	object can be created.
+	
+	(mono_thread_init): Removed much object creation code that is no
+	longer needed.  No managed code is called from here now.
+
+	* object.c (mono_runtime_exec_managed_code): Create a subthread
+	for Main, and call back to the runtime to use it.
+	Set the exit code when Main exits.
+
+	* gc.c: Make sure domain finalisation happens in a subthread.
+	Re-enable threaded GC, fixing bug 31333 (again).
+
+	* environment.c: System.Environment internall calls (so far just
+	ExitCode is here, the others are still in icall.c)
+
+	* appdomain.c (mono_runtime_cleanup): All threads running managed
+	code should have finished before mono_runtime_cleanup() is
+	reached, so no need to clean up threads.
+
 2003-01-22  Martin Baulig  <martin@ximian.com>
 
 	* appdomain.h (MonoThreadStartCB): Added `MonoThread *thread' and
diff --git a/mono/metadata/Makefile.am b/mono/metadata/Makefile.am
index 90f3298e07b..fe5a4f52bc1 100644
--- a/mono/metadata/Makefile.am
+++ b/mono/metadata/Makefile.am
@@ -48,7 +48,9 @@ libmonoruntime_la_SOURCES = \
 	sysmath.h	\
 	sysmath.c	\
 	process.c	\
-	process.h
+	process.h	\
+	environment.c	\
+	environment.h
 
 libmetadata_la_SOURCES = \
 	assembly.c	\
@@ -75,7 +77,8 @@ libmonoruntimeinclude_HEADERS = \
 	debug-mono-symfile.h	\
 	threadpool.h	\
 	threads-types.h	\
-	threads.h
+	threads.h	\
+	environment.h
 
 libmetadatainclude_HEADERS = \
 	assembly.h	\
diff --git a/mono/metadata/appdomain.c b/mono/metadata/appdomain.c
index 68038da50eb..55986e0d768 100644
--- a/mono/metadata/appdomain.c
+++ b/mono/metadata/appdomain.c
@@ -72,7 +72,7 @@ mono_runtime_init (MonoDomain *domain, MonoThreadStartCB start_cb,
 	g_assert (mono_delegate_semaphore != INVALID_HANDLE_VALUE);
 	InitializeCriticalSection (&mono_delegate_section);
 
-	mono_thread_init (domain, start_cb, attach_cb);
+	mono_thread_init (start_cb, attach_cb);
 	
 	/* GC init has to happen after thread init */
 	mono_gc_init ();
@@ -82,14 +82,15 @@ mono_runtime_init (MonoDomain *domain, MonoThreadStartCB start_cb,
 	return;
 }
 
+/* This must not be called while there are still running threads executing
+ * managed code.
+ */
 void
 mono_runtime_cleanup (MonoDomain *domain)
 {
-	mono_thread_cleanup ();
-
-	/* Do this after the thread cleanup, because subthreads might
-	 * still be doing socket calls.
-	 */
+	/* Not really needed, but do it anyway */
+	mono_gc_cleanup ();
+	
 	mono_network_cleanup ();
 }
 
diff --git a/mono/metadata/appdomain.h b/mono/metadata/appdomain.h
index d27e31566d8..e29e0c18db0 100644
--- a/mono/metadata/appdomain.h
+++ b/mono/metadata/appdomain.h
@@ -18,8 +18,9 @@
 #include <mono/utils/mono-hash.h>
 #include <mono/io-layer/io-layer.h>
 
-typedef void (*MonoThreadStartCB) (MonoThread *thread, gpointer stack_start, gpointer func);
-typedef void (*MonoThreadAttachCB) (MonoThread *thread, gpointer stack_start);
+typedef void (*MonoThreadStartCB) (guint32 tid, gpointer stack_start,
+				   gpointer func);
+typedef void (*MonoThreadAttachCB) (guint32 tid, gpointer stack_start);
 
 /* This is a copy of System.AppDomainSetup */
 typedef struct {
diff --git a/mono/metadata/environment.c b/mono/metadata/environment.c
new file mode 100644
index 00000000000..f8406bc998a
--- /dev/null
+++ b/mono/metadata/environment.c
@@ -0,0 +1,25 @@
+/*
+ * environment.c: System.Environment support internal calls
+ *
+ * Author:
+ *	Dick Porter (dick@ximian.com)
+ *
+ * (C) 2002 Ximian, Inc.
+ */
+
+#include <config.h>
+#include <glib.h>
+
+#include <mono/metadata/environment.h>
+
+static gint32 exitcode=0;
+
+gint32 mono_environment_exitcode_get (void)
+{
+	return(exitcode);
+}
+
+void mono_environment_exitcode_set (gint32 value)
+{
+	exitcode=value;
+}
diff --git a/mono/metadata/environment.h b/mono/metadata/environment.h
new file mode 100644
index 00000000000..03ec4b5db9c
--- /dev/null
+++ b/mono/metadata/environment.h
@@ -0,0 +1,16 @@
+/*
+ * environment.h: System.Environment support internal calls
+ *
+ * Author:
+ *	Dick Porter (dick@ximian.com)
+ *
+ * (C) 2002 Ximian, Inc
+ */
+
+#ifndef _MONO_METADATA_ENVIRONMENT_H_
+#define _MONO_METADATA_ENVIRONMENT_H_
+
+extern gint32 mono_environment_exitcode_get (void);
+extern void mono_environment_exitcode_set (gint32 value);
+
+#endif /* _MONO_METADATA_ENVIRONMENT_H_ */
diff --git a/mono/metadata/gc.c b/mono/metadata/gc.c
index f63ddba2f7a..7eb7447e9b7 100644
--- a/mono/metadata/gc.c
+++ b/mono/metadata/gc.c
@@ -158,16 +158,39 @@ finalize_static_data (MonoClass *class, MonoVTable *vtable, GHashTable *todo) {
 	finalize_fields (class, vtable->data, FALSE, todo);
 }
 
-void
-mono_domain_finalize (MonoDomain *domain) {
-
+static guint32
+internal_domain_finalize (gpointer data) {
+	MonoDomain *domain=(MonoDomain *)data;
 	GHashTable *todo = g_hash_table_new (NULL, NULL);
+
+	mono_new_thread_init (GetCurrentThreadId (), todo, NULL);
+	
 #if HAVE_BOEHM_GC
 	GC_gcollect ();
 #endif
 	mono_g_hash_table_foreach (domain->class_vtable_hash, (GHFunc)finalize_static_data, todo);
 	/* FIXME: finalize objects in todo... */
 	g_hash_table_destroy (todo);
+
+	return(0);
+}
+
+void
+mono_domain_finalize (MonoDomain *domain) 
+{
+	HANDLE finalize_thread;
+	
+	/* Need to run managed code in a subthread.
+	 * Mono_domain_finalize() is called from the main thread in
+	 * the jit and the embedded example, hence the thread creation
+	 * here.
+	 */
+	finalize_thread=CreateThread (NULL, 0, internal_domain_finalize, domain, 0, NULL);
+	if(finalize_thread==NULL) {
+		g_assert_not_reached ();
+	}
+	WaitForSingleObject (finalize_thread, INFINITE);
+	CloseHandle (finalize_thread);
 }
 
 void
@@ -415,7 +438,7 @@ static guint32 finalizer_thread (gpointer unused)
 {
 	guint32 stack_start;
 	
-	mono_new_thread_init (NULL, &stack_start, NULL);
+	mono_new_thread_init (GetCurrentThreadId (), &stack_start, NULL);
 	
 	while(!finished) {
 		/* Wait to be notified that there's at least one
@@ -438,7 +461,7 @@ static guint32 finalizer_thread (gpointer unused)
  * It's currently disabled because it still requires some
  * work in the rest of the runtime.
  */
-#undef ENABLE_FINALIZER_THREAD
+#define ENABLE_FINALIZER_THREAD
 
 void mono_gc_init (void)
 {
diff --git a/mono/metadata/icall.c b/mono/metadata/icall.c
index 5edb71f5707..0ecc95b67e2 100644
--- a/mono/metadata/icall.c
+++ b/mono/metadata/icall.c
@@ -38,6 +38,7 @@
 #include <mono/metadata/string-icalls.h>
 #include <mono/metadata/debug-mono-symfile.h>
 #include <mono/metadata/process.h>
+#include <mono/metadata/environment.h>
 #include <mono/io-layer/io-layer.h>
 #include <mono/utils/strtod.h>
 
@@ -3517,6 +3518,8 @@ static gconstpointer icall_map [] = {
 	"System.Environment::get_TickCount", ves_icall_System_Environment_get_TickCount,
 	"System.Environment::Exit", ves_icall_System_Environment_Exit,
 	"System.Environment::get_Platform", ves_icall_System_Environment_get_Platform,
+	"System.Environment::get_ExitCode", mono_environment_exitcode_get,
+	"System.Environment::set_ExitCode", mono_environment_exitcode_set,
 
 	/*
 	 * System.Runtime.Remoting
diff --git a/mono/metadata/object.c b/mono/metadata/object.c
index e2736066c7d..5a9d071f37e 100644
--- a/mono/metadata/object.c
+++ b/mono/metadata/object.c
@@ -24,6 +24,8 @@
 #include <mono/metadata/marshal.h>
 #include "mono/metadata/debug-helpers.h"
 #include "mono/metadata/marshal.h"
+#include <mono/metadata/threads.h>
+#include <mono/metadata/environment.h>
 #include "mono/metadata/profiler-private.h"
 #include <mono/os/gc_wrapper.h>
 
@@ -671,6 +673,25 @@ mono_unhandled_exception (MonoObject *exc)
 }
 
 /*
+ * Launch a new thread to start all setup that requires managed code
+ * to be executed.
+ *
+ * main_func is called back from the thread with main_args as the
+ * parameter.  The callback function is expected to start Main()
+ * eventually.  This function then waits for all managed threads to
+ * finish.
+ */
+void
+mono_runtime_exec_managed_code (MonoDomain *domain,
+				MonoMainThreadFunc main_func,
+				gpointer main_args)
+{
+	mono_thread_create (domain, main_func, main_args);
+
+	mono_thread_manage ();
+}
+
+/*
  * Execute a standard Main() method (args doesn't contain the
  * executable name).
  */
@@ -700,12 +721,21 @@ mono_runtime_exec_main (MonoMethod *method, MonoArray *args, MonoObject **exc)
 			rval = *(guint32 *)((char *)res + sizeof (MonoObject));
 		else
 			rval = -1;
+
+		mono_environment_exitcode_set (rval);
 	} else {
 		mono_runtime_invoke (method, NULL, pa, exc);
 		if (!exc || !*exc)
 			rval = 0;
-		else
+		else {
+			/* If the return type of Main is void, only
+			 * set the exitcode if an exception was thrown
+			 * (we don't want to blow away an
+			 * explicitly-set exit code)
+			 */
 			rval = -1;
+			mono_environment_exitcode_set (rval);
+		}
 	}
 
 	return rval;
diff --git a/mono/metadata/object.h b/mono/metadata/object.h
index f35af0360f2..bd1daad5bc9 100644
--- a/mono/metadata/object.h
+++ b/mono/metadata/object.h
@@ -195,6 +195,7 @@ typedef struct {
 
 typedef MonoObject* (*MonoInvokeFunc)        (MonoMethod *method, void *obj, void **params, MonoObject **exc);
 typedef gpointer    (*MonoCompileFunc)       (MonoMethod *method);
+typedef void	    (*MonoMainThreadFunc)    (gpointer user_data);
 
 #define mono_object_class(obj) (((MonoObject*)(obj))->vtable->klass)
 #define mono_object_domain(obj) (((MonoObject*)(obj))->vtable->domain)
@@ -323,6 +324,11 @@ mono_runtime_invoke_array   (MonoMethod *method, void *obj, MonoArray *params,
 MonoArray*
 mono_runtime_get_main_args  (void);
 
+void
+mono_runtime_exec_managed_code (MonoDomain *domain,
+				MonoMainThreadFunc main_func,
+				gpointer main_args);
+
 int
 mono_runtime_run_main       (MonoMethod *method, int argc, char* argv[], 
 			     MonoObject **exc);
diff --git a/mono/metadata/threadpool.c b/mono/metadata/threadpool.c
index 8ec21717225..0a3008732ac 100644
--- a/mono/metadata/threadpool.c
+++ b/mono/metadata/threadpool.c
@@ -92,11 +92,8 @@ mono_thread_pool_add (MonoObject *target, MonoMethodMessage *msg, MonoDelegate *
 	ReleaseSemaphore (mono_delegate_semaphore, 1, NULL);
 
 	if (workers == 0) {
-		MonoThread *thread;
 		workers++;
-		thread = mono_thread_create (domain, async_invoke_thread,
-					     NULL);
-		g_assert (thread != NULL);
+		mono_thread_create (domain, async_invoke_thread, NULL);
 	}
 	LeaveCriticalSection (&mono_delegate_section);
 
diff --git a/mono/metadata/threads.c b/mono/metadata/threads.c
index 79fc53c6826..bd0bf1c3e1a 100644
--- a/mono/metadata/threads.c
+++ b/mono/metadata/threads.c
@@ -18,6 +18,7 @@
 #include <mono/metadata/threads.h>
 #include <mono/metadata/threads-types.h>
 #include <mono/metadata/exception.h>
+#include <mono/metadata/environment.h>
 #include <mono/io-layer/io-layer.h>
 
 #include <mono/os/gc_wrapper.h>
@@ -30,6 +31,7 @@ struct StartInfo
 {
 	guint32 (*func)(void *);
 	MonoThread *obj;
+	gboolean fake_thread;
 	void *this;
 	MonoDomain *domain;
 };
@@ -57,9 +59,6 @@ static CRITICAL_SECTION monitor_mutex;
  */
 static MonoGHashTable *threads=NULL;
 
-/* The MonoObject associated with the main thread */
-static MonoThread *main_thread;
-
 /* The TLS key that holds the MonoObject assigned to each thread */
 static guint32 current_object_key;
 
@@ -101,6 +100,7 @@ static void handle_store(MonoThread *thread)
 	 * the window in which the thread exists but we haven't
 	 * recorded it.
 	 */
+	mono_g_hash_table_remove (threads, &thread->tid);
 	
 	/* We don't need to duplicate thread->handle, because it is
 	 * only closed when the thread object is finalized by the GC.
@@ -148,30 +148,53 @@ static guint32 start_wrapper(void *data)
 	guint32 (*start_func)(void *);
 	void *this;
 	guint32 tid;
+	MonoThread *thread;
 	
 #ifdef THREAD_DEBUG
 	g_message(G_GNUC_PRETTY_FUNCTION ": Start wrapper");
 #endif
 	
-	start_func = start_info->func;
+	/* We can be sure start_info->obj->tid and
+	 * start_info->obj->handle have been set, because the thread
+	 * was created suspended, and these values were set before the
+	 * thread resumed
+	 */
+
+	tid=start_info->obj->tid;
+	
 	mono_domain_set (start_info->domain);
-	this = start_info->this;
 
-	tid=GetCurrentThreadId ();
-	/* Set the thread ID here as well as in the parent thread,
-	 * because we don't know whether the thread object will
-	 * already have its ID set before we get to it.  This isn't a
-	 * race condition, because if we're not guaranteed to get the
-	 * same number in both the parent and child threads, then
-	 * something else is seriously broken.
+	/* This MUST be called before any managed code can be
+	 * executed, as it calls the callback function that (for the
+	 * jit) sets the lmf marker.
 	 */
-	start_info->obj->tid=tid;
+	mono_new_thread_init (tid, &tid, start_func);
+
+	if(start_info->fake_thread) {
+		thread = (MonoThread *)mono_object_new (start_info->domain, mono_defaults.thread_class);
+
+		thread->handle=start_info->obj->handle;
+		thread->tid=tid;
+	} else {
+		thread=start_info->obj;
+	}
 	
-	handle_store(start_info->obj);
+	start_func = start_info->func;
+	this = start_info->this;
 
-	mono_profiler_thread_start (tid);
+	TlsSetValue (current_object_key, thread);
 
-	mono_new_thread_init (start_info->obj, &tid, start_func);
+	handle_store(thread);
+
+	if(start_info->fake_thread) {
+		/* This has to happen _after_ handle_store(), because
+		 * the fake thread is still in the threads hash until
+		 * this call.
+		 */
+		g_free (start_info->obj);
+	}
+
+	mono_profiler_thread_start (tid);
 
 	g_free (start_info);
 
@@ -191,39 +214,39 @@ static guint32 start_wrapper(void *data)
 	return(0);
 }
 
-void mono_new_thread_init (MonoThread *thread_object, gpointer stack_start, gpointer func)
+void mono_new_thread_init (guint32 tid, gpointer stack_start, gpointer func)
 {
-	/* FIXME: GC problem here with recorded object
-	 * pointer!
-	 *
-	 * This is recorded so CurrentThread can return the
-	 * Thread object.
-	 */
-	TlsSetValue (current_object_key, thread_object);
-
 	if (mono_thread_start_cb) {
-		mono_thread_start_cb (thread_object, stack_start, func);
+		mono_thread_start_cb (tid, stack_start, func);
 	}
 }
 
-MonoThread *mono_thread_create (MonoDomain *domain, gpointer func,
-				gpointer arg)
+void mono_thread_create (MonoDomain *domain, gpointer func, gpointer arg)
 {
 	MonoThread *thread;
 	HANDLE thread_handle;
 	struct StartInfo *start_info;
 	guint32 tid;
 	
-	thread = (MonoThread *)mono_object_new (domain,
-						mono_defaults.thread_class);
+	/* This is just a temporary allocation.  The object will be
+	 * created properly with mono_object_new() inside
+	 * start_wrapper().  (This is so the main thread can be
+	 * created without needing to run any managed code.)
+	 */
+	thread=g_new0 (MonoThread, 1);
 
 	start_info=g_new0 (struct StartInfo, 1);
 	start_info->func = func;
 	start_info->obj = thread;
+	start_info->fake_thread = TRUE;
 	start_info->domain = domain;
 	start_info->this = arg;
 		
-	thread_handle = CreateThread(NULL, 0, start_wrapper, start_info, 0, &tid);
+	/* Create suspended, so we can do some housekeeping before the thread
+	 * starts
+	 */
+	thread_handle = CreateThread(NULL, 0, start_wrapper, start_info,
+				     CREATE_SUSPENDED, &tid);
 #ifdef THREAD_DEBUG
 	g_message(G_GNUC_PRETTY_FUNCTION ": Started thread ID %d (handle %p)",
 		  tid, thread_handle);
@@ -235,7 +258,7 @@ MonoThread *mono_thread_create (MonoDomain *domain, gpointer func,
 
 	handle_store(thread);
 
-	return thread;
+	ResumeThread (thread_handle);
 }
 
 MonoThread *
@@ -248,7 +271,7 @@ mono_thread_attach (MonoDomain *domain)
 	if ((thread = mono_thread_current ())) {
 		g_warning ("mono_thread_attach called for an already attached thread");
 		if (mono_thread_attach_cb) {
-			mono_thread_attach_cb (thread, &tid);
+			mono_thread_attach_cb (tid, &tid);
 		}
 		return thread;
 	}
@@ -275,7 +298,7 @@ mono_thread_attach (MonoDomain *domain)
 	mono_domain_set (domain);
 
 	if (mono_thread_attach_cb) {
-		mono_thread_attach_cb (thread, &tid);
+		mono_thread_attach_cb (tid, &tid);
 	}
 
 	return(thread);
@@ -315,6 +338,7 @@ HANDLE ves_icall_System_Threading_Thread_Thread_internal(MonoThread *this,
 		start_info->func = start_func;
 		start_info->this = delegate;
 		start_info->obj = this;
+		start_info->fake_thread = FALSE;
 		start_info->domain = mono_domain_get ();
 
 		thread=CreateThread(NULL, 0, start_wrapper, start_info,
@@ -695,18 +719,31 @@ void ves_icall_System_Threading_Monitor_Monitor_pulse(MonoObject *obj)
 	MONO_ARCH_SAVE_REGS;
 
 #ifdef THREAD_LOCK_DEBUG
-	g_message("(%d) Pulsing %p", GetCurrentThreadId (), obj);
+	g_message(G_GNUC_PRETTY_FUNCTION "(%d) Pulsing %p",
+		  GetCurrentThreadId (), obj);
 #endif
 
 	EnterCriticalSection(&monitor_mutex);
 	
 	mon=obj->synchronisation;
 	if(mon==NULL) {
+#ifdef THREAD_LOCK_DEBUG
+		g_message (G_GNUC_PRETTY_FUNCTION
+			   "(%d) object %p not locked", GetCurrentThreadId (),
+			   obj);
+#endif
+
 		LeaveCriticalSection(&monitor_mutex);
 		return;
 	}
 
 	if(mon->tid!=GetCurrentThreadId()) {
+#ifdef THREAD_LOCK_DEBUG
+		g_message (G_GNUC_PRETTY_FUNCTION
+			   "(%d) doesn't own lock (owned by %d)",
+			   GetCurrentThreadId (), mon->tid);
+#endif
+
 		LeaveCriticalSection(&monitor_mutex);
 		return;
 	}
@@ -774,7 +811,8 @@ gboolean ves_icall_System_Threading_Monitor_Monitor_wait(MonoObject *obj,
 	MONO_ARCH_SAVE_REGS;
 
 #ifdef THREAD_LOCK_DEBUG
-	g_message("(%d) Trying to wait for %p with timeout %dms",
+	g_message(G_GNUC_PRETTY_FUNCTION
+		  "(%d) Trying to wait for %p with timeout %dms",
 		  GetCurrentThreadId (), obj, ms);
 #endif
 
@@ -1193,29 +1231,9 @@ ves_icall_System_Threading_Thread_ResetAbort (void)
 	}
 }
 
-void mono_thread_init (MonoDomain *domain, MonoThreadStartCB start_cb, MonoThreadAttachCB attach_cb)
+void mono_thread_init (MonoThreadStartCB start_cb,
+		       MonoThreadAttachCB attach_cb)
 {
-	/* Build a System.Threading.Thread object instance to return
-	 * for the main line's Thread.CurrentThread property.
-	 */
-	
-	/* I wonder what happens if someone tries to destroy this
-	 * object? In theory, I guess the whole program should act as
-	 * though exit() were called :-)
-	 */
-#ifdef THREAD_DEBUG
-	g_message(G_GNUC_PRETTY_FUNCTION
-		  ": Starting to build main Thread object");
-#endif
-	main_thread = (MonoThread *)mono_object_new (domain, mono_defaults.thread_class);
-
-	main_thread->handle = GetCurrentThread ();
-
-#ifdef THREAD_DEBUG
-	g_message(G_GNUC_PRETTY_FUNCTION
-		  ": Finished building main Thread object: %p", main_thread);
-#endif
-
 	InitializeCriticalSection(&threads_mutex);
 	InitializeCriticalSection(&monitor_mutex);
 	InitializeCriticalSection(&interlocked_mutex);
@@ -1226,8 +1244,6 @@ void mono_thread_init (MonoDomain *domain, MonoThreadStartCB start_cb, MonoThrea
 		   current_object_key);
 #endif
 
-	TlsSetValue(current_object_key, main_thread);
-
 	mono_thread_start_cb = start_cb;
 	mono_thread_attach_cb = attach_cb;
 
@@ -1327,7 +1343,7 @@ static void build_wait_tids (gpointer key, gpointer value, gpointer user)
 	}
 }
 
-void mono_thread_cleanup(void)
+void mono_thread_manage (void)
 {
 	struct wait_data *wait=g_new0 (struct wait_data, 1);
 	
@@ -1367,3 +1383,30 @@ void mono_thread_cleanup(void)
 	mono_g_hash_table_destroy(threads);
 	threads=NULL;
 }
+
+static void terminate_thread (gpointer key, gpointer value, gpointer user)
+{
+	MonoThread *thread=(MonoThread *)value;
+	guint32 self=GPOINTER_TO_UINT (user);
+	
+	if(thread->tid!=self) {
+		/*TerminateThread (thread->handle, -1);*/
+	}
+}
+
+void mono_thread_abort_all_other_threads (void)
+{
+	guint32 self=GetCurrentThreadId ();
+
+	EnterCriticalSection (&threads_mutex);
+#ifdef THREAD_DEBUG
+	g_message(G_GNUC_PRETTY_FUNCTION ":There are %d threads to abort",
+		  mono_g_hash_table_size (threads));
+	mono_g_hash_table_foreach (threads, print_tids, NULL);
+#endif
+
+	mono_g_hash_table_foreach (threads, terminate_thread,
+				   GUINT_TO_POINTER (self));
+	
+	LeaveCriticalSection (&threads_mutex);
+}
diff --git a/mono/metadata/threads.h b/mono/metadata/threads.h
index 26d5b07302c..b4445fda146 100644
--- a/mono/metadata/threads.h
+++ b/mono/metadata/threads.h
@@ -18,9 +18,11 @@
 
 extern int  mono_thread_get_abort_signal (void);
 
-extern void mono_thread_init (MonoDomain *domain, MonoThreadStartCB start_cb,
+extern void mono_thread_init (MonoThreadStartCB start_cb,
 			      MonoThreadAttachCB attach_cb);
-extern void mono_thread_cleanup(void);
+extern void mono_thread_manage(void);
+extern void mono_thread_abort_all_other_threads (void);
+
 extern MonoThread *mono_thread_current (void);
 
 typedef struct {
@@ -31,10 +33,9 @@ typedef struct {
 
 extern void mono_install_thread_callbacks (MonoThreadCallbacks *callbacks);
 
-extern void mono_new_thread_init (MonoThread *thread_object,
-				  gpointer stack_start,
+extern void mono_new_thread_init (guint32 tid, gpointer stack_start,
 				  gpointer func);
-extern MonoThread *mono_thread_create (MonoDomain *domain, gpointer func,
+extern void mono_thread_create (MonoDomain *domain, gpointer func,
 				       gpointer arg);
 extern MonoThread *mono_thread_attach (MonoDomain *domain);
 
diff --git a/samples/embed/teste.c b/samples/embed/teste.c
index ee7b84227d7..eb7131508c6 100644
--- a/samples/embed/teste.c
+++ b/samples/embed/teste.c
@@ -1,4 +1,5 @@
 #include <mono/jit/jit.h>
+#include <mono/metadata/environment.h>
 
 /*
  * Very simple mono embedding example.
@@ -14,13 +15,40 @@ gimme () {
 	return mono_string_new (mono_domain_get (), "All your monos are belong to us!");
 }
 
+typedef struct
+{
+	MonoDomain *domain;
+	const char *file;
+	int argc;
+	char **argv;
+} MainThreadArgs;
+
+static void main_thread_handler (gpointer user_data)
+{
+	MainThreadArgs *main_args=(MainThreadArgs *)user_data;
+	MonoAssembly *assembly;
+
+	assembly = mono_domain_assembly_open (main_args->domain,
+					      main_args->file);
+	if (!assembly)
+		exit (2);
+	/*
+	 * mono_jit_exec() will run the Main() method in the assembly.
+	 * The return value needs to be looked up from
+	 * System.Environment.ExitCode.
+	 */
+	mono_jit_exec (main_args->domain, assembly, main_args->argc,
+		       main_args->argv);
+}
+
+
 int 
 main(int argc, char* argv[]) {
 	MonoDomain *domain;
-	MonoAssembly *assembly;
 	const char *file;
 	int retval;
-
+	MainThreadArgs main_args;
+	
 	if (argc < 2){
 		fprintf (stderr, "Please provide an assembly to load");
 		return 1;
@@ -36,14 +64,17 @@ main(int argc, char* argv[]) {
 	 * can call us back.
 	 */
 	mono_add_internal_call ("Mono::gimme", gimme);
-	assembly = mono_domain_assembly_open (domain, file);
-	if (!assembly)
-		return 2;
-	/*
-	 * mono_jit_exec() will run the Main() method in the assembly
-	 * and return the value.
-	 */
-	retval = mono_jit_exec (domain, assembly, argc - 1, argv + 1);
+
+	main_args.domain=domain;
+	main_args.file=file;
+	main_args.argc=argc-1;
+	main_args.argv=argv+1;
+	
+	mono_runtime_exec_managed_code (domain, main_thread_handler,
+					&main_args);
+	
+	retval=mono_environment_exitcode_get ();
+	
 	mono_jit_cleanup (domain);
 	return retval;
 }