[System] Fix process creation when UseShellExecute is set to false.mono-3.2.7

This fixes the behavior of process spawning when UseShellExecute is
set to true.

The right behavior is to return no process only when we used the system
launcher. So, spawning /bin/cat should behave in the same way independent
of how UseShellExecute is set.

2 files changed, 11 insertions, 8 deletions

diff --git a/mcs/class/System/System.Diagnostics/Process.cs b/mcs/class/System/System.Diagnostics/Process.cs
index 63a045fd628..7a63ce0377c 100644
--- a/mcs/class/System/System.Diagnostics/Process.cs
+++ b/mcs/class/System/System.Diagnostics/Process.cs
@@ -918,7 +918,7 @@ namespace System.Diagnostics {
 								  IntPtr stderr,
 								  ref ProcInfo proc_info);
 
-		private static bool Start_shell (ProcessStartInfo startInfo)
+		private static bool Start_shell (ProcessStartInfo startInfo, Process process)
 		{
 			ProcInfo proc_info=new ProcInfo();
 			bool ret;
@@ -945,6 +945,9 @@ namespace System.Diagnostics {
 				throw new Win32Exception (-proc_info.pid);
 			}
 
+			process.process_handle = proc_info.process_handle;
+			process.pid = proc_info.pid;
+			process.StartExitCallbackIfNeeded ();
 			return(ret);
 		}
 
@@ -1156,7 +1159,7 @@ namespace System.Diagnostics {
 			if (startInfo.UseShellExecute) {
 				if (!String.IsNullOrEmpty (startInfo.UserName))
 					throw new InvalidOperationException ("UserShellExecute must be false if an explicit UserName is specified when starting a process");
-				return (Start_shell (startInfo));
+				return (Start_shell (startInfo, process));
 			} else {
 				return (Start_noshell (startInfo, process));
 			}
@@ -1176,12 +1179,9 @@ namespace System.Diagnostics {
 			if (startInfo == null)
 				throw new ArgumentNullException ("startInfo");
 
-			Process process = null;
-			if (!startInfo.UseShellExecute) {
-				process = new Process();
-				process.StartInfo = startInfo;
-			}
-			if (Start_common(startInfo, process))
+			Process process = new Process();
+			process.StartInfo = startInfo;
+			if (Start_common(startInfo, process) && process.process_handle != IntPtr.Zero)
 				return process;
 			return null;
 		}
diff --git a/mono/io-layer/processes.c b/mono/io-layer/processes.c
index 6cf55aca068..3fbff353f28 100644
--- a/mono/io-layer/processes.c
+++ b/mono/io-layer/processes.c
@@ -380,6 +380,9 @@ gboolean ShellExecuteEx (WapiShellExecuteInfo *sei)
 				SetLastError (ERROR_INVALID_DATA);
 			return FALSE;
 		}
+		/* Shell exec should not return a process handle when it spawned a GUI thing, like a browser. */
+		CloseHandle (process_info.hProcess);
+		process_info.hProcess = NULL;
 	}
 	
 	if (sei->fMask & SEE_MASK_NOCLOSEPROCESS) {