Cygwin: console: Fix console mode for non-cygwin inferior of GDB.

- Currently, there is no chance to change console mode for non-cygwin
  inferior of GDB. With this patch, the console mode is changed to
  tty::native in CreateProcess() and ContinueDebugEvent() hooked in
  fhandler_console.

3 files changed, 107 insertions, 62 deletions

diff --git a/winsup/cygwin/fhandler.h b/winsup/cygwin/fhandler.h
index 4226bfd69..c2d91ea11 100644
--- a/winsup/cygwin/fhandler.h
+++ b/winsup/cygwin/fhandler.h
@@ -1941,6 +1941,8 @@ class fhandler_termios: public fhandler_base
     fh->copy_from (this);
     return fh;
   }
+  static bool path_iscygexec_a (LPCSTR n, LPSTR c);
+  static bool path_iscygexec_w (LPCWSTR n, LPWSTR c);
 };
 
 enum ansi_intensity
diff --git a/winsup/cygwin/fhandler_console.cc b/winsup/cygwin/fhandler_console.cc
index 9b94f3c42..9373e2602 100644
--- a/winsup/cygwin/fhandler_console.cc
+++ b/winsup/cygwin/fhandler_console.cc
@@ -32,6 +32,7 @@ details. */
 #include "sync.h"
 #include "child_info.h"
 #include "cygwait.h"
+#include "winf.h"
 
 /* Don't make this bigger than NT_MAX_PATH as long as the temporary buffer
    is allocated using tmp_pathbuf!!! */
@@ -3595,10 +3596,32 @@ set_console_title (char *title)
   debug_printf ("title '%W'", buf);
 }
 
+static bool NO_COPY gdb_inferior_noncygwin = false;
+static void
+set_console_mode_to_native ()
+{
+  cygheap_fdenum cfd (false);
+  while (cfd.next () >= 0)
+    if (cfd->get_major () == DEV_CONS_MAJOR)
+      {
+	fhandler_console *cons = (fhandler_console *) (fhandler_base *) cfd;
+	if (cons->get_device () == cons->tc ()->getntty ())
+	  {
+	    termios *cons_ti = &((tty *) cons->tc ())->ti;
+	    fhandler_console::set_input_mode (tty::native, cons_ti,
+					      cons->get_handle_set ());
+	    fhandler_console::set_output_mode (tty::native, cons_ti,
+					       cons->get_handle_set ());
+	    break;
+	  }
+      }
+}
+
 #define DEF_HOOK(name) static __typeof__ (name) *name##_Orig
 /* CreateProcess() is hooked for GDB etc. */
 DEF_HOOK (CreateProcessA);
 DEF_HOOK (CreateProcessW);
+DEF_HOOK (ContinueDebugEvent);
 
 static BOOL WINAPI
 CreateProcessA_Hooked
@@ -3608,6 +3631,9 @@ CreateProcessA_Hooked
 {
   if (f & (DEBUG_PROCESS | DEBUG_ONLY_THIS_PROCESS))
     mutex_timeout = 0; /* to avoid deadlock in GDB */
+  gdb_inferior_noncygwin = !fhandler_termios::path_iscygexec_a (n, c);
+  if (gdb_inferior_noncygwin)
+    set_console_mode_to_native ();
   return CreateProcessA_Orig (n, c, pa, ta, inh, f, e, d, si, pi);
 }
 
@@ -3619,9 +3645,21 @@ CreateProcessW_Hooked
 {
   if (f & (DEBUG_PROCESS | DEBUG_ONLY_THIS_PROCESS))
     mutex_timeout = 0; /* to avoid deadlock in GDB */
+  gdb_inferior_noncygwin = !fhandler_termios::path_iscygexec_w (n, c);
+  if (gdb_inferior_noncygwin)
+    set_console_mode_to_native ();
   return CreateProcessW_Orig (n, c, pa, ta, inh, f, e, d, si, pi);
 }
 
+static BOOL WINAPI
+ContinueDebugEvent_Hooked
+     (DWORD p, DWORD t, DWORD s)
+{
+  if (gdb_inferior_noncygwin)
+    set_console_mode_to_native ();
+  return ContinueDebugEvent_Orig (p, t, s);
+}
+
 void
 fhandler_console::fixup_after_fork_exec (bool execing)
 {
@@ -3641,6 +3679,7 @@ fhandler_console::fixup_after_fork_exec (bool execing)
   /* CreateProcess() is hooked for GDB etc. */
   DO_HOOK (NULL, CreateProcessA);
   DO_HOOK (NULL, CreateProcessW);
+  DO_HOOK (NULL, ContinueDebugEvent);
 }
 
 // #define WINSTA_ACCESS (WINSTA_READATTRIBUTES | STANDARD_RIGHTS_READ | STANDARD_RIGHTS_WRITE | WINSTA_CREATEDESKTOP | WINSTA_EXITWINDOWS)
diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index 8c9a10c23..5ba50cc73 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -157,6 +157,66 @@ set_switch_to_pcon (HANDLE *in, HANDLE *out, HANDLE *err, bool iscygwin)
     *err = replace_err->get_output_handle_nat ();
 }
 
+static bool
+path_iscygexec_a_w (LPCSTR na, LPSTR ca, LPCWSTR nw, LPWSTR cw)
+{
+  path_conv path;
+  tmp_pathbuf tp;
+  char *prog =tp.c_get ();
+  if (na)
+    {
+      __small_sprintf (prog, "%s", na);
+      find_exec (prog, path);
+    }
+  else if (nw)
+    {
+      __small_sprintf (prog, "%W", nw);
+      find_exec (prog, path);
+    }
+  else
+    {
+      if (ca)
+	__small_sprintf (prog, "%s", ca);
+      else if (cw)
+	__small_sprintf (prog, "%W", cw);
+      else
+	return true;
+      char *p = prog;
+      char *p1;
+      do
+	if ((p1 = strchr (p, ' ')) || (p1 = p + strlen (p)))
+	  {
+	    p = p1;
+	    if (*p == ' ')
+	      {
+		*p = '\0';
+		find_exec (prog, path);
+		*p = ' ';
+		p ++;
+	      }
+	    else if (*p == '\0')
+	      find_exec (prog, path);
+	  }
+      while (!path.exists() && *p);
+    }
+  const char *argv[] = {"", NULL}; /* Dummy */
+  av av1;
+  av1.setup ("", path, "", 1, argv, false);
+  return path.iscygexec ();
+}
+
+bool
+fhandler_termios::path_iscygexec_a (LPCSTR n, LPSTR c)
+{
+  return path_iscygexec_a_w (n, c, NULL, NULL);
+}
+
+bool
+fhandler_termios::path_iscygexec_w (LPCWSTR n, LPWSTR c)
+{
+  return path_iscygexec_a_w (NULL, NULL, n, c);
+}
+
 static bool atexit_func_registered = false;
 static bool debug_process = false;
 
@@ -220,37 +280,9 @@ CreateProcessA_Hooked
       siov->hStdOutput = GetStdHandle (STD_OUTPUT_HANDLE);
       siov->hStdError = GetStdHandle (STD_ERROR_HANDLE);
     }
-  path_conv path;
-  tmp_pathbuf tp;
-  char *prog =tp.c_get ();
-  if (n)
-    __small_sprintf (prog, "%s", n);
-  else
-    {
-      __small_sprintf (prog, "%s", c);
-      char *p = prog;
-      char *p1;
-      do
-	if ((p1 = strchr (p, ' ')) || (p1 = p + strlen (p)))
-	  {
-	    p = p1;
-	    if (*p == ' ')
-	      {
-		*p = '\0';
-		find_exec (prog, path);
-		*p = ' ';
-		p ++;
-	      }
-	    else if (*p == '\0')
-	      find_exec (prog, path);
-	  }
-      while (!path.exists() && *p);
-    }
-  const char *argv[] = {"", NULL}; /* Dummy */
-  av av1;
-  av1.setup ("", path, "", 1, argv, false);
+  bool path_iscygexec = fhandler_termios::path_iscygexec_a (n, c);
   set_switch_to_pcon (&siov->hStdInput, &siov->hStdOutput, &siov->hStdError,
-		      path.iscygexec ());
+		      path_iscygexec);
   BOOL ret = CreateProcessA_Orig (n, c, pa, ta, inh, f, e, d, siov, pi);
   h_gdb_process = pi->hProcess;
   DuplicateHandle (GetCurrentProcess (), h_gdb_process,
@@ -259,7 +291,7 @@ CreateProcessA_Hooked
   debug_process = !!(f & (DEBUG_PROCESS | DEBUG_ONLY_THIS_PROCESS));
   if (debug_process)
     mutex_timeout = 0; /* to avoid deadlock in GDB */
-  if (!atexit_func_registered && !path.iscygexec ())
+  if (!atexit_func_registered && !path_iscygexec)
     {
       atexit (atexit_func);
       atexit_func_registered = true;
@@ -286,37 +318,9 @@ CreateProcessW_Hooked
       siov->hStdOutput = GetStdHandle (STD_OUTPUT_HANDLE);
       siov->hStdError = GetStdHandle (STD_ERROR_HANDLE);
     }
-  path_conv path;
-  tmp_pathbuf tp;
-  char *prog =tp.c_get ();
-  if (n)
-    __small_sprintf (prog, "%W", n);
-  else
-    {
-      __small_sprintf (prog, "%W", c);
-      char *p = prog;
-      char *p1;
-      do
-	if ((p1 = strchr (p, ' ')) || (p1 = p + strlen (p)))
-	  {
-	    p = p1;
-	    if (*p == ' ')
-	      {
-		*p = '\0';
-		find_exec (prog, path);
-		*p = ' ';
-		p ++;
-	      }
-	    else if (*p == '\0')
-	      find_exec (prog, path);
-	  }
-      while (!path.exists() && *p);
-    }
-  const char *argv[] = {"", NULL}; /* Dummy */
-  av av1;
-  av1.setup ("", path, "", 1, argv, false);
+  bool path_iscygexec = fhandler_termios::path_iscygexec_w (n, c);
   set_switch_to_pcon (&siov->hStdInput, &siov->hStdOutput, &siov->hStdError,
-		      path.iscygexec ());
+		      path_iscygexec);
   BOOL ret = CreateProcessW_Orig (n, c, pa, ta, inh, f, e, d, siov, pi);
   h_gdb_process = pi->hProcess;
   DuplicateHandle (GetCurrentProcess (), h_gdb_process,
@@ -325,7 +329,7 @@ CreateProcessW_Hooked
   debug_process = !!(f & (DEBUG_PROCESS | DEBUG_ONLY_THIS_PROCESS));
   if (debug_process)
     mutex_timeout = 0; /* to avoid deadlock in GDB */
-  if (!atexit_func_registered && !path.iscygexec ())
+  if (!atexit_func_registered && !path_iscygexec)
     {
       atexit (atexit_func);
       atexit_func_registered = true;