* dtable.cc (dtable::init_std_file_from_handle): Attempt stronger detection of

invalid handle.
(handle_to_fn): Detect pathological condition where NT resets the buffer
pointer to NULL on an invalid handle.

2 files changed, 61 insertions, 43 deletions

diff --git a/winsup/cygwin/ChangeLog b/winsup/cygwin/ChangeLog
index 0affaf2c3..a846a015b 100644
--- a/winsup/cygwin/ChangeLog
+++ b/winsup/cygwin/ChangeLog
@@ -1,3 +1,10 @@
+2002-05-29  Christopher Faylor  <cgf@redhat.com>
+
+	* dtable.cc (dtable::init_std_file_from_handle): Attempt stronger
+	detection of invalid handle.
+	(handle_to_fn): Detect pathological condition where NT resets the
+	buffer pointer to NULL on an invalid handle.
+
 2002-05-28  Christopher Faylor  <cgf@redhat.com>
 
 	* fhandler_disk_file.cc (fhandler_disk_file::fstat_helper): Properly
diff --git a/winsup/cygwin/dtable.cc b/winsup/cygwin/dtable.cc
index b848eabc6..028a5f879 100644
--- a/winsup/cygwin/dtable.cc
+++ b/winsup/cygwin/dtable.cc
@@ -217,7 +217,7 @@ cygwin_attach_handle_to_fd (char *name, int fd, HANDLE handle, mode_t bin,
 void
 dtable::init_std_file_from_handle (int fd, HANDLE handle, DWORD myaccess)
 {
-  int bin;
+  int bin = 0;
   const char *name;
   CONSOLE_SCREEN_BUFFER_INFO buf;
   struct sockaddr sa;
@@ -229,55 +229,61 @@ dtable::init_std_file_from_handle (int fd, HANDLE handle, DWORD myaccess)
   if (!not_open (fd))
     return;
 
-  if (!handle || handle == INVALID_HANDLE_VALUE)
-    {
-      fds[fd] = NULL;
-      return;
-    }
-
-  if (__fmode)
-    bin = __fmode;
+  SetLastError (0);
+  DWORD ft = GetFileType (handle);
+  if (ft == FILE_TYPE_UNKNOWN && GetLastError () == ERROR_INVALID_HANDLE)
+    name = NULL;
   else
-    bin = binmode ?: 0;
-
-  /* See if we can consoleify it  - if it is a console,
-   don't open it in binary.  That will screw up our crlfs*/
-  if (GetConsoleScreenBufferInfo (handle, &buf))
     {
-      if (ISSTATE (myself, PID_USETTY))
-	name = "/dev/tty";
+      if (__fmode)
+	bin = __fmode;
       else
-	name = "/dev/conout";
-      bin = 0;
-    }
-  else if (GetNumberOfConsoleInputEvents (handle, (DWORD *) &buf))
-    {
-      if (ISSTATE (myself, PID_USETTY))
-	name = "/dev/tty";
+	bin = binmode ?: 0;
+
+      /* See if we can consoleify it  - if it is a console,
+       don't open it in binary.  That will screw up our crlfs*/
+      if (GetConsoleScreenBufferInfo (handle, &buf))
+	{
+	  if (ISSTATE (myself, PID_USETTY))
+	    name = "/dev/tty";
+	  else
+	    name = "/dev/conout";
+	  bin = 0;
+	}
+      else if (GetNumberOfConsoleInputEvents (handle, (DWORD *) &buf))
+	{
+	  if (ISSTATE (myself, PID_USETTY))
+	    name = "/dev/tty";
+	  else
+	    name = "/dev/conin";
+	  bin = 0;
+	}
+      else if (ft == FILE_TYPE_PIPE)
+	{
+	  if (fd == 0)
+	    name = "/dev/piper";
+	  else
+	    name = "/dev/pipew";
+	  if (bin == 0)
+	    bin = O_BINARY;
+	}
+      else if (wsock_started && getpeername ((SOCKET) handle, &sa, &sal) == 0)
+	name = "/dev/socket";
+      else if (GetCommState (handle, &dcb))
+	name = "/dev/ttyS0"; // FIXME - determine correct device
       else
-	name = "/dev/conin";
-      bin = 0;
+	name = handle_to_fn (handle, (char *) alloca (MAX_PATH + 100));
     }
-  else if (GetFileType (handle) == FILE_TYPE_PIPE)
+
+  if (!name)
+    fds[fd] = NULL;
+  else
     {
-      if (fd == 0)
-	name = "/dev/piper";
-      else
-	name = "/dev/pipew";
-      if (bin == 0)
-	bin = O_BINARY;
+      path_conv pc;
+      build_fhandler_from_name (fd, name, handle, pc)->init (handle, myaccess, bin);
+      set_std_handle (fd);
+      paranoid_printf ("fd %d, handle %p", fd, handle);
     }
-  else if (wsock_started && getpeername ((SOCKET) handle, &sa, &sal) == 0)
-    name = "/dev/socket";
-  else if (GetCommState (handle, &dcb))
-    name = "/dev/ttyS0"; // FIXME - determine correct device
-  else
-    name = handle_to_fn (handle, (char *) alloca (MAX_PATH + 100));
-
-  path_conv pc;
-  build_fhandler_from_name (fd, name, handle, pc)->init (handle, myaccess, bin);
-  set_std_handle (fd);
-  paranoid_printf ("fd %d, handle %p", fd, handle);
 }
 
 fhandler_base *
@@ -736,6 +742,11 @@ handle_to_fn (HANDLE h, char *posix_fn)
   ntfn->Name.Buffer = (WCHAR *) ntfn + 1;
 
   DWORD res = NtQueryObject (h, ObjectNameInformation, ntfn, sizeof (fnbuf), NULL);
+
+  // NT seems to do this on an unopened file
+  if (!ntfn->Name.Buffer)
+    return NULL;
+
   if (res)
     {
       strcpy (posix_fn, "some disk file");