* errno.cc (errmap): Add error codes for invalid binaries.

	* exec.cc (execvp): Call spawnve with _P_PATH_TYPE_EXEC flag
	from here.
	(execvpe): Ditto.
	* spawn.cc (spawn_guts): Filter _P_PATH_TYPE_EXEC from mode and
	store in p_type_exec.  Call av::fixup with addtional p_type_exec
	argument.
	(spawnve): Check for filtered mode.
	(spawnvpe): Add _P_PATH_TYPE_EXEC flag when calling spawnve.
	(av::fixup): Accept additional bool parameter p_type_exec.  Only check
	for script if p_type_exec is true.
	* winf.h (_P_PATH_TYPE_EXEC): Define.
	(_P_MODE): Define.
	(av::fixup): Declare with additional bool parameter.

5 files changed, 72 insertions, 11 deletions

diff --git a/winsup/cygwin/ChangeLog b/winsup/cygwin/ChangeLog
index fb8aba469..aa6af3076 100644
--- a/winsup/cygwin/ChangeLog
+++ b/winsup/cygwin/ChangeLog
@@ -1,3 +1,20 @@
+2011-01-19  Corinna Vinschen  <corinna@vinschen.de>
+
+	* errno.cc (errmap): Add error codes for invalid binaries.
+	* exec.cc (execvp): Call spawnve with _P_PATH_TYPE_EXEC flag
+	from here.
+	(execvpe): Ditto.
+	* spawn.cc (spawn_guts): Filter _P_PATH_TYPE_EXEC from mode and
+	store in p_type_exec.  Call av::fixup with addtional p_type_exec
+	argument.
+	(spawnve): Check for filtered mode.
+	(spawnvpe): Add _P_PATH_TYPE_EXEC flag when calling spawnve.
+	(av::fixup): Accept additional bool parameter p_type_exec.  Only check
+	for script if p_type_exec is true.
+	* winf.h (_P_PATH_TYPE_EXEC): Define.
+	(_P_MODE): Define.
+	(av::fixup): Declare with additional bool parameter.
+
 2011-01-17  Corinna Vinschen  <corinna@vinschen.de>
 
 	* fhandler_proc.cc (format_proc_partitions): Fix compiler warning.
diff --git a/winsup/cygwin/errno.cc b/winsup/cygwin/errno.cc
index 58cdf4ae8..a9860f4a6 100644
--- a/winsup/cygwin/errno.cc
+++ b/winsup/cygwin/errno.cc
@@ -1,7 +1,7 @@
 /* errno.cc: errno-related functions
 
    Copyright 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
-   2006, 2008, 2009, 2010 Red Hat, Inc.
+   2006, 2008, 2009, 2010, 2011 Red Hat, Inc.
 
 This file is part of Cygwin.
 
@@ -37,6 +37,7 @@ static NO_COPY struct
   X (ACTIVE_CONNECTIONS,	EAGAIN),
   X (ALREADY_EXISTS,		EEXIST),
   X (BAD_DEVICE,		ENODEV),
+  X (BAD_EXE_FORMAT,		ENOEXEC),
   X (BAD_NETPATH,		ENOENT),
   X (BAD_NET_NAME,		ENOENT),
   X (BAD_NET_RESP,		ENOSYS),
@@ -67,6 +68,7 @@ static NO_COPY struct
   X (EA_TABLE_FULL,		ENOSPC),
   X (END_OF_MEDIA,		ENOSPC),
   X (EOM_OVERFLOW,		EIO),
+  X (EXE_MARKED_INVALID,	ENOEXEC),
   X (FILEMARK_DETECTED,		EIO),
   X (FILENAME_EXCED_RANGE,	ENAMETOOLONG),
   X (FILE_CORRUPT,		EEXIST),
@@ -81,11 +83,13 @@ static NO_COPY struct
   X (INVALID_DATA,		EINVAL),
   X (INVALID_DRIVE,		ENODEV),
   X (INVALID_EA_NAME,		EINVAL),
+  X (INVALID_EXE_SIGNATURE,	ENOEXEC),
   X (INVALID_FUNCTION,		EBADRQC),
   X (INVALID_HANDLE,		EBADF),
   X (INVALID_NAME,		ENOENT),
   X (INVALID_PARAMETER,		EINVAL),
   X (INVALID_SIGNAL_NUMBER,	EINVAL),
+  X (IOPL_NOT_ENABLED,		ENOEXEC),
   X (IO_DEVICE,			EIO),
   X (IO_PENDING,		EAGAIN),
   X (LOCK_VIOLATION,		EACCES),
diff --git a/winsup/cygwin/exec.cc b/winsup/cygwin/exec.cc
index b1fd52783..d4462b534 100644
--- a/winsup/cygwin/exec.cc
+++ b/winsup/cygwin/exec.cc
@@ -1,6 +1,6 @@
 /* exec.cc: exec system call support.
 
-   Copyright 1996, 1997, 1998, 2000, 2001, 2002, 2009 Red Hat, Inc.
+   Copyright 1996, 1997, 1998, 2000, 2001, 2002, 2009, 2011 Red Hat, Inc.
 
 This file is part of Cygwin.
 
@@ -18,6 +18,7 @@ details. */
 #include "fhandler.h"
 #include "dtable.h"
 #include "cygheap.h"
+#include "winf.h"
 #undef _execve
 
 /* This is called _execve and not execve because the real execve is defined
@@ -86,14 +87,22 @@ extern "C" int
 execvp (const char *path, char * const *argv)
 {
   path_conv buf;
-  return execv (find_exec (path, buf, "PATH=", FE_NNF) ?: "", argv);
+  return spawnve (_P_OVERLAY | _P_PATH_TYPE_EXEC,
+		  find_exec (path, buf, "PATH=", FE_NNF) ?: "",
+		  argv, cur_environ ());
 }
 
 extern "C" int
 execvpe (const char *path, char * const *argv, char *const *envp)
 {
+  static char *const empty_env[] = { 0 };
+  MALLOC_CHECK;
+  if (!envp)
+    envp = empty_env;
   path_conv buf;
-  return execve (find_exec (path, buf, "PATH=", FE_NNF) ?: "", argv, envp);
+  return spawnve (_P_OVERLAY | _P_PATH_TYPE_EXEC,
+		  find_exec (path, buf, "PATH=", FE_NNF) ?: "",
+		  argv, envp);
 }
 
 extern "C" int
diff --git a/winsup/cygwin/spawn.cc b/winsup/cygwin/spawn.cc
index 2005592f7..90ba65efa 100644
--- a/winsup/cygwin/spawn.cc
+++ b/winsup/cygwin/spawn.cc
@@ -1,7 +1,7 @@
 /* spawn.cc
 
    Copyright 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-   2005, 2006, 2007, 2008, 2009, 2010 Red Hat, Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011 Red Hat, Inc.
 
 This file is part of Cygwin.
 
@@ -280,6 +280,11 @@ spawn_guts (const char *prog_arg, const char *const *argv,
   pid_t cygpid;
   int res = -1;
 
+  /* Check if we have been called from exec{lv}p or spawn{lv}p and mask
+     mode to keep only the spawn mode. */
+  bool p_type_exec = !!(mode & _P_PATH_TYPE_EXEC);
+  mode = _P_MODE (mode);
+
   if (prog_arg == NULL)
     {
       syscall_printf ("prog_arg is NULL");
@@ -375,7 +380,7 @@ spawn_guts (const char *prog_arg, const char *const *argv,
 
 
   wascygexec = real_path.iscygexec ();
-  res = newargv.fixup (prog_arg, real_path, ext);
+  res = newargv.fixup (prog_arg, real_path, ext, p_type_exec);
 
   if (res)
     goto out;
@@ -861,7 +866,7 @@ spawnve (int mode, const char *path, const char *const *argv,
 
   syscall_printf ("spawnve (%s, %s, %x)", path, argv[0], envp);
 
-  switch (mode)
+  switch (_P_MODE (mode))
     {
     case _P_OVERLAY:
       /* We do not pass _P_SEARCH_PATH here. execve doesn't search PATH.*/
@@ -1002,11 +1007,12 @@ spawnvpe (int mode, const char *file, const char * const *argv,
 					   const char * const *envp)
 {
   path_conv buf;
-  return spawnve (mode, find_exec (file, buf), argv, envp);
+  return spawnve (mode | _P_PATH_TYPE_EXEC, find_exec (file, buf), argv, envp);
 }
 
 int
-av::fixup (const char *prog_arg, path_conv& real_path, const char *ext)
+av::fixup (const char *prog_arg, path_conv& real_path, const char *ext,
+	   bool p_type_exec)
 {
   const char *p;
   bool exeext = ascii_strcasematch (ext, ".exe");
@@ -1053,6 +1059,13 @@ av::fixup (const char *prog_arg, path_conv& real_path, const char *ext)
 	  /* ERROR_FILE_INVALID indicates very likely an empty file. */
 	  if (GetLastError () == ERROR_FILE_INVALID)
 	    {
+	      if (!p_type_exec)
+		{
+		  /* Not called from exec[lv]p.  Just leave. */
+		  debug_printf ("zero length file.");
+		  set_errno (ENOEXEC);
+		  return -1;
+		}
 	      debug_printf ("zero length file, treat as script.");
 	      goto just_shell;
 	    }
@@ -1085,6 +1098,14 @@ av::fixup (const char *prog_arg, path_conv& real_path, const char *ext)
 	  }
       }
 
+      if (!p_type_exec)
+	{
+	  /* Not called from exec[lv]p.  Don't try to treat as script. */
+	  debug_printf ("%s is not a valid executable", real_path.get_win32 ());
+	  set_errno (ENOEXEC);
+	  return -1;
+	}
+
       debug_printf ("%s is possibly a script", real_path.get_win32 ());
 
       ptr = buf;
diff --git a/winsup/cygwin/winf.h b/winsup/cygwin/winf.h
index a58fb49a9..dccaf5b32 100644
--- a/winsup/cygwin/winf.h
+++ b/winsup/cygwin/winf.h
@@ -1,6 +1,6 @@
 /* winf.h
 
-   Copyright 2006, 2007 Red Hat, Inc.
+   Copyright 2006, 2007, 2011 Red Hat, Inc.
 
 This software is a copyrighted work licensed under the terms of the
 Cygwin license.  Please consult the file "CYGWIN_LICENSE" for
@@ -20,6 +20,16 @@ details. */
 #define MAXWINCMDLEN 32767
 #define LINE_BUF_CHUNK (MAX_PATH * 2)
 
+/* Add this flag in calls to spawn_guts if the calling function is one of
+   'p' type functions: execlp, execvp, spawnlp, spawnvp.  Per POSIX, only
+   these p-type functions fall back to call /bin/sh if the file is not a
+   binary.  The setting of _P_PATH_TYPE_EXEC is used as a bool value in
+   av::fixup to decide if the file should be evaluated as a script, or if
+   ENOEXEC should be returned. */
+#define _P_PATH_TYPE_EXEC	0x100
+/* Helper macro to mask actual mode and drop additional flags defined above. */
+#define _P_MODE(x)		((x) & 0xff)
+
 class av
 {
   char **argv;
@@ -67,7 +77,7 @@ class av
     for (int i = calloced; i < argc; i++)
       argv[i] = cstrdup1 (argv[i]);
   }
-  int fixup (const char *, path_conv&, const char *);
+  int fixup (const char *, path_conv&, const char *, bool);
 };
 
 class linebuf