* dtable.cc (dtable::fixup_close): Define new function.

(dtable::fixup_after_exec): Use fixup_close() and detect when it was not
possible to open an inherited file handle.
(dtable::fixup_after_fork): Defensively close any file handles which were not,
for some reason, inheritable.
* dtable.h: Make #pragma once.
(dtable::fixup_close): Declare new function.
* fhandler_console.cc (fhandler_console::set_unit): Set I/O handles to NULL
when this function fails.

4 files changed, 47 insertions, 14 deletions

diff --git a/winsup/cygwin/ChangeLog b/winsup/cygwin/ChangeLog
index e2a63c1ed..e7fa46e2d 100644
--- a/winsup/cygwin/ChangeLog
+++ b/winsup/cygwin/ChangeLog
@@ -1,3 +1,15 @@
+2012-04-01  Christopher Faylor  <me.cygwin2012@cgf.cx>
+
+	* dtable.cc (dtable::fixup_close): Define new function.
+	(dtable::fixup_after_exec): Use fixup_close() and detect when it was
+	not possible to open an inherited file handle.
+	(dtable::fixup_after_fork): Defensively close any file handles which
+	were not, for some reason, inheritable.
+	* dtable.h: Make #pragma once.
+	(dtable::fixup_close): Declare new function.
+	* fhandler_console.cc (fhandler_console::set_unit): Set I/O handles to
+	NULL when this function fails.
+
 2012-04-01  Corinna Vinschen  <corinna@vinschen.de>
 
 	* fhandler_dev.cc (fhandler_dev::readdir): Fix formatting.  Simplify
diff --git a/winsup/cygwin/dtable.cc b/winsup/cygwin/dtable.cc
index 1d59eccfb..2cc8229af 100644
--- a/winsup/cygwin/dtable.cc
+++ b/winsup/cygwin/dtable.cc
@@ -835,6 +835,17 @@ dtable::set_file_pointers_for_exec ()
 }
 
 void
+dtable::fixup_close (size_t i, fhandler_base *fh)
+{
+  if (fh->archetype)
+    {
+      debug_printf ("closing fd %d since it is an archetype", i);
+      fh->close_with_arch ();
+    }
+  release (i);
+}
+
+void
 dtable::fixup_after_exec ()
 {
   first_fd_for_open = 0;
@@ -844,15 +855,11 @@ dtable::fixup_after_exec ()
       {
 	fh->clear_readahead ();
 	fh->fixup_after_exec ();
-	if (fh->close_on_exec ())
-	  {
-	    if (fh->archetype)
-	      {
-		debug_printf ("closing fd %d since it is an archetype", i);
-		fh->close_with_arch ();
-	      }
-	    release (i);
-	  }
+	/* Close the handle if it's close-on-exec or if an error was detected
+	   (typically with opening a console in a gui app) by fixup_after_exec.
+	 */
+	if (fh->close_on_exec () || !fh->get_io_handle ())
+	  fixup_close (i, fh);
 	else if (fh->get_popen_pid ())
 	  close (i);
 	else if (i == 0)
@@ -873,6 +880,13 @@ dtable::fixup_after_fork (HANDLE parent)
 	  {
 	    debug_printf ("fd %d (%s)", i, fh->get_name ());
 	    fh->fixup_after_fork (parent);
+	    if (!fh->get_io_handle ())
+	      {
+		/* This should actually never happen but it's here to make sure
+		   we don't crash due to access of an unopened file handle.  */
+		fixup_close (i, fh);
+		continue;
+	      }
 	  }
 	if (i == 0)
 	  SetStdHandle (std_consts[i], fh->get_io_handle ());
diff --git a/winsup/cygwin/dtable.h b/winsup/cygwin/dtable.h
index d0065f421..288b4767f 100644
--- a/winsup/cygwin/dtable.h
+++ b/winsup/cygwin/dtable.h
@@ -9,6 +9,8 @@ This software is a copyrighted work licensed under the terms of the
 Cygwin license.  Please consult the file "CYGWIN_LICENSE" for
 details. */
 
+#pragma once
+
 /* Initial and increment values for cygwin's fd table */
 #define NOFILE_INCR    32
 /* Maximum size we allow expanding to.  */
@@ -54,6 +56,8 @@ public:
   fhandler_base *dup_worker (fhandler_base *oldfh, int flags);
   int extend (int howmuch);
   void fixup_after_fork (HANDLE);
+  void fixup_close (size_t, fhandler_base *);
+
   inline int not_open (int fd)
   {
     lock ();
diff --git a/winsup/cygwin/fhandler_console.cc b/winsup/cygwin/fhandler_console.cc
index 5be7111ff..7478eae19 100644
--- a/winsup/cygwin/fhandler_console.cc
+++ b/winsup/cygwin/fhandler_console.cc
@@ -141,18 +141,15 @@ fhandler_console::set_unit ()
       fh_devices this_unit = dev ();
       fh_devices shared_unit =
 	(fh_devices) shared_console_info->tty_min_state.getntty ();
-      created = false;
       devset = (shared_unit == this_unit || this_unit == FH_CONSOLE
 		|| this_unit == FH_CONIN || this_unit == FH_CONOUT
 		|| this_unit == FH_TTY) ?
 		shared_unit : FH_ERROR;
+      created = false;
     }
   else if ((myself->ctty != -1 && !iscons_dev (myself->ctty))
 	   || !(me = GetConsoleWindow ()))
-    {
-      created = false;
-      devset = FH_ERROR;
-    }
+    devset = FH_ERROR;
   else
     {
       created = true;
@@ -166,6 +163,12 @@ fhandler_console::set_unit ()
   dev ().parse (devset);
   if (devset != FH_ERROR)
     pc.file_attributes (FILE_ATTRIBUTE_NORMAL);
+  else
+    {
+      set_io_handle (NULL);
+      set_output_handle (NULL);
+      created = false;
+    }
   return created;
 }