bpo-31804: Fix multiprocessing.Process with broken standard streams (#6079)

In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows). Avoid failing with a non-0 exit code in those conditions. Report and initial patch by poxthegreat.

Author: pitrou

Lib/multiprocessing/popen_fork.py

@@ -14,14 +14,7 @@ class Popen(object):
  method = 'fork'
 
  def __init__(self, process_obj):
- try:
- sys.stdout.flush()
- except (AttributeError, ValueError):
- pass
- try:
- sys.stderr.flush()
- except (AttributeError, ValueError):
- pass
+ util._flush_std_streams()
  self.returncode = None
  self.finalizer = None
  self._launch(process_obj)

Lib/multiprocessing/process.py

@@ -314,8 +314,7 @@ def _bootstrap(self):
  finally:
  threading._shutdown()
  util.info('process exiting with exitcode %d' % exitcode)
- sys.stdout.flush()
- sys.stderr.flush()
+ util._flush_std_streams()
 
  return exitcode
 

Lib/multiprocessing/util.py

@@ -391,6 +391,20 @@ def _close_stdin():
  except (OSError, ValueError):
  pass
 
+#
+# Flush standard streams, if any
+#
+
+def _flush_std_streams():
+ try:
+ sys.stdout.flush()
+ except (AttributeError, ValueError):
+ pass
+ try:
+ sys.stderr.flush()
+ except (AttributeError, ValueError):
+ pass
+
 #
 # Start a program with only specified fds kept open
 #

Lib/test/_test_multiprocessing.py

@@ -584,10 +584,19 @@ def test_wait_for_threads(self):
  self.assertTrue(evt.is_set())
 
  @classmethod
- def _test_error_on_stdio_flush(self, evt):
+ def _test_error_on_stdio_flush(self, evt, break_std_streams={}):
+ for stream_name, action in break_std_streams.items():
+ if action == 'close':
+ stream = io.StringIO()
+ stream.close()
+ else:
+ assert action == 'remove'
+ stream = None
+ setattr(sys, stream_name, None)
  evt.set()
 
- def test_error_on_stdio_flush(self):
+ def test_error_on_stdio_flush_1(self):
+ # Check that Process works with broken standard streams
  streams = [io.StringIO(), None]
  streams[0].close()
  for stream_name in ('stdout', 'stderr'):
@@ -601,6 +610,24 @@ def test_error_on_stdio_flush(self):
  proc.start()
  proc.join()
  self.assertTrue(evt.is_set())
+ self.assertEqual(proc.exitcode, 0)
+ finally:
+ setattr(sys, stream_name, old_stream)
+
+ def test_error_on_stdio_flush_2(self):
+ # Same as test_error_on_stdio_flush_1(), but standard streams are
+ # broken by the child process
+ for stream_name in ('stdout', 'stderr'):
+ for action in ('close', 'remove'):
+ old_stream = getattr(sys, stream_name)
+ try:
+ evt = self.Event()
+ proc = self.Process(target=self._test_error_on_stdio_flush,
+ args=(evt, {stream_name: action}))
+ proc.start()
+ proc.join()
+ self.assertTrue(evt.is_set())
+ self.assertEqual(proc.exitcode, 0)
  finally:
  setattr(sys, stream_name, old_stream)
 

Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst

@@ -0,0 +1,2 @@
+Avoid failing in multiprocessing.Process if the standard streams are closed
+or None at exit.