diff --git a/Lib/subprocess.py b/Lib/subprocess.py index e2dac4938d57d2..3e9f58fb1f4f76 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -237,11 +237,17 @@ def __repr__(self): __del__ = Close else: - # When select or poll has indicated that the file is writable, - # we can write up to _PIPE_BUF bytes without risk of blocking. - # POSIX defines PIPE_BUF as >= 512. + # Write at most _PIPE_BUF bytes per writable event. POSIX defines + # PIPE_BUF as >= 512. Some platforms report a full pipe as writable, + # so communicate() keeps stdin non-blocking while it writes. _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) + # A full pipe that is reported as writable gives no event to wait for, + # so communicate() retries the write after a delay that doubles from + # _STDIN_BACKOFF_MIN to _STDIN_BACKOFF_MAX seconds. + _STDIN_BACKOFF_MIN = 0.001 + _STDIN_BACKOFF_MAX = 0.05 + # poll/select have the advantage of not requiring any extra file # descriptor, contrarily to epoll/kqueue (also, they require a single # syscall). @@ -282,23 +288,46 @@ def _communicate_io_posix(selector, stdin, input_view, input_offset, - Appends to output_buffers lists in place """ stdin_fd = stdin.fileno() if stdin else None - - while selector.get_map(): + # Seconds to keep stdin out of the selector after a write that + # would have blocked; 0 when stdin is registered or finished. + stdin_backoff = 0 + # The previous such delay, which the next one doubles. A + # successful write resets it. + stdin_paused_for = _STDIN_BACKOFF_MIN / 2 + + while selector.get_map() or stdin_backoff: remaining = _deadline_remaining(endtime) if remaining is not None and remaining <= 0: return (input_offset, False) # Timed out - ready = selector.select(remaining) + if stdin_backoff and (remaining is None + or stdin_backoff < remaining): + ready = selector.select(stdin_backoff) + else: + ready = selector.select(remaining) # Check timeout after select (may have woken spuriously) if endtime is not None and _time() > endtime: return (input_offset, False) # Timed out + if stdin_backoff: + # The wait is over or the child made progress on another + # pipe. Either way, ask about stdin again. + selector.register(stdin, selectors.EVENT_WRITE) + stdin_paused_for, stdin_backoff = stdin_backoff, 0 + for key, events in ready: if key.fd == stdin_fd: chunk = input_view[input_offset:input_offset + _PIPE_BUF] try: input_offset += os.write(key.fd, chunk) + except BlockingIOError: + # The pipe was reported writable but is full, and + # will be reported writable again right away. Stop + # asking for a while instead of spinning. + selector.unregister(key.fd) + stdin_backoff = min(stdin_paused_for * 2, + _STDIN_BACKOFF_MAX) except BrokenPipeError: selector.unregister(key.fd) try: @@ -306,6 +335,7 @@ def _communicate_io_posix(selector, stdin, input_view, input_offset, except BrokenPipeError: pass else: + stdin_paused_for = _STDIN_BACKOFF_MIN / 2 if input_offset >= len(input_view): selector.unregister(key.fd) try: @@ -2430,17 +2460,28 @@ def _communicate(self, input, endtime, orig_timeout): stdin_to_write = (self.stdin if self.stdin and self._input and not self.stdin.closed else None) + if stdin_to_write is not None: + # poll() and select() can report a pipe as writable + # when it has no free space (gh-162016). A blocking + # write() would then hang past the timeout. + os.set_blocking(stdin_to_write.fileno(), False) + try: + new_offset, completed = _communicate_io_posix( + selector, + stdin_to_write, + input_view, + input_offset, + self._fileobj2output, + endtime, + close_on_eof=True) + finally: + # Code that uses self.stdin after communicate() returns + # or times out expects writes to it to block. + if stdin_to_write is not None and not stdin_to_write.closed: + os.set_blocking(stdin_to_write.fileno(), True) # Persist the returned offset on self so a subsequent # communicate() after a TimeoutExpired resumes mid-input # rather than re-sending bytes the child already consumed. - new_offset, completed = _communicate_io_posix( - selector, - stdin_to_write, - input_view, - input_offset, - self._fileobj2output, - endtime, - close_on_eof=True) if self._input: self._input_offset = new_offset diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index dedd3cc0e9e867..23359d3147ade3 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3590,6 +3590,74 @@ def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): proc.communicate(timeout=999) mock_proc_stdin.close.assert_called_once_with() + def _spuriously_writable_selector(self): + # A selector that reports stdin as writable whenever it is + # registered. poll() on macOS and FreeBSD can do this for a full + # pipe when many pipes are in use system-wide (gh-162016). + class AlwaysWritableSelector(subprocess._PopenSelector): + spurious_events = 0 + + def select(self, timeout=None): + ready = super().select(0) + writable = {key.fd for key, events in ready + if events & selectors.EVENT_WRITE} + for key in self.get_map().values(): + if (key.events & selectors.EVENT_WRITE + and key.fd not in writable): + ready.append((key, selectors.EVENT_WRITE)) + type(self).spurious_events += 1 + return ready or super().select(timeout) + + return AlwaysWritableSelector + + def test_communicate_timeout_stdin_spuriously_writable(self): + # The child never reads stdin, so the pipe fills up. communicate() + # must neither block in write() past its timeout nor spin on a pipe + # that keeps being reported as writable. + selector = self._spuriously_writable_selector() + sleep = support.SHORT_TIMEOUT + with mock.patch.object(subprocess, '_PopenSelector', selector), \ + subprocess.Popen([sys.executable, '-c', + f'import time; time.sleep({sleep})'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) as proc: + try: + start = time.monotonic() + with self.assertRaises(subprocess.TimeoutExpired): + proc.communicate(b'x' * support.PIPE_MAX_SIZE, + timeout=0.5) + self.assertLess(time.monotonic() - start, sleep) + # communicate() must wait between attempts to write to a + # full pipe. Retrying in a busy loop would produce many + # thousands of these events within the timeout. + self.assertLess(selector.spurious_events, 100) + self.assertTrue(os.get_blocking(proc.stdin.fileno())) + finally: + proc.kill() + + def test_communicate_stdin_spuriously_writable_last_pipe(self): + # All input must still arrive when stdin is full, is reported as + # writable, and is the only pipe communicate() has left to service. + selector = self._spuriously_writable_selector() + data = bytes(range(256)) * (support.PIPE_MAX_SIZE // 256) + code = textwrap.dedent(f""" + import os, sys, time + os.close(1) + time.sleep(0.1) # Let the parent fill the stdin pipe. + data = bytes(range(256)) * {len(data) // 256} + sys.exit(sys.stdin.buffer.read() != data) + """) + with mock.patch.object(subprocess, '_PopenSelector', selector), \ + subprocess.Popen([sys.executable, '-c', code], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) as proc: + try: + proc.communicate(data, timeout=support.SHORT_TIMEOUT) + self.assertEqual(proc.returncode, 0) + self.assertGreater(selector.spurious_events, 0) + finally: + proc.kill() + @unittest.skipUnless(_testcapi is not None and hasattr(_testcapi, 'W_STOPCODE'), 'need _testcapi.W_STOPCODE') diff --git a/Misc/NEWS.d/next/Library/2026-11-15-17-30-00.gh-issue-162016.pNbStd.rst b/Misc/NEWS.d/next/Library/2026-11-15-17-30-00.gh-issue-162016.pNbStd.rst new file mode 100644 index 00000000000000..5fa107c8c64271 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-15-17-30-00.gh-issue-162016.pNbStd.rst @@ -0,0 +1,4 @@ +Fix :meth:`subprocess.Popen.communicate` hanging or blocking past its +*timeout* while writing to the child's stdin on macOS and FreeBSD. This +works around a kernel bug most likely seen on loaded hosts with many pipes +in active use at once.