commit: 8b96ca57e15bbc989f39ec4973fc3c9492fcc059
Author: Florian Schmaus <flow <AT> gentoo <DOT> org>
AuthorDate: Thu Aug 28 09:41:59 2025 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Fri Aug 29 12:19:50 2025 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b96ca57
process: do not poll join() in MultiprocessingProcesss
Based on the initial analysis of Esteve Varela Colominas, the polling
on join() with the fixed 100ms delay incurs a significant performance
penalty, especially for short-lived processes. And since portage is
prone to spawning many of those, the penalty adds up easily.
Instead of pooling proc.join() with a fixed 100ms delay, we now use
the blocking variant of join() started in an executor which we await
in the coroutine.
Before this change:
( cd lib; python3 -m timeit 'import portage.process;
portage.process.spawn("true")' )
2 loops, best of 5: 104 msec per loop
After this change:
( cd lib; python3 -m timeit 'import portage.process;
portage.process.spawn("true")' )
50 loops, best of 5: 4.45 msec per loop
Bug: https://bugs.gentoo.org/958635
Signed-off-by: Florian Schmaus <flow <AT> gentoo.org>
Part-of: https://github.com/gentoo/portage/pull/1452
Closes: https://github.com/gentoo/portage/pull/1452
Signed-off-by: Sam James <sam <AT> gentoo.org>
lib/portage/process.py | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 3f2c48e927..502f49f81f 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -426,10 +426,6 @@ class MultiprocessingProcess(AbstractProcess):
An object that wraps OS processes created by multiprocessing.Process.
"""
- # Number of seconds between poll attempts for process exit status
- # (after the sentinel has become ready).
- _proc_join_interval = 0.1
-
def __init__(self, proc: multiprocessing.Process):
self._proc = proc
self.pid = proc.pid
@@ -480,13 +476,7 @@ class MultiprocessingProcess(AbstractProcess):
except ValueError:
pass
- # Now that proc.sentinel is ready, poll until process exit
- # status has become available.
- while True:
- proc.join(0)
- if proc.exitcode is not None:
- break
- await asyncio.sleep(self._proc_join_interval, loop=loop)
+ await loop.run_in_executor(None, proc.join)
def _proc_join_done(self, future):
# The join task should never be cancelled, so let it raise
