- Revision
- 278380
- Author
- jbed...@apple.com
- Date
- 2021-06-02 16:10:10 -0700 (Wed, 02 Jun 2021)
Log Message
[webkitcorepy] TaskPool shouldn't fork when 1 process is needed
https://bugs.webkit.org/show_bug.cgi?id=226506
<rdar://problem/78724554>
Reviewed by Dewei Zhu.
* Scripts/libraries/webkitcorepy/setup.py: Bump version.
* Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py: Ditto.
* Scripts/libraries/webkitcorepy/webkitcorepy/task_pool.py:
(TaskPool.__init__): Allow user to force fork, even with a single process.
(TaskPool.__enter__): If only a single worker is needed and the caller is not forcing
a fork, run the setup function in the parent process.
(TaskPool.do): If no queue has been constructed, we're running in the parent process.
(TaskPool.wait): Nothing to wait for if we're running in the parent process.
(TaskPool.__exit__): If no queue has been constructed, we're running in the parent process,
so run the teardown in this process and reset the process name.
* Scripts/libraries/webkitcorepy/webkitcorepy/tests/task_pool_unittest.py:
(TaskPoolUnittest.test_single): Force fork.
(TaskPoolUnittest.test_single_no_fork):
(TaskPoolUnittest.test_exception): Force fork.
(TaskPoolUnittest.test_exception_no_fork):
(TaskPoolUnittest.test_invalid_shutdown): Force fork.
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (278379 => 278380)
--- trunk/Tools/ChangeLog 2021-06-02 22:31:09 UTC (rev 278379)
+++ trunk/Tools/ChangeLog 2021-06-02 23:10:10 UTC (rev 278380)
@@ -1,3 +1,28 @@
+2021-06-02 Jonathan Bedard <jbed...@apple.com>
+
+ [webkitcorepy] TaskPool shouldn't fork when 1 process is needed
+ https://bugs.webkit.org/show_bug.cgi?id=226506
+ <rdar://problem/78724554>
+
+ Reviewed by Dewei Zhu.
+
+ * Scripts/libraries/webkitcorepy/setup.py: Bump version.
+ * Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py: Ditto.
+ * Scripts/libraries/webkitcorepy/webkitcorepy/task_pool.py:
+ (TaskPool.__init__): Allow user to force fork, even with a single process.
+ (TaskPool.__enter__): If only a single worker is needed and the caller is not forcing
+ a fork, run the setup function in the parent process.
+ (TaskPool.do): If no queue has been constructed, we're running in the parent process.
+ (TaskPool.wait): Nothing to wait for if we're running in the parent process.
+ (TaskPool.__exit__): If no queue has been constructed, we're running in the parent process,
+ so run the teardown in this process and reset the process name.
+ * Scripts/libraries/webkitcorepy/webkitcorepy/tests/task_pool_unittest.py:
+ (TaskPoolUnittest.test_single): Force fork.
+ (TaskPoolUnittest.test_single_no_fork):
+ (TaskPoolUnittest.test_exception): Force fork.
+ (TaskPoolUnittest.test_exception_no_fork):
+ (TaskPoolUnittest.test_invalid_shutdown): Force fork.
+
2021-06-02 W.D. Xiong <w_xi...@apple.com>
[resultsdbpy] "legend" is misspelled as "lengend"
Modified: trunk/Tools/Scripts/libraries/webkitcorepy/setup.py (278379 => 278380)
--- trunk/Tools/Scripts/libraries/webkitcorepy/setup.py 2021-06-02 22:31:09 UTC (rev 278379)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/setup.py 2021-06-02 23:10:10 UTC (rev 278380)
@@ -30,7 +30,7 @@
setup(
name='webkitcorepy',
- version='0.5.15',
+ version='0.5.16',
description='Library containing various Python support classes and functions.',
long_description=readme(),
classifiers=[
Modified: trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py (278379 => 278380)
--- trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py 2021-06-02 22:31:09 UTC (rev 278379)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py 2021-06-02 23:10:10 UTC (rev 278380)
@@ -37,7 +37,7 @@
from webkitcorepy.task_pool import TaskPool
from webkitcorepy.credentials import credentials
-version = Version(0, 5, 15)
+version = Version(0, 5, 16)
from webkitcorepy.autoinstall import Package, AutoInstall
if sys.version_info > (3, 0):
Modified: trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/task_pool.py (278379 => 278380)
--- trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/task_pool.py 2021-06-02 22:31:09 UTC (rev 278379)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/task_pool.py 2021-06-02 23:10:10 UTC (rev 278380)
@@ -159,6 +159,19 @@
self.incoming.join_thread()
+class _DummyQueue(object):
+ def send(self, object):
+ if isinstance(object, _Message):
+ object(None)
+ return True
+
+ def receive(self, blocking=True):
+ pass
+
+ def close(self):
+ pass
+
+
class _Process(object):
name = None
working = False
@@ -317,6 +330,7 @@
self, workers=1, name=None, setup=None, teardown=None, grace_period=5, block_size=1000,
setupargs=None, setupkwargs=None,
teardownargs=None, teardownkwargs=None,
+ force_fork=False,
):
# Ensure tblib is installed before creating child processes
import tblib
@@ -332,8 +346,8 @@
self.queue = None
self.workers = []
- self._setup_args = (setup, setupargs, setupkwargs)
- self._teardown_args = (teardown, teardownargs, teardownkwargs)
+ self._setup_args = (setup, setupargs or [], setupkwargs or {})
+ self._teardown_args = (teardown, teardownargs or [], teardownkwargs or {})
self._num_workers = int(workers)
self._started = 0
@@ -342,8 +356,20 @@
self._id_count = 0
self.grace_period = grace_period
self.block_size = block_size
+ self.force_fork = force_fork
+ if not self.force_fork and self._num_workers == 1 and TaskPool.Process.queue:
+ raise ValueError('Illegal single-process TaskPool nesting')
+
def __enter__(self):
+ if not self.force_fork and self._num_workers == 1:
+ TaskPool.Process.queue = _DummyQueue()
+ TaskPool.Process.name = TaskPool.Process.name or '{}/0'.format(self.name)
+ if self._setup_args[0]:
+ self._setup_args[0](*self._setup_args[1], **self._setup_args[2])
+ TaskPool.Process.working = True
+ return self
+
from mock import patch
self.queue = self.BiDirectionalQueue()
@@ -362,10 +388,17 @@
worker.start()
while self._started < len(self.workers):
self.queue.receive()(self)
+
return self
def do(self, function, *args, **kwargs):
callback = kwargs.pop('callback', None)
+ if not self.queue:
+ result = function(*args, **kwargs)
+ if callback:
+ callback(result)
+ return
+
if callback:
self.callbacks[self._id_count] = callback
self.queue.send(self.Task(function, self._id_count, *args, **kwargs))
@@ -380,6 +413,9 @@
break
def wait(self):
+ if not self.queue:
+ return
+
for _ in self.workers:
self.queue.send(None)
@@ -387,6 +423,16 @@
self.queue.receive()(self)
def __exit__(self, *args, **kwargs):
+ if not self.queue:
+ TaskPool.Process.working = False
+ try:
+ if self._teardown_args[0]:
+ self._teardown_args[0](*self._teardown_args[1], **self._teardown_args[2])
+ finally:
+ TaskPool.Process.queue = None
+ TaskPool.Process.name = None
+ return
+
from six import reraise
try:
inflight = sys.exc_info()
Modified: trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/tests/task_pool_unittest.py (278379 => 278380)
--- trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/tests/task_pool_unittest.py 2021-06-02 22:31:09 UTC (rev 278379)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/tests/task_pool_unittest.py 2021-06-02 23:10:10 UTC (rev 278380)
@@ -61,7 +61,7 @@
def test_single(self):
with OutputCapture(level=logging.WARNING) as captured:
- with TaskPool(workers=1) as pool:
+ with TaskPool(workers=1, force_fork=True) as pool:
pool.do(action, 'a')
pool.do(log, logging.WARNING, '1')
pool.wait()
@@ -69,6 +69,16 @@
self.assertEqual(captured.stdout.getvalue(), 'action(a)\n')
self.assertEqual(captured.webkitcorepy.log.getvalue(), 'worker/0 1\n')
+ def test_single_no_fork(self):
+ with OutputCapture(level=logging.WARNING) as captured:
+ with TaskPool(workers=1, force_fork=False) as pool:
+ pool.do(action, 'a')
+ pool.do(log, logging.WARNING, '1')
+ pool.wait()
+
+ self.assertEqual(captured.stdout.getvalue(), 'action(a)\n')
+ self.assertEqual(captured.webkitcorepy.log.getvalue(), '1\n')
+
def test_multiple(self):
with OutputCapture(level=logging.INFO) as captured:
with TaskPool(workers=4) as pool:
@@ -99,7 +109,7 @@
def test_exception(self):
with OutputCapture(level=logging.INFO) as captured:
with self.assertRaises(RuntimeError):
- with TaskPool(workers=1) as pool:
+ with TaskPool(workers=1, force_fork=True) as pool:
pool.do(exception, 'Testing exception')
pool.wait()
self.assertEqual(
@@ -107,6 +117,14 @@
['worker/0 starting', 'worker/0 stopping'],
)
+ def test_exception_no_fork(self):
+ with OutputCapture(level=logging.INFO) as captured:
+ with self.assertRaises(RuntimeError):
+ with TaskPool(workers=1, force_fork=False) as pool:
+ pool.do(exception, 'Testing exception')
+ pool.wait()
+ self.assertEqual(captured.webkitcorepy.log.getvalue(), '')
+
def test_setup(self):
with OutputCapture() as captured:
with TaskPool(workers=4, setup=setup) as pool:
@@ -154,5 +172,5 @@
def test_invalid_shutdown(self):
with OutputCapture():
with self.assertRaises(TaskPool.Exception):
- with TaskPool(workers=1, teardown=teardown, grace_period=1) as pool:
+ with TaskPool(workers=1, teardown=teardown, grace_period=1, force_fork=True) as pool:
pool.do(wait, 2)
