This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow-site-archive.git
The following commit(s) were added to refs/heads/main by this push:
new c7d46be4d7 Fix prioritization in pool map
c7d46be4d7 is described below
commit c7d46be4d73b139f1a9f685d00e39cdeee63f65d
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun May 11 14:03:01 2025 +0200
Fix prioritization in pool map
---
scripts/transfer_utils.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/scripts/transfer_utils.py b/scripts/transfer_utils.py
index 838ed9f778..1ba51e5c21 100644
--- a/scripts/transfer_utils.py
+++ b/scripts/transfer_utils.py
@@ -101,12 +101,15 @@ class CommonTransferUtils:
@staticmethod
def run_with_pool(func: Callable, args: Any, processes: int = 4):
-
+ # Chunksize is set to 1 - otherwise map / starmap will send tasks in
chunks
+ # and the prioritization we set for folders will be lost.
+ # Our tasks are big enough to not cause overhead of sending
+ # them one at a time.
with Pool(processes=processes) as pool:
if all(isinstance(arg, tuple) for arg in args):
- pool.starmap(func, args)
+ pool.starmap(func, args, 1)
else:
- pool.map(func, args)
+ pool.map(func, args, 1)
@staticmethod
def copy(source, destination):
