bin/get-bugzilla-attachments-by-mimetype | 3 ++-
bin/get-forum-attachments.py | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
New commits:
commit 7716a43e76fe26ba31393b31cfa967f5fcb6e747
Author: Thorsten Behrens <thorsten.behr...@allotropia.de>
AuthorDate: Sat Aug 6 00:48:16 2022 +0200
Commit: Thorsten Behrens <thorsten.behr...@allotropia.de>
CommitDate: Sat Aug 6 00:53:47 2022 +0200
Make parallelism configurable for BZ attachment fetchers
Both forum and BZ document fetchers use a hardcoded size for their
threadpools. Use the habitual env var PARALLELISM to override that
optionally.
Change-Id: I374a33ab0cfe481e0940a1f4d5d9a53ae647f135
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/137888
Tested-by: Thorsten Behrens <thorsten.behr...@allotropia.de>
Reviewed-by: Thorsten Behrens <thorsten.behr...@allotropia.de>
diff --git a/bin/get-bugzilla-attachments-by-mimetype
b/bin/get-bugzilla-attachments-by-mimetype
index 349f428f73bc..6ce2a82d30eb 100755
--- a/bin/get-bugzilla-attachments-by-mimetype
+++ b/bin/get-bugzilla-attachments-by-mimetype
@@ -415,7 +415,8 @@ def generate_multi_threading():
jobs.join()
-max_threads = 20 # Number of threads to create, (1 = without multi-threading)
+# Number of threads to create, (1 = without multi-threading, default = 20)
+max_threads = int(os.environ.get('PARALLELISM', 20))
jobs = queue.Queue()
generate_multi_threading()
diff --git a/bin/get-forum-attachments.py b/bin/get-forum-attachments.py
index 18e9259d2133..92d30ccb183a 100755
--- a/bin/get-forum-attachments.py
+++ b/bin/get-forum-attachments.py
@@ -160,8 +160,8 @@ if __name__ == '__main__':
sys.exit(1)
processes = []
- # 10 at a time seems to work fine
- with ThreadPoolExecutor(max_workers=10) as executor:
+ # by default, 10 at a time seems to work fine
+ with ThreadPoolExecutor(max_workers=int(os.environ.get('PARALLELISM',
10))) as executor:
for lang, config in forums.items():
processes.append(executor.submit(get_attachments_from_url, lang,
config, pathes))
