Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-threadpoolctl for
openSUSE:Factory checked in at 2024-04-03 17:18:23
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-threadpoolctl (Old)
and /work/SRC/openSUSE:Factory/.python-threadpoolctl.new.1905 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-threadpoolctl"
Wed Apr 3 17:18:23 2024 rev:9 rq:1164225 version:3.4.0
Changes:
--------
---
/work/SRC/openSUSE:Factory/python-threadpoolctl/python-threadpoolctl.changes
2023-12-09 22:49:56.892354323 +0100
+++
/work/SRC/openSUSE:Factory/.python-threadpoolctl.new.1905/python-threadpoolctl.changes
2024-04-03 17:18:41.645689566 +0200
@@ -1,0 +2,16 @@
+Tue Apr 2 18:54:36 UTC 2024 - Dirk Müller <dmuel...@suse.com>
+
+- update to 3.4.0:
+ * Added support for Python interpreters statically linked
+ against libc or linked against alternative implementations
+ of libc like musl
+ * Added support for Pyodide
+ * Extended FlexiBLAS support to be able to switch backend at
+ runtime.
+ * Added support for FlexiBLAS
+ * Fixed a bug where an unsupported library would be detected
+ because it shares a common prefix with one of the supported
+ libraries. Now the symbols are also checked to
+ identify the supported libraries.
+
+-------------------------------------------------------------------
Old:
----
threadpoolctl-3.2.0.tar.gz
New:
----
threadpoolctl-3.4.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-threadpoolctl.spec ++++++
--- /var/tmp/diff_new_pack.6Jzvuq/_old 2024-04-03 17:18:42.617725381 +0200
+++ /var/tmp/diff_new_pack.6Jzvuq/_new 2024-04-03 17:18:42.617725381 +0200
@@ -1,7 +1,7 @@
#
# spec file for package python-threadpoolctl
#
-# Copyright (c) 2023 SUSE LLC
+# Copyright (c) 2024 SUSE LLC
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -18,7 +18,7 @@
%{?sle15_python_module_pythons}
Name: python-threadpoolctl
-Version: 3.2.0
+Version: 3.4.0
Release: 0
Summary: Thread-pool Controls
License: BSD-3-Clause
++++++ threadpoolctl-3.2.0.tar.gz -> threadpoolctl-3.4.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/.azure_pipeline.yml
new/threadpoolctl-3.4.0/.azure_pipeline.yml
--- old/threadpoolctl-3.2.0/.azure_pipeline.yml 2023-07-13 16:19:36.077411700
+0200
+++ new/threadpoolctl-3.4.0/.azure_pipeline.yml 2024-02-14 16:28:49.147424000
+0100
@@ -25,7 +25,7 @@
steps:
- task: UsePythonVersion@0
inputs:
- versionSpec: '3.11'
+ versionSpec: '3.12'
- script: |
pip install black
displayName: install black
@@ -39,21 +39,20 @@
vmImage: windows-latest
matrix:
pylatest_conda_forge_mkl:
- VERSION_PYTHON: '*'
+ PYTHON_VERSION: '*'
PACKAGER: 'conda-forge'
BLAS: 'mkl'
py310_conda_forge_openblas:
- VERSION_PYTHON: '3.10'
+ PYTHON_VERSION: '3.10'
PACKAGER: 'conda-forge'
BLAS: 'openblas'
py39_conda:
- VERSION_PYTHON: '3.9'
+ PYTHON_VERSION: '3.9'
PACKAGER: 'conda'
py38_pip:
- VERSION_PYTHON: '3.8'
+ PYTHON_VERSION: '3.8'
PACKAGER: 'pip'
-
- template: continuous_integration/posix.yml
parameters:
name: Linux
@@ -64,19 +63,19 @@
py38_ubuntu_atlas_gcc_gcc:
PACKAGER: 'ubuntu'
APT_BLAS: 'libatlas3-base libatlas-base-dev'
- VERSION_PYTHON: '3.8'
+ PYTHON_VERSION: '3.8'
CC_OUTER_LOOP: 'gcc'
CC_INNER_LOOP: 'gcc'
py38_ubuntu_openblas_gcc_gcc:
PACKAGER: 'ubuntu'
APT_BLAS: 'libopenblas-base libopenblas-dev'
- VERSION_PYTHON: '3.8'
+ PYTHON_VERSION: '3.8'
CC_OUTER_LOOP: 'gcc'
CC_INNER_LOOP: 'gcc'
# Linux + Python 3.9 and homogeneous runtime nesting.
py39_conda_openblas_clang_clang:
PACKAGER: 'conda'
- VERSION_PYTHON: '3.9'
+ PYTHON_VERSION: '3.9'
BLAS: 'openblas'
CC_OUTER_LOOP: 'clang-10'
CC_INNER_LOOP: 'clang-10'
@@ -84,7 +83,7 @@
# threadpoolctl) to only test the warning from multiple OpenMP.
pylatest_conda_mkl_clang_gcc:
PACKAGER: 'conda'
- VERSION_PYTHON: '*'
+ PYTHON_VERSION: '*'
BLAS: 'mkl'
CC_OUTER_LOOP: 'clang-10'
CC_INNER_LOOP: 'gcc'
@@ -92,7 +91,7 @@
# Linux environment with MKL, safe for threadpoolctl.
pylatest_conda_mkl_gcc_gcc:
PACKAGER: 'conda'
- VERSION_PYTHON: '*'
+ PYTHON_VERSION: '*'
BLAS: 'mkl'
CC_OUTER_LOOP: 'gcc'
CC_INNER_LOOP: 'gcc'
@@ -101,13 +100,13 @@
# and heterogeneous OpenMP runtimes.
py38_pip_openblas_gcc_clang:
PACKAGER: 'pip'
- VERSION_PYTHON: '3.8'
+ PYTHON_VERSION: '3.8'
CC_OUTER_LOOP: 'gcc'
CC_INNER_LOOP: 'clang-10'
# Linux environment with numpy from conda-forge channel and
openblas-openmp
pylatest_conda_forge:
PACKAGER: 'conda-forge'
- VERSION_PYTHON: '*'
+ PYTHON_VERSION: '*'
BLAS: 'openblas'
OPENBLAS_THREADING_LAYER: 'openmp'
CC_OUTER_LOOP: 'gcc'
@@ -116,14 +115,14 @@
pylatest_conda_nonumpy_gcc_clang:
PACKAGER: 'conda'
NO_NUMPY: 'true'
- VERSION_PYTHON: '*'
+ PYTHON_VERSION: '*'
CC_OUTER_LOOP: 'gcc'
CC_INNER_LOOP: 'clang-10'
# Linux environment with numpy linked to BLIS
pylatest_blis_gcc_clang_openmp:
PACKAGER: 'conda'
- VERSION_PYTHON: '*'
- INSTALL_BLIS: 'true'
+ PYTHON_VERSION: '*'
+ INSTALL_BLAS: 'blis'
BLIS_NUM_THREADS: '4'
CC_OUTER_LOOP: 'gcc'
CC_INNER_LOOP: 'gcc'
@@ -131,8 +130,8 @@
BLIS_ENABLE_THREADING: 'openmp'
pylatest_blis_clang_gcc_pthreads:
PACKAGER: 'conda'
- VERSION_PYTHON: '*'
- INSTALL_BLIS: 'true'
+ PYTHON_VERSION: '*'
+ INSTALL_BLAS: 'blis'
BLIS_NUM_THREADS: '4'
CC_OUTER_LOOP: 'clang-10'
CC_INNER_LOOP: 'clang-10'
@@ -140,13 +139,21 @@
BLIS_ENABLE_THREADING: 'pthreads'
pylatest_blis_no_threading:
PACKAGER: 'conda'
- VERSION_PYTHON: '*'
- INSTALL_BLIS: 'true'
+ PYTHON_VERSION: '*'
+ INSTALL_BLAS: 'blis'
BLIS_NUM_THREADS: '1'
CC_OUTER_LOOP: 'gcc'
CC_INNER_LOOP: 'gcc'
BLIS_CC: 'gcc-8'
BLIS_ENABLE_THREADING: 'no'
+ pylatest_flexiblas:
+ PACKAGER: 'conda'
+ PYTHON_VERSION: '*'
+ INSTALL_BLAS: 'flexiblas'
+ PLATFORM_SPECIFIC_PACKAGES: 'mkl'
+ CC_OUTER_LOOP: 'gcc'
+ CC_INNER_LOOP: 'gcc'
+
- template: continuous_integration/posix.yml
parameters:
@@ -156,7 +163,7 @@
# MacOS environment with OpenMP installed through homebrew
py38_conda_homebrew_libomp:
PACKAGER: 'conda'
- VERSION_PYTHON: '3.8'
+ PYTHON_VERSION: '3.8'
BLAS: 'openblas'
CC_OUTER_LOOP: 'clang'
CC_INNER_LOOP: 'clang'
@@ -164,7 +171,7 @@
# MacOS env with OpenBLAS and OpenMP installed through conda-forge
compilers
py39_conda_forge_clang_openblas:
PACKAGER: 'conda-forge'
- VERSION_PYTHON: '*'
+ PYTHON_VERSION: '*'
BLAS: 'openblas'
CC_OUTER_LOOP: 'clang'
CC_INNER_LOOP: 'clang'
@@ -172,11 +179,18 @@
# MacOS environment with OpenMP installed through conda-forge compilers
pylatest_conda_forge_clang:
PACKAGER: 'conda-forge'
- VERSION_PYTHON: '*'
+ PYTHON_VERSION: '*'
BLAS: 'mkl'
CC_OUTER_LOOP: 'clang'
CC_INNER_LOOP: 'clang'
INSTALL_LIBOMP: 'conda-forge'
+ pylatest_flexiblas:
+ PACKAGER: 'conda'
+ PYTHON_VERSION: '*'
+ INSTALL_BLAS: 'flexiblas'
+ PLATFORM_SPECIFIC_PACKAGES: 'mkl'
+ CC_OUTER_LOOP: 'clang'
+ CC_INNER_LOOP: 'clang'
- stage:
jobs:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/CHANGES.md
new/threadpoolctl-3.4.0/CHANGES.md
--- old/threadpoolctl-3.2.0/CHANGES.md 2023-07-13 16:44:13.625080800 +0200
+++ new/threadpoolctl-3.4.0/CHANGES.md 2024-03-20 14:36:20.768322000 +0100
@@ -1,3 +1,27 @@
+3.4.0 (2024-03-20)
+==================
+
+- Added support for Python interpreters statically linked against libc or
linked against
+ alternative implementations of libc like musl (on Alpine Linux for instance).
+ https://github.com/joblib/threadpoolctl/pull/171
+
+- Added support for Pyodide
+ https://github.com/joblib/threadpoolctl/pull/169
+
+3.3.0 (2024-02-14)
+==================
+
+- Extended FlexiBLAS support to be able to switch backend at runtime.
+ https://github.com/joblib/threadpoolctl/pull/163
+
+- Added support for FlexiBLAS
+ https://github.com/joblib/threadpoolctl/pull/156
+
+- Fixed a bug where an unsupported library would be detected because it shares
a common
+ prefix with one of the supported libraries. Now the symbols are also checked
to
+ identify the supported libraries.
+ https://github.com/joblib/threadpoolctl/pull/151
+
3.2.0 (2023-07-13)
==================
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/PKG-INFO
new/threadpoolctl-3.4.0/PKG-INFO
--- old/threadpoolctl-3.2.0/PKG-INFO 1970-01-01 01:00:00.000000000 +0100
+++ new/threadpoolctl-3.4.0/PKG-INFO 1970-01-01 01:00:00.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: threadpoolctl
-Version: 3.2.0
+Version: 3.4.0
Summary: threadpoolctl
Home-page: https://github.com/joblib/threadpoolctl
License: BSD-3-Clause
@@ -15,6 +15,7 @@
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
Classifier: Topic :: Software Development :: Libraries :: Python Modules
# Thread-pool Controls [](https://dev.azure.com/joblib/threadpoolctl/_build/latest?definitionId=1&branchName=master)
[](https://codecov.io/gh/joblib/threadpoolctl)
@@ -211,6 +212,92 @@
...
```
+### Switching the FlexiBLAS backend
+
+`FlexiBLAS` is a BLAS wrapper for which the BLAS backend can be switched at
runtime.
+`threadpoolctl` exposes python bindings for this feature. Here's an example
but note
+that this part of the API is experimental and subject to change without
deprecation:
+
+```python
+>>> from threadpoolctl import ThreadpoolController
+>>> import numpy as np
+>>> controller = ThreadpoolController()
+
+>>> controller.info()
+[{'user_api': 'blas',
+ 'internal_api': 'flexiblas',
+ 'num_threads': 1,
+ 'prefix': 'libflexiblas',
+ 'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+ 'version': '3.3.1',
+ 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+ 'loaded_backends': ['NETLIB'],
+ 'current_backend': 'NETLIB'}]
+
+# Retrieve the flexiblas controller
+>>> flexiblas_ct =
controller.select(internal_api="flexiblas").lib_controllers[0]
+
+# Switch the backend with one predefined at build time (listed in
"available_backends")
+>>> flexiblas_ct.switch_backend("OPENBLASPTHREAD")
+>>> controller.info()
+[{'user_api': 'blas',
+ 'internal_api': 'flexiblas',
+ 'num_threads': 4,
+ 'prefix': 'libflexiblas',
+ 'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+ 'version': '3.3.1',
+ 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+ 'loaded_backends': ['NETLIB', 'OPENBLASPTHREAD'],
+ 'current_backend': 'OPENBLASPTHREAD'},
+ {'user_api': 'blas',
+ 'internal_api': 'openblas',
+ 'num_threads': 4,
+ 'prefix': 'libopenblas',
+ 'filepath':
'/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so',
+ 'version': '0.3.8',
+ 'threading_layer': 'pthreads',
+ 'architecture': 'Haswell'}]
+
+# It's also possible to directly give the path to a shared library
+>>>
flexiblas_controller.switch_backend("/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so")
+>>> controller.info()
+[{'user_api': 'blas',
+ 'internal_api': 'flexiblas',
+ 'num_threads': 2,
+ 'prefix': 'libflexiblas',
+ 'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+ 'version': '3.3.1',
+ 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+ 'loaded_backends': ['NETLIB',
+ 'OPENBLASPTHREAD',
+ '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'],
+ 'current_backend':
'/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'},
+ {'user_api': 'openmp',
+ 'internal_api': 'openmp',
+ 'num_threads': 4,
+ 'prefix': 'libomp',
+ 'filepath':
'/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libomp.so',
+ 'version': None},
+ {'user_api': 'blas',
+ 'internal_api': 'openblas',
+ 'num_threads': 4,
+ 'prefix': 'libopenblas',
+ 'filepath':
'/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so',
+ 'version': '0.3.8',
+ 'threading_layer': 'pthreads',
+ 'architecture': 'Haswell'},
+ {'user_api': 'blas',
+ 'internal_api': 'mkl',
+ 'num_threads': 2,
+ 'prefix': 'libmkl_rt',
+ 'filepath':
'/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so.2',
+ 'version': '2024.0-Product',
+ 'threading_layer': 'gnu'}]
+```
+
+You can observe that the previously linked OpenBLAS shared object stays loaded
by
+the Python program indefinitely, but FlexiBLAS itself no longer delegates BLAS
calls
+to OpenBLAS as indicated by the `current_backend` attribute.
### Writing a custom library controller
Currently, `threadpoolctl` has support for `OpenMP` and the main `BLAS`
libraries.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/README.md
new/threadpoolctl-3.4.0/README.md
--- old/threadpoolctl-3.2.0/README.md 2023-07-12 10:17:49.739681000 +0200
+++ new/threadpoolctl-3.4.0/README.md 2024-02-14 16:28:49.151424000 +0100
@@ -192,6 +192,92 @@
...
```
+### Switching the FlexiBLAS backend
+
+`FlexiBLAS` is a BLAS wrapper for which the BLAS backend can be switched at
runtime.
+`threadpoolctl` exposes python bindings for this feature. Here's an example
but note
+that this part of the API is experimental and subject to change without
deprecation:
+
+```python
+>>> from threadpoolctl import ThreadpoolController
+>>> import numpy as np
+>>> controller = ThreadpoolController()
+
+>>> controller.info()
+[{'user_api': 'blas',
+ 'internal_api': 'flexiblas',
+ 'num_threads': 1,
+ 'prefix': 'libflexiblas',
+ 'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+ 'version': '3.3.1',
+ 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+ 'loaded_backends': ['NETLIB'],
+ 'current_backend': 'NETLIB'}]
+
+# Retrieve the flexiblas controller
+>>> flexiblas_ct =
controller.select(internal_api="flexiblas").lib_controllers[0]
+
+# Switch the backend with one predefined at build time (listed in
"available_backends")
+>>> flexiblas_ct.switch_backend("OPENBLASPTHREAD")
+>>> controller.info()
+[{'user_api': 'blas',
+ 'internal_api': 'flexiblas',
+ 'num_threads': 4,
+ 'prefix': 'libflexiblas',
+ 'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+ 'version': '3.3.1',
+ 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+ 'loaded_backends': ['NETLIB', 'OPENBLASPTHREAD'],
+ 'current_backend': 'OPENBLASPTHREAD'},
+ {'user_api': 'blas',
+ 'internal_api': 'openblas',
+ 'num_threads': 4,
+ 'prefix': 'libopenblas',
+ 'filepath':
'/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so',
+ 'version': '0.3.8',
+ 'threading_layer': 'pthreads',
+ 'architecture': 'Haswell'}]
+
+# It's also possible to directly give the path to a shared library
+>>>
flexiblas_controller.switch_backend("/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so")
+>>> controller.info()
+[{'user_api': 'blas',
+ 'internal_api': 'flexiblas',
+ 'num_threads': 2,
+ 'prefix': 'libflexiblas',
+ 'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+ 'version': '3.3.1',
+ 'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+ 'loaded_backends': ['NETLIB',
+ 'OPENBLASPTHREAD',
+ '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'],
+ 'current_backend':
'/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'},
+ {'user_api': 'openmp',
+ 'internal_api': 'openmp',
+ 'num_threads': 4,
+ 'prefix': 'libomp',
+ 'filepath':
'/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libomp.so',
+ 'version': None},
+ {'user_api': 'blas',
+ 'internal_api': 'openblas',
+ 'num_threads': 4,
+ 'prefix': 'libopenblas',
+ 'filepath':
'/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so',
+ 'version': '0.3.8',
+ 'threading_layer': 'pthreads',
+ 'architecture': 'Haswell'},
+ {'user_api': 'blas',
+ 'internal_api': 'mkl',
+ 'num_threads': 2,
+ 'prefix': 'libmkl_rt',
+ 'filepath':
'/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so.2',
+ 'version': '2024.0-Product',
+ 'threading_layer': 'gnu'}]
+```
+
+You can observe that the previously linked OpenBLAS shared object stays loaded
by
+the Python program indefinitely, but FlexiBLAS itself no longer delegates BLAS
calls
+to OpenBLAS as indicated by the `current_backend` attribute.
### Writing a custom library controller
Currently, `threadpoolctl` has support for `OpenMP` and the main `BLAS`
libraries.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/continuous_integration/install.cmd
new/threadpoolctl-3.4.0/continuous_integration/install.cmd
--- old/threadpoolctl-3.2.0/continuous_integration/install.cmd 2022-01-20
18:05:26.048133900 +0100
+++ new/threadpoolctl-3.4.0/continuous_integration/install.cmd 2023-12-13
10:38:54.935551200 +0100
@@ -9,8 +9,10 @@
@rem Deactivate any environment
call deactivate
@rem Clean up any left-over from a previous build and install version of python
+conda update -n base conda conda-libmamba-solver -q -y
+conda config --set solver libmamba
conda remove --all -q -y -n %VIRTUALENV%
-conda create -n %VIRTUALENV% -q -y python=%VERSION_PYTHON%
+conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION%
call activate %VIRTUALENV%
python -m pip install -U pip
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/continuous_integration/install.sh
new/threadpoolctl-3.4.0/continuous_integration/install.sh
--- old/threadpoolctl-3.2.0/continuous_integration/install.sh 2023-07-12
15:11:15.498224700 +0200
+++ new/threadpoolctl-3.4.0/continuous_integration/install.sh 2023-12-13
10:38:54.939551400 +0100
@@ -17,7 +17,6 @@
if [[ "$UNAMESTR" == "Darwin" ]]; then
if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then
# Install an OpenMP-enabled clang/llvm from conda-forge
-
# assumes conda-forge is set on priority channel
TO_INSTALL="$TO_INSTALL compilers llvm-openmp"
@@ -35,12 +34,14 @@
export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib
-L/usr/local/opt/libomp/lib -lomp"
fi
fi
+ conda update -n base conda conda-libmamba-solver -q --yes
+ conda config --set solver libmamba
conda create -n $VIRTUALENV -q --yes $TO_INSTALL
source activate $VIRTUALENV
}
if [[ "$PACKAGER" == "conda" ]]; then
- TO_INSTALL="python=$VERSION_PYTHON pip"
+ TO_INSTALL="python=$PYTHON_VERSION pip"
if [[ "$NO_NUMPY" != "true" ]]; then
TO_INSTALL="$TO_INSTALL numpy scipy blas[build=$BLAS]"
fi
@@ -49,7 +50,7 @@
elif [[ "$PACKAGER" == "conda-forge" ]]; then
conda config --prepend channels conda-forge
conda config --set channel_priority strict
- TO_INSTALL="python=$VERSION_PYTHON numpy scipy blas[build=$BLAS]"
+ TO_INSTALL="python=$PYTHON_VERSION numpy scipy blas[build=$BLAS]"
if [[ "$BLAS" == "openblas" && "$OPENBLAS_THREADING_LAYER" == "openmp" ]];
then
TO_INSTALL="$TO_INSTALL libopenblas=*=*openmp*"
fi
@@ -58,7 +59,7 @@
elif [[ "$PACKAGER" == "pip" ]]; then
# Use conda to build an empty python env and then use pip to install
# numpy and scipy
- TO_INSTALL="python=$VERSION_PYTHON pip"
+ TO_INSTALL="python=$PYTHON_VERSION pip"
make_conda $TO_INSTALL
if [[ "$NO_NUMPY" != "true" ]]; then
pip install numpy scipy
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/continuous_integration/install_with_blis.sh
new/threadpoolctl-3.4.0/continuous_integration/install_with_blis.sh
--- old/threadpoolctl-3.2.0/continuous_integration/install_with_blis.sh
2023-07-12 15:09:55.561690000 +0200
+++ new/threadpoolctl-3.4.0/continuous_integration/install_with_blis.sh
2024-02-06 09:45:04.404626800 +0100
@@ -13,7 +13,10 @@
sudo apt-get install libomp-dev
# create conda env
-conda create -n $VIRTUALENV -q --yes -c conda-forge python=$VERSION_PYTHON pip
cython
+conda update -n base conda conda-libmamba-solver -q --yes
+conda config --set solver libmamba
+conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \
+ pip cython meson-python pkg-config
source activate $VIRTUALENV
if [[ "$BLIS_CC" == "gcc-8" ]]; then
@@ -36,20 +39,31 @@
git clone https://github.com/numpy/numpy.git
pushd numpy
git submodule update --init
-echo "[blis]
-libraries = blis
-library_dirs = $ABS_PATH/BLIS_install/lib
-include_dirs = $ABS_PATH/BLIS_install/include/blis
-runtime_library_dirs = $ABS_PATH/BLIS_install/lib" > site.cfg
-python setup.py develop
+
+echo "libdir=$ABS_PATH/BLIS_install/lib/
+includedir=$ABS_PATH/BLIS_install/include/blis/
+version=latest
+extralib=-lm -lpthread -lgfortran
+Name: blis
+Description: BLIS
+Version: \${version}
+Libs: -L\${libdir} -lblis
+Libs.private: \${extralib}
+Cflags: -I\${includedir}" > blis.pc
+
+PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation
-Csetup-args=-Dblas=blis
popd
popd
python -m pip install -q -r dev-requirements.txt
-CFLAGS=-I$ABS_PATH/BLIS_install/include/blis
LDFLAGS=-L$ABS_PATH/BLIS_install/lib \
+CFLAGS=-I$ABS_PATH/BLIS_install/include/blis \
+ LDFLAGS="-L$ABS_PATH/BLIS_install/lib
-Wl,-rpath,$ABS_PATH/BLIS_install/lib" \
bash ./continuous_integration/build_test_ext.sh
+# Check that BLIS is linked
+ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so
+
python --version
python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no
numpy"
python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no
scipy"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/continuous_integration/install_with_flexiblas.sh
new/threadpoolctl-3.4.0/continuous_integration/install_with_flexiblas.sh
--- old/threadpoolctl-3.2.0/continuous_integration/install_with_flexiblas.sh
1970-01-01 01:00:00.000000000 +0100
+++ new/threadpoolctl-3.4.0/continuous_integration/install_with_flexiblas.sh
2024-02-14 16:28:49.151424000 +0100
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -e
+
+pushd ..
+ABS_PATH=$(pwd)
+popd
+
+# create conda env
+conda update -n base conda conda-libmamba-solver -q --yes
+conda config --set solver libmamba
+conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \
+ pip cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config
compilers
+source activate $VIRTUALENV
+
+pushd ..
+
+# build & install FlexiBLAS
+mkdir flexiblas_install
+git clone https://github.com/mpimd-csc/flexiblas.git
+pushd flexiblas
+
+mkdir build
+pushd build
+
+EXTENSION=".so"
+if [[ $(uname) == "Darwin" ]]; then
+ EXTENSION=".dylib"
+fi
+
+# We intentionally restrict the list of backends to make it easier to
+# write platform agnostic tests. In particular, we do not detect OS
+# provided backends such as macOS' Apple/Accelerate/vecLib nor plaftorm
+# specific BLAS implementations such as MKL that cannot be installed on
+# arm64 hardware.
+cmake ../ -DCMAKE_INSTALL_PREFIX=$ABS_PATH"/flexiblas_install" \
+ -DBLAS_AUTO_DETECT="OFF" \
+ -DEXTRA="OPENBLAS_CONDA" \
+ -DFLEXIBLAS_DEFAULT="OPENBLAS_CONDA" \
+ -DOPENBLAS_CONDA_LIBRARY=$CONDA_PREFIX"/lib/libopenblas"$EXTENSION \
+make
+make install
+
+# Check that all 3 BLAS are listed in FlexiBLAS configuration
+$ABS_PATH/flexiblas_install/bin/flexiblas list
+popd
+popd
+
+# build & install numpy
+git clone https://github.com/numpy/numpy.git
+pushd numpy
+git submodule update --init
+
+echo "libdir=$ABS_PATH/flexiblas_install/lib/
+includedir=$ABS_PATH/flexiblas_install/include/flexiblas/
+version=3.3.1
+extralib=-lm -lpthread -lgfortran
+Name: flexiblas
+Description: FlexiBLAS - a BLAS wrapper
+Version: \${version}
+Libs: -L\${libdir} -lflexiblas
+Libs.private: \${extralib}
+Cflags: -I\${includedir}" > flexiblas.pc
+
+PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation
-Csetup-args=-Dblas=flexiblas -Csetup-args=-Dlapack=flexiblas
+popd
+
+popd
+
+python -m pip install -q -r dev-requirements.txt
+CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas \
+ LDFLAGS="-L$ABS_PATH/flexiblas_install/lib
-Wl,-rpath,$ABS_PATH/flexiblas_install/lib" \
+ bash ./continuous_integration/build_test_ext.sh
+
+# Check that FlexiBLAS is linked
+if [[ $(uname) != "Darwin" ]]; then
+ ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so
+fi
+
+python --version
+python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no
numpy"
+python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no
scipy"
+
+python -m flit install --symlink
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/continuous_integration/posix.yml
new/threadpoolctl-3.4.0/continuous_integration/posix.yml
--- old/threadpoolctl-3.2.0/continuous_integration/posix.yml 2023-07-13
16:19:36.077411700 +0200
+++ new/threadpoolctl-3.4.0/continuous_integration/posix.yml 2024-02-07
13:42:18.877825700 +0100
@@ -22,12 +22,16 @@
condition: eq('${{ parameters.name }}', 'macOS')
- script: |
continuous_integration/install.sh
- displayName: 'Install without BLIS'
- condition: ne(variables['INSTALL_BLIS'], 'true')
+ displayName: 'Install without custom BLAS'
+ condition: eq(variables['INSTALL_BLAS'], '')
- script: |
continuous_integration/install_with_blis.sh
displayName: 'Install with BLIS'
- condition: eq(variables['INSTALL_BLIS'], 'true')
+ condition: eq(variables['INSTALL_BLAS'], 'blis')
+ - script: |
+ continuous_integration/install_with_flexiblas.sh
+ displayName: 'Install with FlexiBLAS'
+ condition: eq(variables['INSTALL_BLAS'], 'flexiblas')
- script: |
continuous_integration/test_script.sh
displayName: 'Test Library'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/continuous_integration/test_script.cmd
new/threadpoolctl-3.4.0/continuous_integration/test_script.cmd
--- old/threadpoolctl-3.2.0/continuous_integration/test_script.cmd
2022-01-20 18:05:26.048133900 +0100
+++ new/threadpoolctl-3.4.0/continuous_integration/test_script.cmd
2023-12-13 10:38:54.939551400 +0100
@@ -1,5 +1,8 @@
call activate %VIRTUALENV%
+# Display version information
+python -m pip list
+
# Use the CLI to display the effective runtime environment prior to
# launching the tests:
python -m threadpoolctl -i numpy scipy.linalg
tests._openmp_test_helper.openmp_helpers_inner
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/continuous_integration/test_script.sh
new/threadpoolctl-3.4.0/continuous_integration/test_script.sh
--- old/threadpoolctl-3.2.0/continuous_integration/test_script.sh
2022-01-20 18:05:26.048133900 +0100
+++ new/threadpoolctl-3.4.0/continuous_integration/test_script.sh
2024-01-17 11:19:28.785481200 +0100
@@ -4,11 +4,14 @@
if [[ "$PACKAGER" == conda* ]]; then
source activate $VIRTUALENV
+ conda list
elif [[ "$PACKAGER" == "pip" ]]; then
# we actually use conda to install the base environment:
source activate $VIRTUALENV
+ pip list
elif [[ "$PACKAGER" == "ubuntu" ]]; then
source $VIRTUALENV/bin/activate
+ pip list
fi
set -x
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/pyproject.toml
new/threadpoolctl-3.4.0/pyproject.toml
--- old/threadpoolctl-3.2.0/pyproject.toml 2023-07-12 10:17:49.743681400
+0200
+++ new/threadpoolctl-3.4.0/pyproject.toml 2024-02-09 15:27:12.738413800
+0100
@@ -1,7 +1,9 @@
[build-system]
-requires = ["flit_core"]
+requires = ["flit_core >=2,<4"]
build-backend = "flit_core.buildapi"
+# TODO: replace the following section by the standard [project] section to be
able
+# to use flit v4.
[tool.flit.metadata]
module = "threadpoolctl"
author = "Thomas Moreau"
@@ -18,10 +20,11 @@
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
"Topic :: Software Development :: Libraries :: Python Modules",
]
[tool.black]
line-length = 88
-target_version = ['py38', 'py39', 'py310', 'py311']
+target_version = ['py38', 'py39', 'py310', 'py311', 'py312']
preview = true
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/tests/_openmp_test_helper/nested_prange_blas.pyx
new/threadpoolctl-3.4.0/tests/_openmp_test_helper/nested_prange_blas.pyx
--- old/threadpoolctl-3.2.0/tests/_openmp_test_helper/nested_prange_blas.pyx
2021-09-29 18:23:34.090675000 +0200
+++ new/threadpoolctl-3.4.0/tests/_openmp_test_helper/nested_prange_blas.pyx
2023-12-13 10:38:54.939551400 +0100
@@ -2,23 +2,7 @@
from cython.parallel import parallel, prange
import numpy as np
-
-IF USE_BLIS:
- cdef extern from 'cblas.h' nogil:
- ctypedef enum CBLAS_ORDER:
- CblasRowMajor=101
- CblasColMajor=102
- ctypedef enum CBLAS_TRANSPOSE:
- CblasNoTrans=111
- CblasTrans=112
- CblasConjTrans=113
- void dgemm 'cblas_dgemm' (
- CBLAS_ORDER Order, CBLAS_TRANSPOSE TransA,
- CBLAS_TRANSPOSE TransB, int M, int N,
- int K, double alpha, double *A, int lda,
- double *B, int ldb, double beta, double *C, int ldc)
-ELSE:
- from scipy.linalg.cython_blas cimport dgemm
+from scipy.linalg.cython_blas cimport dgemm
from threadpoolctl import ThreadpoolController
@@ -53,13 +37,8 @@
prange_num_threads_ptr[0] = openmp.omp_get_num_threads()
for i in prange(n_chunks):
- IF USE_BLIS:
- dgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
- chunk_size, n, k, alpha, &A[i * chunk_size, 0], k,
- &B[0, 0], k, beta, &C[i * chunk_size, 0], n)
- ELSE:
- dgemm(trans, no_trans, &n, &chunk_size, &k,
- &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k,
- &beta, &C[i * chunk_size, 0], &n)
+ dgemm(trans, no_trans, &n, &chunk_size, &k,
+ &alpha, &B[0, 0], &k, &A[i * chunk_size, 0], &k,
+ &beta, &C[i * chunk_size, 0], &n)
return np.asarray(C), prange_num_threads, inner_info[0]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/tests/_openmp_test_helper/nested_prange_blas_custom.pyx
new/threadpoolctl-3.4.0/tests/_openmp_test_helper/nested_prange_blas_custom.pyx
---
old/threadpoolctl-3.2.0/tests/_openmp_test_helper/nested_prange_blas_custom.pyx
1970-01-01 01:00:00.000000000 +0100
+++
new/threadpoolctl-3.4.0/tests/_openmp_test_helper/nested_prange_blas_custom.pyx
2024-02-07 13:42:18.885825900 +0100
@@ -0,0 +1,55 @@
+cimport openmp
+from cython.parallel import parallel, prange
+
+import numpy as np
+
+cdef extern from 'cblas.h' nogil:
+ ctypedef enum CBLAS_ORDER:
+ CblasRowMajor=101
+ CblasColMajor=102
+ ctypedef enum CBLAS_TRANSPOSE:
+ CblasNoTrans=111
+ CblasTrans=112
+ CblasConjTrans=113
+ void dgemm 'cblas_dgemm' (
+ CBLAS_ORDER Order, CBLAS_TRANSPOSE TransA,
+ CBLAS_TRANSPOSE TransB, int M, int N,
+ int K, double alpha, double *A, int lda,
+ double *B, int ldb, double beta, double *C, int ldc)
+
+from threadpoolctl import ThreadpoolController
+
+
+def check_nested_prange_blas(double[:, ::1] A, double[:, ::1] B, int nthreads):
+ """Run multithreaded BLAS calls within OpenMP parallel loop"""
+ cdef:
+ int m = A.shape[0]
+ int n = B.shape[0]
+ int k = A.shape[1]
+
+ double[:, ::1] C = np.empty((m, n))
+ int n_chunks = 100
+ int chunk_size = A.shape[0] // n_chunks
+
+ double alpha = 1.0
+ double beta = 0.0
+
+ int i
+ int prange_num_threads
+ int *prange_num_threads_ptr = &prange_num_threads
+
+ inner_info = [None]
+
+ with nogil, parallel(num_threads=nthreads):
+ if openmp.omp_get_thread_num() == 0:
+ with gil:
+ inner_info[0] = ThreadpoolController().info()
+
+ prange_num_threads_ptr[0] = openmp.omp_get_num_threads()
+
+ for i in prange(n_chunks):
+ dgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
+ chunk_size, n, k, alpha, &A[i * chunk_size, 0], k,
+ &B[0, 0], k, beta, &C[i * chunk_size, 0], n)
+
+ return np.asarray(C), prange_num_threads, inner_info[0]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/tests/_openmp_test_helper/openmp_helpers_inner.pxd
new/threadpoolctl-3.4.0/tests/_openmp_test_helper/openmp_helpers_inner.pxd
--- old/threadpoolctl-3.2.0/tests/_openmp_test_helper/openmp_helpers_inner.pxd
2021-09-15 15:26:21.493843300 +0200
+++ new/threadpoolctl-3.4.0/tests/_openmp_test_helper/openmp_helpers_inner.pxd
2024-02-06 09:45:04.408626800 +0100
@@ -1 +1 @@
-cdef int inner_openmp_loop(int) nogil
+cdef int inner_openmp_loop(int) noexcept nogil
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/tests/_openmp_test_helper/openmp_helpers_inner.pyx
new/threadpoolctl-3.4.0/tests/_openmp_test_helper/openmp_helpers_inner.pyx
--- old/threadpoolctl-3.2.0/tests/_openmp_test_helper/openmp_helpers_inner.pyx
2021-12-31 17:52:30.808669800 +0100
+++ new/threadpoolctl-3.4.0/tests/_openmp_test_helper/openmp_helpers_inner.pyx
2024-02-06 09:45:04.408626800 +0100
@@ -15,7 +15,7 @@
return num_threads
-cdef int inner_openmp_loop(int n) nogil:
+cdef int inner_openmp_loop(int n) noexcept nogil:
"""Run a short parallel section with OpenMP
Return the number of threads that where effectively used by the
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/threadpoolctl-3.2.0/tests/_openmp_test_helper/setup_nested_prange_blas.py
new/threadpoolctl-3.4.0/tests/_openmp_test_helper/setup_nested_prange_blas.py
---
old/threadpoolctl-3.2.0/tests/_openmp_test_helper/setup_nested_prange_blas.py
2021-09-24 10:18:13.956710000 +0200
+++
new/threadpoolctl-3.4.0/tests/_openmp_test_helper/setup_nested_prange_blas.py
2024-02-07 13:42:18.877825700 +0100
@@ -12,13 +12,15 @@
set_cc_variables("CC_OUTER_LOOP")
openmp_flag = get_openmp_flag()
- use_blis = os.getenv("INSTALL_BLIS", False)
- libraries = ["blis"] if use_blis else []
+ use_custom_blas = os.getenv("INSTALL_BLAS", False)
+ libraries = [use_custom_blas] if use_custom_blas else []
+ custom_suffix = "_custom" if use_custom_blas else ""
+ filename = f"nested_prange_blas{custom_suffix}.pyx"
ext_modules = [
Extension(
"nested_prange_blas",
- ["nested_prange_blas.pyx"],
+ [filename],
extra_compile_args=openmp_flag,
extra_link_args=openmp_flag,
libraries=libraries,
@@ -29,7 +31,6 @@
name="_openmp_test_helper_nested_prange_blas",
ext_modules=cythonize(
ext_modules,
- compile_time_env={"USE_BLIS": use_blis},
compiler_directives={
"language_level": 3,
"boundscheck": False,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/tests/_pyMylib/__init__.py
new/threadpoolctl-3.4.0/tests/_pyMylib/__init__.py
--- old/threadpoolctl-3.2.0/tests/_pyMylib/__init__.py 2023-07-12
10:17:49.743681400 +0200
+++ new/threadpoolctl-3.4.0/tests/_pyMylib/__init__.py 2023-12-13
10:38:54.943551300 +0100
@@ -19,6 +19,14 @@
# instance.
filename_prefixes = ("my_threaded_lib",)
+ # (Optional) Symbols that the linked library is expected to expose. It is
used along
+ # with the `filename_prefixes` to make sure that the correct library is
identified.
+ check_symbols = (
+ "mylib_get_num_threads",
+ "mylib_set_num_threads",
+ "mylib_get_version",
+ )
+
def get_num_threads(self):
# This function should return the current maximum number of threads,
# which is reported as "num_threads" by `ThreadpoolController.info`.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/tests/test_threadpoolctl.py
new/threadpoolctl-3.4.0/tests/test_threadpoolctl.py
--- old/threadpoolctl-3.2.0/tests/test_threadpoolctl.py 2023-07-12
10:17:49.743681400 +0200
+++ new/threadpoolctl-3.4.0/tests/test_threadpoolctl.py 2024-02-14
16:28:49.155423900 +0100
@@ -10,6 +10,7 @@
from threadpoolctl import _ALL_PREFIXES, _ALL_USER_APIS
from .utils import cython_extensions_compiled
+from .utils import check_nested_prange_blas
from .utils import libopenblas_paths
from .utils import scipy
from .utils import threadpool_info_from_subprocess
@@ -406,19 +407,20 @@
test_shipped_openblas()
-@pytest.mark.skipif(scipy is None, reason="requires scipy")
@pytest.mark.skipif(
not cython_extensions_compiled, reason="Requires cython extensions to be
compiled"
)
+@pytest.mark.skipif(
+ check_nested_prange_blas is None,
+ reason="Requires nested_prange_blas to be compiled",
+)
@pytest.mark.parametrize("nthreads_outer", [None, 1, 2, 4])
def test_nested_prange_blas(nthreads_outer):
- # Check that the BLAS linked to scipy effectively uses the number of
- # threads requested by the context manager when nested in an outer OpenMP
- # loop.
+ # Check that the BLAS uses the number of threads requested by the context
manager
+ # when nested in an outer OpenMP loop.
+ # Remark: this test also works with sequential BLAS only because we limit
the
+ # number of threads for the BLAS to 1.
import numpy as np
- import tests._openmp_test_helper.nested_prange_blas as prange_blas
-
- check_nested_prange_blas = prange_blas.check_nested_prange_blas
skip_if_openblas_openmp()
@@ -545,9 +547,20 @@
assert "multiple_openmp.md" in str(wm.message)
-def test_command_line_empty():
+def test_command_line_empty_or_system_openmp():
+ # When the command line is called without arguments, no library should be
+ # detected. The only exception is a system OpenMP library that can be
+ # linked to the Python interpreter, for instance via the libb2.so BLAKE2
+ # library that can itself be linked to an OpenMP runtime on Gentoo.
output = subprocess.check_output((sys.executable + " -m
threadpoolctl").split())
- assert json.loads(output.decode("utf-8")) == []
+ results = json.loads(output.decode("utf-8"))
+ conda_prefix = os.getenv("CONDA_PREFIX")
+ managed_by_conda = conda_prefix and sys.executable.startswith(conda_prefix)
+ if not managed_by_conda: # pragma: no cover
+ # When using a Python interpreter that does not come from a conda
+ # environment, we should ignore any system OpenMP library.
+ results = [r for r in results if r["user_api"] != "openmp"]
+ assert results == []
def test_command_line_command_flag():
@@ -600,12 +613,12 @@
expected_openblas_architectures = (
# XXX: add more as needed by CI or developer laptops
"armv8",
- "Haswell",
- "Prescott", # see: https://github.com/xianyi/OpenBLAS/pull/3485
- "SkylakeX",
- "Sandybridge",
- "VORTEX",
- "Zen",
+ "haswell",
+ "prescott", # see: https://github.com/xianyi/OpenBLAS/pull/3485
+ "skylakex",
+ "sandybridge",
+ "vortex",
+ "zen",
)
expected_blis_architectures = (
# XXX: add more as needed by CI or developer laptops
@@ -614,9 +627,9 @@
)
for lib_info in threadpool_info():
if lib_info["internal_api"] == "openblas":
- assert lib_info["architecture"] in expected_openblas_architectures
+ assert lib_info["architecture"].lower() in
expected_openblas_architectures
elif lib_info["internal_api"] == "blis":
- assert lib_info["architecture"] in expected_blis_architectures
+ assert lib_info["architecture"].lower() in
expected_blis_architectures
else:
# Not supported for other libraries
assert "architecture" not in lib_info
@@ -641,6 +654,81 @@
assert threading_layer in expected_openblas_threading_layers
+# skip test if not run in a azure pipelines job since it relies on a specific
flexiblas
+# installation.
+@pytest.mark.skipif(
+ "TF_BUILD" not in os.environ, reason="not running in azure pipelines"
+)
+def test_flexiblas():
+ # Check that threadpool_info correctly recovers the FlexiBLAS backends.
+ flexiblas_controller =
ThreadpoolController().select(internal_api="flexiblas")
+
+ if not flexiblas_controller:
+ pytest.skip("requires FlexiBLAS.")
+ flexiblas_controller = flexiblas_controller.lib_controllers[0]
+
+ expected_backends = {"NETLIB", "OPENBLAS_CONDA"}
+ expected_backends_loaded = {"OPENBLAS_CONDA"}
+ expected_current_backend = "OPENBLAS_CONDA" # set as default at build time
+
+ flexiblas_backends = flexiblas_controller.available_backends
+ flexiblas_backends_loaded = flexiblas_controller.loaded_backends
+ current_backend = flexiblas_controller.current_backend
+
+ assert set(flexiblas_backends) == expected_backends
+ assert set(flexiblas_backends_loaded) == expected_backends_loaded
+ assert current_backend == expected_current_backend
+
+
+def test_flexiblas_switch_error():
+ # Check that an error is raised when trying to switch to an invalid
backend.
+ flexiblas_controller =
ThreadpoolController().select(internal_api="flexiblas")
+
+ if not flexiblas_controller:
+ pytest.skip("requires FlexiBLAS.")
+ flexiblas_controller = flexiblas_controller.lib_controllers[0]
+
+ with pytest.raises(RuntimeError, match="Failed to load backend"):
+ flexiblas_controller.switch_backend("INVALID_BACKEND")
+
+
+# skip test if not run in a azure pipelines job since it relies on a specific
flexiblas
+# installation.
+@pytest.mark.skipif(
+ "TF_BUILD" not in os.environ, reason="not running in azure pipelines"
+)
+def test_flexiblas_switch():
+ # Check that the backend can be switched.
+ controller = ThreadpoolController()
+ fb_controller = controller.select(internal_api="flexiblas")
+
+ if not fb_controller:
+ pytest.skip("requires FlexiBLAS.")
+ fb_controller = fb_controller.lib_controllers[0]
+
+ # at first mkl is not loaded in the CI jobs where this test runs
+ assert len(controller.select(internal_api="mkl").lib_controllers) == 0
+
+ # at first, only "OPENBLAS_CONDA" is loaded
+ assert fb_controller.current_backend == "OPENBLAS_CONDA"
+ assert fb_controller.loaded_backends == ["OPENBLAS_CONDA"]
+
+ fb_controller.switch_backend("NETLIB")
+ assert fb_controller.current_backend == "NETLIB"
+ assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB"]
+
+ ext = ".so" if sys.platform == "linux" else ".dylib"
+ mkl_path = f"{os.getenv('CONDA_PREFIX')}/lib/libmkl_rt{ext}"
+ fb_controller.switch_backend(mkl_path)
+ assert fb_controller.current_backend == mkl_path
+ assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB",
mkl_path]
+ # switching the backend triggered a new search for loaded shared libs
+ assert len(controller.select(internal_api="mkl").lib_controllers) == 1
+
+ # switch back to default to avoid side effects
+ fb_controller.switch_backend("OPENBLAS_CONDA")
+
+
def test_threadpool_controller_as_decorator():
# Check that using the decorator can be nested and is restricted to the
scope of
# the decorated function.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/tests/utils.py
new/threadpoolctl-3.4.0/tests/utils.py
--- old/threadpoolctl-3.2.0/tests/utils.py 2021-09-29 18:23:34.090675000
+0200
+++ new/threadpoolctl-3.4.0/tests/utils.py 2024-02-07 13:42:18.877825700
+0100
@@ -47,6 +47,12 @@
cython_extensions_compiled = False
+try:
+ from tests._openmp_test_helper.nested_prange_blas import
check_nested_prange_blas
+except ImportError:
+ check_nested_prange_blas = None
+
+
def threadpool_info_from_subprocess(module):
"""Utility to call threadpool_info in a subprocess
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/threadpoolctl-3.2.0/threadpoolctl.py
new/threadpoolctl-3.4.0/threadpoolctl.py
--- old/threadpoolctl-3.2.0/threadpoolctl.py 2023-07-13 16:44:13.629080800
+0200
+++ new/threadpoolctl-3.4.0/threadpoolctl.py 2024-03-20 14:36:20.768322000
+0100
@@ -4,6 +4,7 @@
thread pools (notably BLAS and OpenMP implementations) and dynamically set the
maximal number of threads they can use.
"""
+
# License: BSD 3-Clause
# The code to introspect dynamically loaded libraries on POSIX systems is
@@ -22,7 +23,7 @@
from functools import lru_cache
from contextlib import ContextDecorator
-__version__ = "3.2.0"
+__version__ = "3.4.0"
__all__ = [
"threadpool_limits",
"threadpool_info",
@@ -104,20 +105,17 @@
"""
@final
- def __init__(self, *, filepath=None, prefix=None):
+ def __init__(self, *, filepath=None, prefix=None, parent=None):
"""This is not meant to be overriden by subclasses."""
+ self.parent = parent
self.prefix = prefix
self.filepath = filepath
self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD)
self.version = self.get_version()
self.set_additional_attributes()
- @final
def info(self):
- """Return relevant info wrapped in a dict
-
- This is not meant to be overriden by subclasses.
- """
+ """Return relevant info wrapped in a dict"""
exposed_attrs = {
"user_api": self.user_api,
"internal_api": self.internal_api,
@@ -125,6 +123,7 @@
**vars(self),
}
exposed_attrs.pop("dynlib")
+ exposed_attrs.pop("parent")
return exposed_attrs
def set_additional_attributes(self):
@@ -157,7 +156,18 @@
user_api = "blas"
internal_api = "openblas"
filename_prefixes = ("libopenblas", "libblas")
- check_symbols = ("openblas_get_num_threads", "openblas_get_num_threads64_")
+ check_symbols = (
+ "openblas_get_num_threads",
+ "openblas_get_num_threads64_",
+ "openblas_set_num_threads",
+ "openblas_set_num_threads64_",
+ "openblas_get_config",
+ "openblas_get_config64_",
+ "openblas_get_parallel",
+ "openblas_get_parallel64_",
+ "openblas_get_corename",
+ "openblas_get_corename64_",
+ )
def set_additional_attributes(self):
self.threading_layer = self._get_threading_layer()
@@ -237,7 +247,15 @@
user_api = "blas"
internal_api = "blis"
filename_prefixes = ("libblis", "libblas")
- check_symbols = ("bli_thread_get_num_threads",)
+ check_symbols = (
+ "bli_thread_get_num_threads",
+ "bli_thread_set_num_threads",
+ "bli_info_get_version_str",
+ "bli_info_get_enable_openmp",
+ "bli_info_get_enable_pthreads",
+ "bli_arch_query_id",
+ "bli_arch_string",
+ )
def set_additional_attributes(self):
self.threading_layer = self._get_threading_layer()
@@ -266,9 +284,9 @@
def _get_threading_layer(self):
"""Return the threading layer of BLIS"""
- if self.dynlib.bli_info_get_enable_openmp():
+ if getattr(self.dynlib, "bli_info_get_enable_openmp", lambda: False)():
return "openmp"
- elif self.dynlib.bli_info_get_enable_pthreads():
+ elif getattr(self.dynlib, "bli_info_get_enable_pthreads", lambda:
False)():
return "pthreads"
return "disabled"
@@ -286,13 +304,146 @@
return bli_arch_string(bli_arch_query_id()).decode("utf-8")
+class FlexiBLASController(LibController):
+ """Controller class for FlexiBLAS"""
+
+ user_api = "blas"
+ internal_api = "flexiblas"
+ filename_prefixes = ("libflexiblas",)
+ check_symbols = (
+ "flexiblas_get_num_threads",
+ "flexiblas_set_num_threads",
+ "flexiblas_get_version",
+ "flexiblas_list",
+ "flexiblas_list_loaded",
+ "flexiblas_current_backend",
+ )
+
+ @property
+ def loaded_backends(self):
+ return self._get_backend_list(loaded=True)
+
+ @property
+ def current_backend(self):
+ return self._get_current_backend()
+
+ def info(self):
+ """Return relevant info wrapped in a dict"""
+ # We override the info method because the loaded and current backends
+ # are dynamic properties
+ exposed_attrs = super().info()
+ exposed_attrs["loaded_backends"] = self.loaded_backends
+ exposed_attrs["current_backend"] = self.current_backend
+
+ return exposed_attrs
+
+ def set_additional_attributes(self):
+ self.available_backends = self._get_backend_list(loaded=False)
+
+ def get_num_threads(self):
+ get_func = getattr(self.dynlib, "flexiblas_get_num_threads", lambda:
None)
+ num_threads = get_func()
+ # by default BLIS is single-threaded and get_num_threads
+ # returns -1. We map it to 1 for consistency with other libraries.
+ return 1 if num_threads == -1 else num_threads
+
+ def set_num_threads(self, num_threads):
+ set_func = getattr(
+ self.dynlib, "flexiblas_set_num_threads", lambda num_threads: None
+ )
+ return set_func(num_threads)
+
+ def get_version(self):
+ get_version_ = getattr(self.dynlib, "flexiblas_get_version", None)
+ if get_version_ is None:
+ return None
+
+ major = ctypes.c_int()
+ minor = ctypes.c_int()
+ patch = ctypes.c_int()
+ get_version_(ctypes.byref(major), ctypes.byref(minor),
ctypes.byref(patch))
+ return f"{major.value}.{minor.value}.{patch.value}"
+
+ def _get_backend_list(self, loaded=False):
+ """Return the list of available backends for FlexiBLAS.
+
+ If loaded is False, return the list of available backends from the
FlexiBLAS
+ configuration. If loaded is True, return the list of actually loaded
backends.
+ """
+ func_name = f"flexiblas_list{'_loaded' if loaded else ''}"
+ get_backend_list_ = getattr(self.dynlib, func_name, None)
+ if get_backend_list_ is None:
+ return None
+
+ n_backends = get_backend_list_(None, 0, 0)
+
+ backends = []
+ for i in range(n_backends):
+ backend_name = ctypes.create_string_buffer(1024)
+ get_backend_list_(backend_name, 1024, i)
+ if backend_name.value.decode("utf-8") != "__FALLBACK__":
+ # We don't know when to expect __FALLBACK__ but it is not a
real
+ # backend and does not show up when running flexiblas list.
+ backends.append(backend_name.value.decode("utf-8"))
+ return backends
+
+ def _get_current_backend(self):
+ """Return the backend of FlexiBLAS"""
+ get_backend_ = getattr(self.dynlib, "flexiblas_current_backend", None)
+ if get_backend_ is None:
+ return None
+
+ backend = ctypes.create_string_buffer(1024)
+ get_backend_(backend, ctypes.sizeof(backend))
+ return backend.value.decode("utf-8")
+
+ def switch_backend(self, backend):
+ """Switch the backend of FlexiBLAS
+
+ Parameters
+ ----------
+ backend : str
+ The name or the path to the shared library of the backend to
switch to. If
+ the backend is not already loaded, it will be loaded first.
+ """
+ if backend not in self.loaded_backends:
+ if backend in self.available_backends:
+ load_func = getattr(self.dynlib, "flexiblas_load_backend",
lambda _: -1)
+ else: # assume backend is a path to a shared library
+ load_func = getattr(
+ self.dynlib, "flexiblas_load_backend_library", lambda _: -1
+ )
+ res = load_func(str(backend).encode("utf-8"))
+ if res == -1:
+ raise RuntimeError(
+ f"Failed to load backend {backend!r}. It must either be
the name of"
+ " a backend available in the FlexiBLAS configuration "
+ f"{self.available_backends} or the path to a valid shared
library."
+ )
+
+ # Trigger a new search of loaded shared libraries since loading a
new
+ # backend caused a dlopen.
+ self.parent._load_libraries()
+
+ switch_func = getattr(self.dynlib, "flexiblas_switch", lambda _: -1)
+ idx = self.loaded_backends.index(backend)
+ res = switch_func(idx)
+ if res == -1:
+ raise RuntimeError(f"Failed to switch to backend {backend!r}.")
+
+
class MKLController(LibController):
"""Controller class for MKL"""
user_api = "blas"
internal_api = "mkl"
filename_prefixes = ("libmkl_rt", "mkl_rt", "libblas")
- check_symbols = ("MKL_Get_Max_Threads",)
+ check_symbols = (
+ "MKL_Get_Max_Threads",
+ "MKL_Set_Num_Threads",
+ "MKL_Get_Version_String",
+ "MKL_Set_Threading_Layer",
+ )
def set_additional_attributes(self):
self.threading_layer = self._get_threading_layer()
@@ -343,6 +494,10 @@
user_api = "openmp"
internal_api = "openmp"
filename_prefixes = ("libiomp", "libgomp", "libomp", "vcomp")
+ check_symbols = (
+ "omp_get_max_threads",
+ "omp_get_num_threads",
+ )
def get_num_threads(self):
get_func = getattr(self.dynlib, "omp_get_max_threads", lambda: None)
@@ -359,7 +514,13 @@
# Controllers for the libraries that we'll look for in the loaded libraries.
# Third party libraries can register their own controllers.
-_ALL_CONTROLLERS = [OpenBLASController, BLISController, MKLController,
OpenMPController]
+_ALL_CONTROLLERS = [
+ OpenBLASController,
+ BLISController,
+ MKLController,
+ OpenMPController,
+ FlexiBLASController,
+]
# Helpers for the doc and test names
_ALL_USER_APIS = list(set(lib.user_api for lib in _ALL_CONTROLLERS))
@@ -818,6 +979,8 @@
self._find_libraries_with_dyld()
elif sys.platform == "win32":
self._find_libraries_with_enum_process_module_ex()
+ elif "pyodide" in sys.modules:
+ self._find_libraries_pyodide()
else:
self._find_libraries_with_dl_iterate_phdr()
@@ -833,6 +996,10 @@
"""
libc = self._get_libc()
if not hasattr(libc, "dl_iterate_phdr"): # pragma: no cover
+ warnings.warn(
+ "Could not find dl_iterate_phdr in the C standard library.",
+ RuntimeWarning,
+ )
return []
# Callback function for `dl_iterate_phdr` which is called for every
@@ -865,6 +1032,10 @@
"""
libc = self._get_libc()
if not hasattr(libc, "_dyld_image_count"): # pragma: no cover
+ warnings.warn(
+ "Could not find _dyld_image_count in the C standard library.",
+ RuntimeWarning,
+ )
return []
n_dyld = libc._dyld_image_count()
@@ -939,6 +1110,33 @@
finally:
kernel_32.CloseHandle(h_process)
+ def _find_libraries_pyodide(self):
+ """Pyodide specific implementation for finding loaded libraries.
+
+ Adapted from suggestion in
https://github.com/joblib/threadpoolctl/pull/169#issuecomment-1946696449.
+
+ One day, we may have a simpler solution. libc dl_iterate_phdr needs to
+ be implemented in Emscripten and exposed in Pyodide, see
+ https://github.com/emscripten-core/emscripten/issues/21354 for more
+ details.
+ """
+ try:
+ from pyodide_js._module import LDSO
+ except ImportError:
+ warnings.warn(
+ "Unable to import LDSO from pyodide_js._module. This should
never "
+ "happen."
+ )
+ return
+
+ for filepath in LDSO.loadedLibsByName.as_object_map():
+ # Some libraries are duplicated by Pyodide and do not exist in the
+ # filesystem, so we first check for the existence of the file. For
+ # more details, see
+ #
https://github.com/joblib/threadpoolctl/pull/169#issuecomment-1947946728
+ if os.path.exists(filepath):
+ self._make_controller_from_path(filepath)
+
def _make_controller_from_path(self, filepath):
"""Store a library controller if it is supported and selected"""
# Required to resolve symlinks
@@ -978,11 +1176,24 @@
# duplicate entry in threadpool_info.
continue
- # filename matches a prefix. Create and store the library
+ # filename matches a prefix. Now we check if the library has the
symbols we
+ # are looking for. If none of the symbols exists, it's very likely
not the
+ # expected library (e.g. a library having a common prefix with one
of the
+ # our supported libraries). Otherwise, create and store the library
# controller.
+ lib_controller = controller_class(
+ filepath=filepath, prefix=prefix, parent=self
+ )
- lib_controller = controller_class(filepath=filepath, prefix=prefix)
- self.lib_controllers.append(lib_controller)
+ if filepath in (lib.filepath for lib in self.lib_controllers):
+ # We already have a controller for this library.
+ continue
+
+ if not hasattr(controller_class, "check_symbols") or any(
+ hasattr(lib_controller.dynlib, func)
+ for func in controller_class.check_symbols
+ ):
+ self.lib_controllers.append(lib_controller)
def _check_prefix(self, library_basename, filename_prefixes):
"""Return the prefix library_basename starts with
@@ -997,7 +1208,8 @@
def _warn_if_incompatible_openmp(self):
"""Raise a warning if llvm-OpenMP and intel-OpenMP are both loaded"""
prefixes = [lib_controller.prefix for lib_controller in
self.lib_controllers]
- msg = textwrap.dedent("""
+ msg = textwrap.dedent(
+ """
Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at
the same time. Both libraries are known to be incompatible and this
can cause random crashes or deadlocks on Linux when loaded in the
@@ -1005,7 +1217,8 @@
Using threadpoolctl may cause crashes or deadlocks. For more
information and possible workarounds, please see
https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md
- """)
+ """
+ )
if "libomp" in prefixes and "libiomp" in prefixes:
warnings.warn(msg, RuntimeWarning)
@@ -1014,16 +1227,13 @@
"""Load the lib-C for unix systems."""
libc = cls._system_libraries.get("libc")
if libc is None:
- libc_name = find_library("c")
- if libc_name is None: # pragma: no cover
- warnings.warn(
- "libc not found. The ctypes module in Python"
- f" {sys.version_info.major}.{sys.version_info.minor} is
maybe"
- " too old for this OS.",
- RuntimeWarning,
- )
- return None
- libc = ctypes.CDLL(libc_name, mode=_RTLD_NOLOAD)
+ # Remark: If libc is statically linked or if Python is linked
against an
+ # alternative implementation of libc like musl, find_library will
return
+ # None and CDLL will load the main program itself which should
contain the
+ # libc symbols. We still name it libc for convenience.
+ # If the main program does not contain the libc symbols, it's ok
because
+ # we check their presence later anyway.
+ libc = ctypes.CDLL(find_library("c"), mode=_RTLD_NOLOAD)
cls._system_libraries["libc"] = libc
return libc
