commit: 00354a2443c1a0f7f64d55daadd52324868b085f Author: Robin H. Johnson <robbat2 <AT> gentoo <DOT> org> AuthorDate: Sat Feb 16 06:06:41 2019 +0000 Commit: Robin H. Johnson <robbat2 <AT> gentoo <DOT> org> CommitDate: Sat Feb 16 06:06:41 2019 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=00354a24
cpuinfo: use better available CPU calculation The existing portage.util.cpuinfo.get_cpu_count() behavior is wrong when run in any environment where the cpuset is a subset of online CPUs. The solution recommended by the 'os.cpu_count()' help is to use: len(os.sched_getaffinity(0)) This only works on line, so keep multiprocessing.cpu_count() as a fallback. In newer version of Python, multiprocessing.cpu_count() is a wrapper for os.cpu_count(). Reported-By: Daniel Robbins <drobbins <AT> funtoo.org> Fixes: https://bugs.funtoo.org/browse/FL-6227 Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org> lib/portage/util/cpuinfo.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/cpuinfo.py b/lib/portage/util/cpuinfo.py index 669e707b5..9ab1c119d 100644 --- a/lib/portage/util/cpuinfo.py +++ b/lib/portage/util/cpuinfo.py @@ -1,15 +1,44 @@ -# Copyright 2015 Gentoo Foundation +# Copyright 2015-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ['get_cpu_count'] +# Before you set out to change this function, figure out what you're really +# asking: +# +# - How many CPUs exist in this system (e.g. that the kernel is aware of?) +# This is 'getconf _NPROCESSORS_CONF' / get_nprocs_conf(3) +# In modern Linux, implemented by counting CPUs in /sys/devices/system/cpu/ +# +# - How many CPUs in this system are ONLINE right now? +# This is 'getconf _NPROCESSORS_ONLN' / get_nprocs(3) +# In modern Linux, implemented by parsing /sys/devices/system/cpu/online +# +# - How many CPUs are available to this program? +# This is 'nproc' / sched_getaffinity(2), which is implemented in modern +# Linux kernels by querying the kernel scheduler; This might not be available +# in some non-Linux systems! +# +# - How many CPUs are available to this thread? +# This is pthread_getaffinity_np(3) +# +# As a further warning, the results returned by this function can differ +# between runs, if altered by the scheduler or other external factors. def get_cpu_count(): """ - Try to obtain the number of CPUs available. + Try to obtain the number of CPUs available to this process. @return: Number of CPUs or None if unable to obtain. """ + try: + import os + # This was introduced in Python 3.3 only, but exists in Linux + # all the way back to the 2.5.8 kernel. + # This NOT available in FreeBSD! + return len(os.sched_getaffinity(0)) + except (ImportError, NotImplementedError, AttributeError): + pass try: import multiprocessing
