commit:     00354a2443c1a0f7f64d55daadd52324868b085f
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 16 06:06:41 2019 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sat Feb 16 06:06:41 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=00354a24

cpuinfo: use better available CPU calculation

The existing portage.util.cpuinfo.get_cpu_count() behavior is wrong when
run in any environment where the cpuset is a subset of online CPUs.

The solution recommended by the 'os.cpu_count()' help is to use:
 len(os.sched_getaffinity(0))

This only works on line, so keep multiprocessing.cpu_count() as a
fallback. In newer version of Python, multiprocessing.cpu_count() is a
wrapper for os.cpu_count().

Reported-By: Daniel Robbins <drobbins <AT> funtoo.org>
Fixes: https://bugs.funtoo.org/browse/FL-6227
Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 lib/portage/util/cpuinfo.py | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/portage/util/cpuinfo.py b/lib/portage/util/cpuinfo.py
index 669e707b5..9ab1c119d 100644
--- a/lib/portage/util/cpuinfo.py
+++ b/lib/portage/util/cpuinfo.py
@@ -1,15 +1,44 @@
-# Copyright 2015 Gentoo Foundation
+# Copyright 2015-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ['get_cpu_count']
 
+# Before you set out to change this function, figure out what you're really
+# asking:
+#
+# - How many CPUs exist in this system (e.g. that the kernel is aware of?)
+#   This is 'getconf _NPROCESSORS_CONF' / get_nprocs_conf(3)
+#   In modern Linux, implemented by counting CPUs in /sys/devices/system/cpu/
+#
+# - How many CPUs in this system are ONLINE right now?
+#   This is 'getconf _NPROCESSORS_ONLN' / get_nprocs(3)
+#   In modern Linux, implemented by parsing /sys/devices/system/cpu/online
+#
+# - How many CPUs are available to this program?
+#   This is 'nproc' / sched_getaffinity(2), which is implemented in modern
+#   Linux kernels by querying the kernel scheduler; This might not be available
+#   in some non-Linux systems!
+#
+# - How many CPUs are available to this thread?
+#   This is pthread_getaffinity_np(3)
+#
+# As a further warning, the results returned by this function can differ
+# between runs, if altered by the scheduler or other external factors.
 
 def get_cpu_count():
        """
-       Try to obtain the number of CPUs available.
+       Try to obtain the number of CPUs available to this process.
 
        @return: Number of CPUs or None if unable to obtain.
        """
+        try:
+               import os
+               # This was introduced in Python 3.3 only, but exists in Linux
+               # all the way back to the 2.5.8 kernel.
+               # This NOT available in FreeBSD!
+               return len(os.sched_getaffinity(0))
+       except (ImportError, NotImplementedError, AttributeError):
+               pass
 
        try:
                import multiprocessing

Reply via email to