Git commit 654e1392a90777c3bb53e156cf26098dc128e3a1 by Michael Pyne. Committed on 12/07/2020 at 00:08. Pushed by mpyne into branch 'auto-detect-nr-cpus'.
rc-file: Add "num-cores-low-mem" and move num-cores* defs to initial-setup. It is annoying to be maintaining kdesrc-build-setup and --initial-setup but combining those two can be a subsequent refactoring. This adds a separate num-cores-low-mem to address the qtwebengine case and makes both num-cores and num-cores-low-mem into an option generated during initial setup, and only used by the default config (rather than any part of the kdesrc-build internals directly). There is a fallback reference in the code in case there is a usage of the default configuration file sections (e.g. qt5-build-include) but this is set conservatively (4 cores, 2 cores during low-mem). At this point it's almost "just" a configuration convention with a bit of code in the setup wizard so perhaps it's best not to touch the rest of the code/docs at all, but I'm happy with where this is at. I've tested --initial-setup in a Docker container (Fedora 29 with perl manually installed first) and tested kdesrc-build-setup separately. M +27 -1 doc/index.docbook M +32 -0 kdesrc-build-setup M +9 -0 kdesrc-buildrc-kf5-sample M +5 -4 modules/ksb/BuildContext.pm M +45 -4 modules/ksb/FirstRun.pm M +9 -6 qt5-build-include M +2 -1 vim/syntax/kdesrc-buildrc.vim https://invent.kde.org/sdk/kdesrc-build/commit/654e1392a90777c3bb53e156cf26098dc128e3a1 diff --git a/doc/index.docbook b/doc/index.docbook index df3c703..dbfacc9 100644 --- a/doc/index.docbook +++ b/doc/index.docbook @@ -2628,7 +2628,8 @@ due to fixes in the underlying build system.</entry> <entry>num-cores</entry> <entry>Cannot be overridden</entry> <entry> -<para>This option is automatically set by &kdesrc-build; to the number of +<para>This option is defined by &kdesrc-build; (when using the kdesrc-build-setup tool +or <command>kdesrc-build --initial-setup</command>), set to be the number of available CPUs (as indicated by the external application <application>nproc</application>). If &kdesrc-build; cannot detect the number of CPUs, this value is set to 4.</para> @@ -2638,6 +2639,31 @@ option's usage. This option was added in version 20.07.</para> </entry> </row> +<row id="conf-num-cores-low-mem"> +<entry>num-cores-low-mem</entry> +<entry>Cannot be overridden</entry> +<entry> +<para>This option is defined by &kdesrc-build; (when using the kdesrc-build-setup tool +or <command>kdesrc-build --initial-setup</command>), set to be the number of +CPUs that is deemed safe for heavyweight or other highly-intensive modules, +such as <literal>qtwebengine</literal>, to avoid running out of memory +during the build.</para> + +<para>The typical calculation is one CPU core for every 2 +gigabytes (GiB) of total memory. At least 1 core will be specified, +and no more than <option><link linkend="conf-num-cores">num-cores</link></option> +cores will be specified.</para> + +<para>Although this option is intended to support &Qt; modules, you can use it for your +any module in the same way that <option>num-cores</option> is used.</para> + +<para>If &kdesrc-build; cannot detect available memory then this value will be +set to 2.</para> + +<para>This option was added in version 20.07.</para> +</entry> +</row> + <row id="conf-override-build-system"> <entry>override-build-system</entry> <entry>Module setting overrides global</entry> diff --git a/kdesrc-build-setup b/kdesrc-build-setup index 6d2fd47..7886a64 100755 --- a/kdesrc-build-setup +++ b/kdesrc-build-setup @@ -26,6 +26,7 @@ use File::Copy; use File::Temp qw/tempfile/; use File::Basename; use Cwd qw(abs_path); +use List::Util qw(max min first); our $VERSION = 0.03; # Not user-visible yet. @@ -348,6 +349,25 @@ if (grep /^qt5$/, @chosenModules) { EOF } +chomp(my $num_cores = `nproc`); +$num_cores ||= 4; + +# Try to detect the amount of total memory for a corresponding option for +# heavyweight modules (sorry ade, not sure what's needed for FreeBSD!) +my $mem_total; +my $total_mem_line = first { /MemTotal/ } (`cat /proc/meminfo`); + +if ($total_mem_line && $? == 0) { + ($mem_total) = ($total_mem_line =~ /^MemTotal:\s*([0-9]+) /); # Value in KiB + $mem_total = int $mem_total; +} + +# 4 GiB is assumed if no info on memory is available, as this will +# calculate to 2 cores. sprintf is used since there's no Perl round function +my $rounded_mem = $mem_total ? (int sprintf("%.0f", $mem_total / 1024000.0)) : 4; +my $max_cores_for_mem = max(1, int $rounded_mem / 2); # Assume 2 GiB per core +my $num_cores_low = min($max_cores_for_mem, $num_cores); + print $output <<EOF; # Finds and includes *KDE*-based dependencies into the build. This makes @@ -366,6 +386,18 @@ print $output <<EOF; # relative to source-dir by default build-dir $buildDir + ## kdesrc-build sets 2 options which you can use in options like make-options or set-env + # to help manage the number of compile jobs that # happen during a build: + # + # 1. num-cores, which is just the number of detected CPU cores, and can be passed + # to tools like make (needed for parallel build) or ninja (completely optional). + # + # 2. num-cores-low-mem, which is set to largest value that appears safe for + # particularly heavyweight modules based on total memory, intended for + # modules like qtwebengine + num-cores $num_cores + num-cores-low-mem $num_cores_low + # Use multiple cores for building. Other options to GNU make may also be # set. make-options -j \${num-cores} diff --git a/kdesrc-buildrc-kf5-sample b/kdesrc-buildrc-kf5-sample index 136a992..60032a5 100644 --- a/kdesrc-buildrc-kf5-sample +++ b/kdesrc-buildrc-kf5-sample @@ -14,6 +14,15 @@ global # logs will be kept under this directory as well. source-dir ~/kde/src + # These values should be set to the number of cores to use during build (if + # in doubt, run "nproc" to see how many cores you have) + num-cores 4 + + # This is the same as above but used for heavyweight modules like + # qtwebengine, though you can use it for modules yourself. A conservative + # thumbrule is one core for every 2 GiB of total memory. + num-cores-low-mem 2 + make-options -j ${num-cores} end global diff --git a/modules/ksb/BuildContext.pm b/modules/ksb/BuildContext.pm index 6a5179e..25f25c3 100644 --- a/modules/ksb/BuildContext.pm +++ b/modules/ksb/BuildContext.pm @@ -100,7 +100,9 @@ our %defaultGlobalOptions = ( "branch" => "", "branch-group" => "", # Overrides branch, uses JSON data. "build-dir" => "build", + "cmake-generator" => "", "cmake-options" => "", + "cmake-toolchain" => "", "configure-flags" => "", "custom-build-command" => '', "cxxflags" => "-pipe", @@ -114,6 +116,9 @@ our %defaultGlobalOptions = ( "make-install-prefix" => "", # Some people need sudo "make-options" => "", "module-base-path" => "", # Used for tags and branches + "ninja-options" => "", + "num-cores" => 4, # Used only in rc-file but documented + "num-cores-low-mem" => 2, # Used only in rc-file but documented "override-build-system"=> "", "override-url" => "", "persistent-data-file" => "", @@ -172,10 +177,6 @@ sub new assert_isa($self, 'ksb::Module'); assert_isa($self, 'ksb::BuildContext'); - # Make the number of CPUs available to the rc-file by turning it into a pre-set option - my $nproc = int (eval { (filter_program_output(undef, 'nproc'))[0] } // 3) + 1; - $self->setOption('num-cores', $nproc); - return $self; } diff --git a/modules/ksb/FirstRun.pm b/modules/ksb/FirstRun.pm index c631efb..24d3b5a 100644 --- a/modules/ksb/FirstRun.pm +++ b/modules/ksb/FirstRun.pm @@ -4,6 +4,7 @@ use 5.014; use strict; use warnings; use File::Spec qw(splitpath); +use List::Util qw(min max first); use ksb::BuildException; use ksb::Debug qw(colorize); @@ -126,6 +127,31 @@ DONE } } +# Return the highest number of cores we can use based on available memory. Requires +# the number of cores we have available +sub _getNumCoresForLowMemory +{ + my $num_cores = shift; + + # Try to detect the amount of total memory for a corresponding option for + # heavyweight modules (sorry ade, not sure what's needed for FreeBSD!) + my $mem_total; + my $total_mem_line = first { /MemTotal/ } (`cat /proc/meminfo`); + + if ($total_mem_line && $? == 0) { + ($mem_total) = ($total_mem_line =~ /^MemTotal:\s*([0-9]+) /); # Value in KiB + $mem_total = int $mem_total; + } + + # 4 GiB is assumed if no info on memory is available, as this will + # calculate to 2 cores. sprintf is used since there's no Perl round function + my $rounded_mem = $mem_total ? (int sprintf("%.0f", $mem_total / 1024000.0)) : 4; + my $max_cores_for_mem = max(1, int $rounded_mem / 2); # Assume 2 GiB per core + my $num_cores_low = min($max_cores_for_mem, $num_cores); + + return $num_cores_low; +} + sub _setupBaseConfiguration { my $baseDir = shift; @@ -142,6 +168,11 @@ DONE my $sampleRc = $packages{'sample-rc'} or _throw("Embedded sample file missing!"); + my $numCores = `nproc 2>/dev/null` || 4; + my $numCoresLow = _getNumCoresForLowMemory($numCores); + + $sampleRc =~ s/%\{num_cores}/$numCores/g; + $sampleRc =~ s/%\{num_cores_low}/$numCoresLow/g; $sampleRc =~ s/%\{base_dir}/$baseDir/g; open my $sampleFh, '>', "$ENV{HOME}/.kdesrc-buildrc" @@ -527,6 +558,19 @@ global include-dependencies true cmake-options -DCMAKE_BUILD_TYPE=RelWithDebInfo + + # kdesrc-build sets 2 options which you can use in options like make-options or set-env + # to help manage the number of compile jobs that # happen during a build: + # + # 1. num-cores, which is just the number of detected CPU cores, and can be passed + # to tools like make (needed for parallel build) or ninja (completely optional). + # + # 2. num-cores-low-mem, which is set to largest value that appears safe for + # particularly heavyweight modules based on total memory, intended for + # modules like qtwebengine + num-cores %{num_cores} + num-cores-low-mem %{num_cores_low} + make-options -j ${num-cores} end global @@ -546,7 +590,4 @@ include %{base_dir}/custom-qt5-libs-build-include include %{base_dir}/kf5-qt5-build-include # To change options for modules that have already been defined, use an -# 'options' block -options kcoreaddons - make-options -j4 -end options +# 'options' block. See qt5-build-include for an example diff --git a/qt5-build-include b/qt5-build-include index 28ea98f..dc4c3d6 100644 --- a/qt5-build-include +++ b/qt5-build-include @@ -10,6 +10,8 @@ module-set qt5-set qttools qtwayland qtwebchannel qtwebsockets qtwebview qtx11extras \ qtnetworkauth qtspeech qtxmlpatterns + # qtwebengine is very different to the rest of Qt. You can try ignoring it if + # you cannot get it to compile by uncommenting the next line. # ignore-modules qtwebengine # install path. This *MUST* match your qtdir setting in kdesrc-buildrc! @@ -23,11 +25,12 @@ module-set qt5-set end module-set # qtwebengine is essentially the Chromium Embedded Framework with Qt bindings -# and has source code of unusually large complexity for the compiler. -# TL;DR: This will eat a *ton* of RAM and can lockup your system if you have a -# lot of CPU cores. qtwebengine is disabled by default but if you enable it -# also ensure you don't outstrip your available RAM with too high of a -# parallelism (-j flag). +# It has unusually complex source codes which require a lot of memory to compile.. +# +# This module will eat a *ton* of RAM and can lockup your system if you have a +# lot of CPU cores. qtwebengine uses a lower number of cores to compile by +# default but you can change the setting to -j here (or in your global options) +# to change that. options qtwebengine - set-env NINJAFLAGS -j4 + set-env NINJAFLAGS -j${num-cores-low-mem} end options diff --git a/vim/syntax/kdesrc-buildrc.vim b/vim/syntax/kdesrc-buildrc.vim index b55fdcc..26606fa 100644 --- a/vim/syntax/kdesrc-buildrc.vim +++ b/vim/syntax/kdesrc-buildrc.vim @@ -39,7 +39,7 @@ syn keyword ksbrcOption contained skipwhite nextgroup=ksbrcStringValue \ binpath branch build-dir checkout-only cmake-options configure-flags \ custom-build-command cxxflags dest-dir do-not-compile kdedir \ libpath log-dir make-install-prefix make-options module-base-path - \ ninja-options + \ cmake-generator cmake-toolchain ninja-options \ override-build-system override-url prefix qtdir repository \ revision source-dir svn-server tag remove-after-install \ qmake-options git-user @@ -47,6 +47,7 @@ syn keyword ksbrcOption contained skipwhite nextgroup=ksbrcStringValue syn keyword ksbrcGlobalOption contained skipwhite nextgroup=ksbrcStringValue \ branch-group git-desired-protocol git-repository-base http-proxy \ kde-languages niceness debug-level persistent-data-file set-env + \ num-cores num-cores-low-mem " MUST BE CONSISTENT WITH ABOVE. Used when a module-set option is used in the " wrong spot to highlight the error.