https://github.com/ldionne updated https://github.com/llvm/llvm-project/pull/224670
>From fbe253fe67aa25d1bbbacf8131d3135579b7dfad Mon Sep 17 00:00:00 2001 From: Louis Dionne <[email protected]> Date: Thu, 17 Sep 2026 14:56:08 -0700 Subject: [PATCH 1/7] [cherry-pick #224192] [libc++] Build GoogleBenchmark directly from Lit --- libcxx/test/CMakeLists.txt | 1 - libcxx/test/benchmarks/CMakeLists.txt | 48 ---- libcxx/test/benchmarks/lit.local.cfg | 11 + .../test/configs/harness-configuration.cfg.in | 1 - libcxx/utils/libcxx/test/config.py | 2 +- libcxx/utils/libcxx/test/format.py | 13 +- libcxx/utils/libcxx/test/googlebenchmark.py | 222 ++++++++++++++++++ libcxxabi/test/configs/cmake-bridge.cfg.in | 1 - libunwind/test/configs/cmake-bridge.cfg.in | 1 - 9 files changed, 242 insertions(+), 58 deletions(-) delete mode 100644 libcxx/test/benchmarks/CMakeLists.txt create mode 100644 libcxx/test/benchmarks/lit.local.cfg create mode 100644 libcxx/utils/libcxx/test/googlebenchmark.py diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt index 8fd34c086a992..85fa6aacc7796 100644 --- a/libcxx/test/CMakeLists.txt +++ b/libcxx/test/CMakeLists.txt @@ -26,7 +26,6 @@ set(SERIALIZED_LIT_PARAMS "# Lit parameters serialized here for llvm-lit to pick serialize_lit_string_param(SERIALIZED_LIT_PARAMS compiler "${CMAKE_CXX_COMPILER}") if (LIBCXX_INCLUDE_BENCHMARKS) - add_subdirectory(benchmarks) set(_libcxx_benchmark_mode "dry-run") else() serialize_lit_string_param(SERIALIZED_LIT_PARAMS enable_benchmarks "no") diff --git a/libcxx/test/benchmarks/CMakeLists.txt b/libcxx/test/benchmarks/CMakeLists.txt deleted file mode 100644 index b3f881a008b80..0000000000000 --- a/libcxx/test/benchmarks/CMakeLists.txt +++ /dev/null @@ -1,48 +0,0 @@ -#============================================================================== -# Build Google Benchmark -#============================================================================== - -include(ExternalProject) -set(BENCHMARK_COMPILE_FLAGS - -Wno-unused-command-line-argument - -nostdinc++ - -isystem "${LIBCXX_GENERATED_INCLUDE_DIR}" - -L${LIBCXX_LIBRARY_DIR} - -Wl,-rpath,${LIBCXX_LIBRARY_DIR} - ${SANITIZER_FLAGS} - ) -if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR) - list(APPEND BENCHMARK_COMPILE_FLAGS - -isystem "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}") -endif() -if (DEFINED LIBCXX_CXX_ABI_LIBRARY_PATH) - list(APPEND BENCHMARK_COMPILE_FLAGS - -L${LIBCXX_CXX_ABI_LIBRARY_PATH} - -Wl,-rpath,${LIBCXX_CXX_ABI_LIBRARY_PATH}) -endif() -split_list(BENCHMARK_COMPILE_FLAGS) - -set(BENCHMARK_CXX_LIBRARIES) -list(APPEND BENCHMARK_CXX_LIBRARIES c++) -if (NOT LIBCXX_ENABLE_SHARED) - list(APPEND BENCHMARK_CXX_LIBRARIES c++abi) -endif() - -ExternalProject_Add(google-benchmark - EXCLUDE_FROM_ALL ON - DEPENDS cxx cxx-headers - PREFIX google-benchmark - SOURCE_DIR ${LLVM_THIRD_PARTY_DIR}/benchmark - INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/google-benchmark - CMAKE_CACHE_ARGS - -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER} - -DCMAKE_MAKE_PROGRAM:FILEPATH=${CMAKE_MAKE_PROGRAM} - -DCMAKE_BUILD_TYPE:STRING=RELEASE - -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR> - -DCMAKE_CXX_FLAGS:STRING=${BENCHMARK_COMPILE_FLAGS} - -DBENCHMARK_USE_LIBCXX:BOOL=ON - -DBENCHMARK_ENABLE_TESTING:BOOL=OFF - -DBENCHMARK_CXX_LIBRARIES:STRING=${BENCHMARK_CXX_LIBRARIES}) - -add_dependencies(cxx-test-depends google-benchmark) diff --git a/libcxx/test/benchmarks/lit.local.cfg b/libcxx/test/benchmarks/lit.local.cfg new file mode 100644 index 0000000000000..07a15e4ae5df9 --- /dev/null +++ b/libcxx/test/benchmarks/lit.local.cfg @@ -0,0 +1,11 @@ +# Build the GoogleBenchmark library using the current Lit configuration so +# that benchmarks can link against it. + +import libcxx.test.googlebenchmark + +if "enable-benchmarks=no" in config.available_features: + config.substitutions.append(("%{benchmark_flags}", "")) +else: + config.substitutions.append( + ("%{benchmark_flags}", libcxx.test.googlebenchmark.prepare(config, lit_config)) + ) diff --git a/libcxx/test/configs/harness-configuration.cfg.in b/libcxx/test/configs/harness-configuration.cfg.in index c527d60c64af5..fab0ee6925501 100644 --- a/libcxx/test/configs/harness-configuration.cfg.in +++ b/libcxx/test/configs/harness-configuration.cfg.in @@ -26,5 +26,4 @@ config.test_exec_root = os.path.join('@LIBCXX_BINARY_DIR@', 'test') # Add substitutions for bootstrapping the test suite configuration config.substitutions.append(('%{libcxx-dir}', '@LIBCXX_SOURCE_DIR@')) -config.substitutions.append(('%{benchmark_flags}', '-I @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/include -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib64 -l benchmark')) config.substitutions.append(("%{python}", shlex.quote(sys.executable))) diff --git a/libcxx/utils/libcxx/test/config.py b/libcxx/utils/libcxx/test/config.py index fd6f1bf5accd6..c71e81cf9573a 100644 --- a/libcxx/utils/libcxx/test/config.py +++ b/libcxx/utils/libcxx/test/config.py @@ -51,7 +51,7 @@ def configure(parameters, features, config, lit_config): ) # Print the basic substitutions - for sub in ("%{cxx}", "%{flags}", "%{compile_flags}", "%{link_flags}", "%{benchmark_flags}", "%{exec}"): + for sub in ("%{cxx}", "%{flags}", "%{compile_flags}", "%{link_flags}", "%{exec}"): debug("Using {} substitution: '{}'".format(sub, _getSubstitution(sub, config.substitutions))) # Print all available features diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py index 2f449e234e6d5..3e24e435e7c07 100644 --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -31,7 +31,7 @@ def _getTempPaths(test): def _checkBaseSubstitutions(substitutions): substitutions = [s for (s, _) in substitutions] - for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{benchmark_flags}", "%{flags}", "%{exec}"]: + for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]: assert s in substitutions, "Required substitution {} was not provided".format(s) def _executeScriptInternal(test, litConfig, commands): @@ -235,10 +235,6 @@ class CxxStandardLibraryTest(lit.formats.FileBasedTest): %{compile_flags} - Flags to use when compiling a test case %{link_flags} - Flags to use when linking a test case %{flags} - Flags to use either when compiling or linking a test case - %{benchmark_flags} - Flags to use when compiling benchmarks. These flags should provide access to - GoogleBenchmark but shouldn't hardcode any optimization level or other settings, - since the benchmarks should be run under the same configuration as the rest of - the test suite. %{exec} - A command to prefix the execution of executables Note that when building an executable (as opposed to only compiling a source @@ -359,6 +355,13 @@ def execute(self, test, litConfig): test.getFullName() ), ) + substitutions = [s for (s, _) in test.config.substitutions] + if "%{benchmark_flags}" not in substitutions: + return lit.Test.Result( + lit.Test.UNRESOLVED, + "Test {} is a benchmark, but the %{{benchmark_flags}} substitution " + "isn't provided by the configuration.".format(test.getFullName()), + ) steps = [ "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{benchmark_flags} %{link_flags} -o %t.exe", ] diff --git a/libcxx/utils/libcxx/test/googlebenchmark.py b/libcxx/utils/libcxx/test/googlebenchmark.py new file mode 100644 index 0000000000000..fb994ac19d058 --- /dev/null +++ b/libcxx/utils/libcxx/test/googlebenchmark.py @@ -0,0 +1,222 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +""" +Support for building GoogleBenchmark from the Lit configuration. + +The benchmarks in the test suite are linked against GoogleBenchmark, which must be +built with the same Standard Library (and more generally with the same ABI-affecting +flags) as the benchmarks themselves. This allows building GoogleBenchmark on-demand +from Lit using the flags of the configuration being tested. + +The result is cached inside the build directory so that subsequent invocations are +cheap. +""" + +import hashlib +import os +import shlex +import subprocess + +import lit.TestRunner + +import libcxx.test.config +import libcxx.test.dsl + +THIS_FILE = os.path.abspath(__file__) +LIBCXX_UTILS = os.path.dirname(os.path.dirname(os.path.dirname(THIS_FILE))) +MONOREPO_ROOT = os.path.dirname(os.path.dirname(LIBCXX_UTILS)) +SOURCE_DIR = os.path.join(MONOREPO_ROOT, "third-party", "benchmark") + +# Flags used by the test suite that must not be used when building GoogleBenchmark. +# Anything that isn't listed here is forwarded verbatim. +# +# -Werror +# Avoid failing GoogleBenchmark's build due to warnings. +# -D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER +# Only relevant when testing libc++ itself. +# -fmodules, -fcxx-modules, -fmodules-cache-path= +# Modules are irrelevant when building a third-party static library, and sharing a +# module cache with the test suite is undesirable. +# -std= +# GoogleBenchmark sets CMAKE_CXX_STANDARD itself and requires C++17. +_DROPPED_FLAGS = { + "-Werror", + "-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER", + "-fmodules", + "-fcxx-modules", +} + +_DROPPED_FLAG_PREFIXES = ( + "-std=", + "-fmodules-cache-path=", +) + +# Flags that are dropped along with the '-Xclang' that introduces them. +_DROPPED_XCLANG_FLAGS = { + "-fmodules-local-submodule-visibility", +} + + +def _expand(config, string): + """ + Expand the Lit substitutions in the given string, recursively. + """ + (expanded,) = lit.TestRunner.applySubstitutions( + [string], + config.substitutions, + recursion_limit=config.recursiveExpansionLimit, + ) + return expanded + + +def _filterFlags(flags): + """ + Remove the flags that must not be used when building GoogleBenchmark. + """ + result = [] + flags = iter(flags) + for flag in flags: + if flag == "-Xclang": + arg = next(flags, None) + if arg is None: + result.append(flag) + elif arg not in _DROPPED_XCLANG_FLAGS: + result += [flag, arg] + elif flag not in _DROPPED_FLAGS and not flag.startswith(_DROPPED_FLAG_PREFIXES): + result.append(flag) + return result + + +def _getFlags(config, substitutions): + """ + Return the flags contained in the given substitutions, based on the flags used by the + configuration under test. + """ + flags = [] + for substitution in substitutions: + expanded = _expand(config, _getSubstitution(substitution, config)) + flags += shlex.split(expanded) + return _filterFlags(flags) + + +def _splitLibraries(flags): + """ + Split the given link flags into (flags, libraries), where libraries contains the + name of the libraries that were being linked against. + + We can't simply hand the libraries over to CMake as part of CMAKE_CXX_FLAGS, since + CMake puts those flags before the object files on the link line. + """ + result = [] + libraries = [] + flags = iter(flags) + for flag in flags: + if flag == "-l": + library = next(flags, None) + if library is None: + result.append(flag) + else: + libraries.append(library) + elif flag.startswith("-l"): + libraries.append(flag[len("-l") :]) + else: + result.append(flag) + return (result, libraries) + + +def _getSubstitution(substitution, config): + return libcxx.test.config._getSubstitution(substitution, config.substitutions) + + +def _fingerprint(config, flags, libraries): + """ + Return an opaque value identifying this GoogleBenchmark build. + + This changes whenever the compiler is rebuilt or whenever the flags used to build + GoogleBenchmark change. + """ + compiler = libcxx.test.dsl._compilerFingerprint(config) + return hashlib.sha256(repr((compiler, flags, libraries)).encode()).hexdigest()[:16] + + +def _run(litConfig, what, command, cwd): + result = subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + if result.returncode != 0: + pretty = " ".join(shlex.quote(arg) for arg in command) + litConfig.fatal( + "Failed to {} GoogleBenchmark.\n" + "Command was:\n{}\n\n" + "Output was:\n{}".format(what, pretty, result.stdout) + ) + + +def prepare(config, litConfig): + """ + Make GoogleBenchmark available to the test suite and return the flags required to + build the benchmarks against it. + + GoogleBenchmark is built using the same flags as the rest of the test suite, and the + result is cached inside the build directory. The cache is keyed on the flags being + used, so different Lit configurations do not interfere with each other. + """ + flags, libraries = _splitLibraries( + _getFlags(config, ("%{flags}", "%{compile_flags}", "%{link_flags}")) + ) + root = os.path.join(config.test_exec_root, "__gbench__") + prefix = os.path.join(root, _fingerprint(config, flags, libraries)) + buildDir = os.path.join(prefix, "build") + installDir = os.path.join(prefix, "install") + os.makedirs(root, exist_ok=True) + + cmake = os.environ.get("CMAKE", "cmake") + + if not os.path.exists(os.path.join(buildDir, "CMakeCache.txt")): + litConfig.note("Configuring GoogleBenchmark in {}".format(buildDir)) + compiler = _expand(config, _getSubstitution("%{cxx}", config)) + _run( + litConfig, + "configure", + [ + cmake, + "-S", + SOURCE_DIR, + "-B", + buildDir, + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_COMPILER={}".format(compiler), + "-DCMAKE_CXX_FLAGS={}".format(" ".join(flags)), + "-DCMAKE_INSTALL_PREFIX={}".format(installDir), + "-DCMAKE_INSTALL_LIBDIR=lib", + "-DBENCHMARK_CXX_LIBRARIES={}".format(";".join(libraries)), + "-DBENCHMARK_ENABLE_TESTING=OFF", + "-DBENCHMARK_ENABLE_WERROR=OFF", + "-DBENCHMARK_INSTALL_DOCS=OFF", + ], + cwd=root, + ) + + # Always build: GoogleBenchmark is compiled against the headers of the library + # under test, so it must be rebuilt when those change. This is a no-op when + # nothing changed. + _run( + litConfig, + "build", + [cmake, "--build", buildDir, "--target", "install", "--parallel"], + cwd=root, + ) + + include = os.path.join(installDir, "include") + lib = os.path.join(installDir, "lib") + return "-isystem {} -L {} -l benchmark".format(include, lib) diff --git a/libcxxabi/test/configs/cmake-bridge.cfg.in b/libcxxabi/test/configs/cmake-bridge.cfg.in index f81dd8afb1091..a2fa847f2d6c6 100644 --- a/libcxxabi/test/configs/cmake-bridge.cfg.in +++ b/libcxxabi/test/configs/cmake-bridge.cfg.in @@ -34,7 +34,6 @@ config.substitutions.append(('%{include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/i config.substitutions.append(('%{cxx-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_DIR@')) config.substitutions.append(('%{cxx-target-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR@')) config.substitutions.append(('%{lib}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_LIBRARY_DIR@')) -config.substitutions.append(('%{benchmark_flags}', '')) if @LIBCXXABI_USE_LLVM_UNWINDER@: config.substitutions.append(('%{maybe-include-libunwind}', '-I "@LIBCXXABI_LIBUNWIND_INCLUDES_INTERNAL@"')) diff --git a/libunwind/test/configs/cmake-bridge.cfg.in b/libunwind/test/configs/cmake-bridge.cfg.in index ed0b2afbc0c0d..98e7d8d653f5d 100644 --- a/libunwind/test/configs/cmake-bridge.cfg.in +++ b/libunwind/test/configs/cmake-bridge.cfg.in @@ -45,7 +45,6 @@ config.substitutions.append(('%{libcxx}', '@LIBUNWIND_LIBCXX_PATH@')) config.substitutions.append(('%{install-prefix}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@')) config.substitutions.append(('%{include}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@/include')) config.substitutions.append(('%{lib}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@/@LIBUNWIND_INSTALL_LIBRARY_DIR@')) -config.substitutions.append(('%{benchmark_flags}', '')) # Check for objcopy tools objcopy_path = which('llvm-objcopy', '@LLVM_BUILD_BINARY_DIR@/bin') >From c73d0ad34ab5f39e1b37f78a0eaafd8c5bf54959 Mon Sep 17 00:00:00 2001 From: Louis Dionne <[email protected]> Date: Fri, 18 Sep 2026 07:33:36 -0700 Subject: [PATCH 2/7] [libc++] Extract building libc++ into a composite action This allows reusing the same build script for the PR benchmark and the historical benchmarking jobs. This also opens the door to adding new configurations where the library isn't rebuilt (e.g. where the artifacts are pulled from a pre-built location, or even testing against non-libc++. Assisted by Claude --- .github/workflows/libcxx-benchmark-commit.yml | 20 ++++++- .github/workflows/libcxx-pr-benchmark.yml | 45 +++++++++++---- .../libcxx/build-at-commit/action.yml | 53 +++++++++++++++++ libcxx/utils/ci/lnt/README.md | 37 +++++++++--- libcxx/utils/ci/lnt/machines.json | 3 + libcxx/utils/ci/lnt/run-benchmarks | 57 +++++++++++-------- 6 files changed, 170 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/libcxx/build-at-commit/action.yml diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml index 040c0a7536dc4..a383921259aeb 100644 --- a/.github/workflows/libcxx-benchmark-commit.yml +++ b/.github/workflows/libcxx-benchmark-commit.yml @@ -107,6 +107,9 @@ jobs: runs-on: ${{ matrix.runner }} env: COMPILER: ${{ matrix.cxx }} + # Where the library is installed when we build it ourselves. This lives inside the workspace + # so that actions/checkout cleans it up between runs on self-hosted runners. + INSTALL_DIR: ${{ github.workspace }}/install steps: - name: Checkout the LLVM monorepo uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -140,6 +143,18 @@ jobs: "${COMPILER}" --version python3 --version + # A build failure is tolerated on purpose: the library doesn't build at every historical commit, + # and we still want to submit an (empty) run for such commits. `plan-benchmarks` counts runs + # rather than results, so that is what stops the cron from requesting a commit forever. + - name: Build libc++ at ${{ inputs.commit }} + continue-on-error: true + uses: ./.github/workflows/libcxx/build-at-commit + with: + commit: ${{ inputs.commit }} + install-dir: ${{ env.INSTALL_DIR }} + compiler: ${{ matrix.cxx }} + cmake-cache: ${{ matrix.cmake-cache }} + - name: Setup virtual environment run: | python3 -m venv .venv @@ -149,10 +164,10 @@ jobs: - name: Run the benchmarks env: BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }} - CMAKE_CACHE: ${{ matrix.cmake-cache }} COMMIT: ${{ inputs.commit }} FILTER: ${{ inputs.filter }} LNT_MACHINE: ${{ matrix.lnt-machine }} + TEST_CONFIG: ${{ matrix.test-config }} run: | source .venv/bin/activate filter_arg=() @@ -164,7 +179,8 @@ jobs: --machine "${LNT_MACHINE}" \ --compiler "${COMPILER}" \ --benchmark-commit "${COMMIT}" \ - --cmake-cache "${CMAKE_CACHE}" \ + --test-config "${TEST_CONFIG}" \ + --lit-param "libcxx_installation=${INSTALL_DIR}" \ "${filter_arg[@]}" \ --output "${COMMIT}.json" diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml index 9c31be387b4d1..40bd24bc6fbd1 100644 --- a/.github/workflows/libcxx-pr-benchmark.yml +++ b/.github/workflows/libcxx-pr-benchmark.yml @@ -110,11 +110,11 @@ jobs: - extract-info env: BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }} - CMAKE_CACHE: ${{ matrix.cmake-cache }} COMPILER: ${{ matrix.cxx }} LNT_MACHINE: ${{ matrix.lnt-machine }} PR_HEAD: ${{ needs.extract-info.outputs.pr_head }} PR_BASE: ${{ needs.extract-info.outputs.pr_base }} + TEST_CONFIG: ${{ matrix.test-config }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -153,23 +153,46 @@ jobs: source .venv/bin/activate pip install -r libcxx/utils/requirements.txt - - name: Build the baseline and the candidate - run: | - source .venv/bin/activate - baseline_commit=$(git merge-base $PR_BASE $PR_HEAD) - ./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_CXX_COMPILER="${COMPILER}" -C "${CMAKE_CACHE}" - ./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_CXX_COMPILER="${COMPILER}" -C "${CMAKE_CACHE}" + - name: Determine the baseline commit + id: baseline + run: echo "commit=$(git merge-base "${PR_BASE}" "${PR_HEAD}")" >> "${GITHUB_OUTPUT}" + + - name: Build the baseline + uses: ./.github/workflows/libcxx/build-at-commit + with: + commit: ${{ steps.baseline.outputs.commit }} + install-dir: ${{ github.workspace }}/install/baseline + compiler: ${{ matrix.cxx }} + cmake-cache: ${{ matrix.cmake-cache }} + + - name: Build the candidate + uses: ./.github/workflows/libcxx/build-at-commit + with: + commit: ${{ needs.extract-info.outputs.pr_head }} + install-dir: ${{ github.workspace }}/install/candidate + compiler: ${{ matrix.cxx }} + cmake-cache: ${{ matrix.cmake-cache }} - name: Run baseline and candidate interleaved run: | source .venv/bin/activate # Run 5 times so we can pick the median, and interleave baseline and candidate to mitigate the impact of # environmental noise + # + # TODO: The invocations below should actually become calls to run-benchmarks, which would simply include + # benchmark consolidation for _ in $(seq 1 5); do - ./libcxx/utils/test-at-commit --test-config "${PWD}/libcxx/test/configs/installed-libc++.cfg.in" -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/baseline" "$BENCHMARKS" - ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt - ./libcxx/utils/test-at-commit --test-config "${PWD}/libcxx/test/configs/installed-libc++.cfg.in" -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/candidate" "$BENCHMARKS" - ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt + for which in baseline candidate; do + ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" \ + -B "benchmarks/${which}" \ + --compiler "${COMPILER}" \ + -- \ + -sv -j1 \ + --param optimization=speed \ + --param libcxx_installation="${PWD}/install/${which}" \ + "${BENCHMARKS}" + ./libcxx/utils/consolidate-benchmarks "benchmarks/${which}" | tee -a "${which}.lnt" + done done - name: Compare baseline and candidate runs diff --git a/.github/workflows/libcxx/build-at-commit/action.yml b/.github/workflows/libcxx/build-at-commit/action.yml new file mode 100644 index 0000000000000..1613cd4429548 --- /dev/null +++ b/.github/workflows/libcxx/build-at-commit/action.yml @@ -0,0 +1,53 @@ +# This composite action builds libc++ as of an arbitrary commit and installs it, so that the +# benchmark suite can then be run against that installation. +# +# It is shared by the workflow that benchmarks a single commit (libcxx-benchmark-commit.yml) +# and the one that compares a pull request against its merge base (libcxx-pr-benchmark.yml), +# which builds twice. +# +# Machines that benchmark a Standard Library we don't build ourselves (e.g. libstdc++) simply +# don't use this action: see the `build` key in libcxx/utils/ci/lnt/machines.json. +# +# This action assumes that the monorepo has been checked out with its full history (arbitrary +# historical commits must be reachable) and that CMake, Ninja and the compiler are available. + +name: 'Build libc++ at a commit' +description: 'Build and install libc++ as of the given commit, for benchmarking purposes.' + +inputs: + commit: + description: 'The libc++ commit to build.' + required: true + install-dir: + description: >- + Directory to install the library into. This is the directory that the benchmarks are then + run against, via the `libcxx_installation` Lit parameter. + required: true + compiler: + description: 'The C++ compiler to build the library with.' + required: true + cmake-cache: + description: >- + CMake cache describing how to build the library, as a path relative to the root of the + monorepo (e.g. libcxx/utils/ci/lnt/cmake/generic.cmake). + required: true + +runs: + using: composite + steps: + - name: Build libc++ at ${{ inputs.commit }} + shell: bash + env: + CMAKE_CACHE: ${{ inputs.cmake-cache }} + COMMIT: ${{ inputs.commit }} + COMPILER: ${{ inputs.compiler }} + INSTALL_DIR: ${{ inputs.install-dir }} + run: | + # Let build-at-commit use (and clean up) its own temporary source and build directories: + # only the installation is of interest to the caller. + libcxx/utils/build-at-commit \ + --commit "${COMMIT}" \ + --install-dir "${INSTALL_DIR}" \ + -- \ + -DCMAKE_CXX_COMPILER="${COMPILER}" \ + -C "${GITHUB_WORKSPACE}/${CMAKE_CACHE}" diff --git a/libcxx/utils/ci/lnt/README.md b/libcxx/utils/ci/lnt/README.md index b82908f077e42..3709d60fe1b18 100644 --- a/libcxx/utils/ci/lnt/README.md +++ b/libcxx/utils/ci/lnt/README.md @@ -63,25 +63,44 @@ it for each machine defined in `machines.json` on a schedule. ## Configuring the benchmark machines `machines.json` describes the machines we benchmark on. It is the single source of truth -for both the workflow that runs the benchmarks (`libcxx-benchmark-commit.yml`) and the cron -that requests those runs (`libcxx-benchmark-cron.yml`). Each entry contains variables used -by the various workflows and the LNT machine name that the results will be reported under. +for the workflow that runs the benchmarks (`libcxx-benchmark-commit.yml`), the cron that +requests those runs (`libcxx-benchmark-cron.yml`), and the workflow that benchmarks pull +requests (`libcxx-pr-benchmark.yml`). Each entry contains variables used by the various +workflows and the LNT machine name that the results will be reported under. + +Two keys determine what a machine actually benchmarks: + +- `cmake-cache` is the CMake cache describing how the library is built, which the + `.github/workflows/libcxx/build-at-commit` action is handed. +- `test-config` is the Lit testing configuration to run the benchmarks with, which is what + selects the Standard Library being measured. ## Running benchmarks locally On GitHub, the `libcxx-benchmark-commit.yml` workflow is used to run benchmarks and report results to a LNT instance. This workflow wraps the `libcxx/utils/ci/lnt/run-benchmarks` script, -which can be used to benchmark locally: +which can be used to benchmark locally. + +`run-benchmarks` doesn't build anything: it runs the benchmark suite against whatever Standard +Library the testing configuration points to, so the libc++ being benchmarked must be built first: ``` -run-benchmarks --test-suite-commit <SHA1> --machine <MACHINE> \ - --compiler clang++ --benchmark-commit <SHA2> \ +# Build the libc++ to benchmark +libcxx/utils/build-at-commit --commit <SHA2> --install-dir /tmp/libcxx-install \ + -- -DCMAKE_CXX_COMPILER=clang++ \ + -C libcxx/utils/ci/lnt/cmake/generic.cmake + +# Benchmark it +run-benchmarks --test-suite-commit <SHA1> --machine <MACHINE> \ + --compiler clang++ --benchmark-commit <SHA2> \ + --test-config libcxx/test/configs/installed-libc++.cfg.in \ + --lit-param libcxx_installation=/tmp/libcxx-install \ --output result.json ``` -This will run the benchmarks (using the test suite at the specified `SHA1`) against libc++ -as-of the specified `SHA2`, and produce a LNT-ready JSON report. The results can then be -submitted to a LNT instance if desired. +This will run the benchmarks (using the test suite at the specified `SHA1`) against the libc++ +built from `SHA2`, and produce a LNT-ready JSON report. The results can then be submitted to a +LNT instance if desired. ## Setting up a local LNT instance diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json index ca55a7fc3c241..cd0d633fc013e 100644 --- a/libcxx/utils/ci/lnt/machines.json +++ b/libcxx/utils/ci/lnt/machines.json @@ -6,6 +6,7 @@ "xcode-version": "26.5", "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0", "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake", + "test-config": "libcxx/test/configs/installed-libc++.cfg.in", "coverage": { "since": "2023-01-01", "every": "week", @@ -21,6 +22,7 @@ "xcode-version": "26.5", "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0", "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake", + "test-config": "libcxx/test/configs/installed-libc++.cfg.in", "coverage": { "since": "2023-12-01", "every": "week", @@ -35,6 +37,7 @@ "cxx": "clang++-22", "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0", "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake", + "test-config": "libcxx/test/configs/installed-libc++.cfg.in", "coverage": { "since": "2023-01-01", "every": "week", diff --git a/libcxx/utils/ci/lnt/run-benchmarks b/libcxx/utils/ci/lnt/run-benchmarks index 6365e2465892e..9104d8bcfa4cc 100755 --- a/libcxx/utils/ci/lnt/run-benchmarks +++ b/libcxx/utils/ci/lnt/run-benchmarks @@ -107,12 +107,28 @@ def dict_to_params(d): def main(argv): parser = argparse.ArgumentParser( prog='run-benchmarks', - description='Benchmark libc++ at the given commit and produce a LNT JSON report.', + description='Run the libc++ benchmark suite against a Standard Library and produce a LNT JSON report. ' + 'The library being benchmarked is selected by the Lit configuration passed to --test-config; ' + 'this script does not build anything, so any library it benchmarks must have been built or ' + 'installed beforehand.', epilog='This script depends on the modules listed in `libcxx/utils/ci/lnt/requirements.txt`.') parser.add_argument('--benchmark-commit', type=str, required=True, - help='The SHA representing the version of the library to benchmark.') + help='The SHA that the results are attributed to in LNT. When benchmarking libc++, this is the version ' + 'of the library being benchmarked. When benchmarking another Standard Library, that library is not ' + 'tied to a monorepo commit, so this merely places the results on the LNT time axis.') parser.add_argument('--test-suite-commit', type=str, required=True, help='The SHA representing the version of the test suite to use for benchmarking.') + parser.add_argument('--test-config', type=str, required=True, + help='The Lit testing configuration to use, which determines the Standard Library being benchmarked. ' + 'This is either an absolute path to a Lit configuration file, or a path relative to the Git ' + 'repository (e.g. libcxx/test/configs/installed-libc++.cfg.in). Note that the configuration is ' + 'always taken from the current checkout, never from the older version of the test suite selected ' + 'by --test-suite-commit.') + parser.add_argument('--lit-param', type=str, action='append', default=[], metavar='KEY=VALUE', + help='Additional parameter to pass to Lit when running the benchmarks. May be provided multiple times. ' + 'Testing configurations generally require some: for example installed-libc++.cfg.in requires ' + '`--lit-param libcxx_installation=<PATH>` and stdlib-libstdc++.cfg.in requires ' + '`--lit-param libstdcxx_compiler=<GCC>`.') parser.add_argument('--compiler', type=str, required=True, help='Path to the compiler to use for testing.') parser.add_argument('--machine', type=str, required=True, @@ -125,10 +141,6 @@ def main(argv): "running only a subset of the benchmarks.") parser.add_argument('--spec-dir', type=pathlib.Path, required=False, help='Optional path to a SPEC installation to use for benchmarking.') - parser.add_argument('--cmake-cache', type=pathlib.Path, required=False, default=pathlib.Path(__file__).resolve().parent / 'cmake' / 'generic.cmake', - help='Optional path to a CMake cache to use when building the library. Only used when building ' - 'the library, not when running the test suite. Defaults to ' - 'libcxx/utils/ci/lnt/cmake/generic.cmake, which builds the library with optimizations enabled.') parser.add_argument('--build-dir', type=pathlib.Path, required=False, help='Optional directory in which to keep build artifacts. By default, a temporary directory ' 'is used and the build artifacts are discarded after the run. It is an error to specify ' @@ -171,14 +183,24 @@ def main(argv): if enforce_success: raise + # Resolve the testing configuration against the current checkout, not against the (potentially + # much older) version of the test suite selected by --test-suite-commit. + test_config = pathlib.Path(args.test_config) + if not test_config.is_absolute(): + test_config = args.git_repo / test_config + test_config = test_config.resolve() + # Fail fast before doing any expensive work: refuse to overwrite existing results or artifacts, and # check dependencies. if args.output.exists(): sys.exit(f'error: output report {args.output} already exists; not overwriting it') if args.build_dir is not None and args.build_dir.exists(): sys.exit(f'error: build directory {args.build_dir} already exists; not overwriting it') - if args.cmake_cache is not None and not args.cmake_cache.is_file(): - sys.exit(f'error: CMake cache {args.cmake_cache} does not exist') + if not test_config.is_file(): + sys.exit(f'error: testing configuration {test_config} does not exist') + for param in args.lit_param: + if '=' not in param: + sys.exit(f'error: --lit-param {param} is not of the form KEY=VALUE') if shutil.which('lnt') is None: sys.exit('error: cannot find `lnt`; install libcxx/utils/ci/lnt/requirements.txt') @@ -193,24 +215,13 @@ def main(argv): artifacts.mkdir(parents=True, exist_ok=True) logging.info(f'Storing build artifacts in {artifacts}') - logging.info(f'Building libc++ at commit {args.benchmark_commit}') - cmake_args = [f'-DCMAKE_CXX_COMPILER={args.compiler}'] - if args.cmake_cache is not None: - cmake_args += ['-C', args.cmake_cache.resolve()] - build_cmd = [args.git_repo / 'libcxx/utils/build-at-commit', - '--git-repo', args.git_repo, - '--install-dir', artifacts / 'libcxx-install', - '--tmp-src-dir', artifacts / 'libcxx-src', - '--tmp-build-dir', artifacts / 'libcxx-build', - '--commit', args.benchmark_commit, '--', *cmake_args] - run(build_cmd, enforce_success=False) # if the build fails, carry on: we'll fail later and submit empty LNT results - - logging.info(f'Running benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}') + logging.info(f'Running benchmarks from {args.test_suite_commit} against {test_config.name}') lit_params = ['--param', f'compiler={args.compiler}', - '--param', f'libcxx_installation={artifacts / "libcxx-install"}', '--param', 'optimization=speed', '--param', 'std=c++26', '--param', 'enable_werror=False'] # older versions of the library trigger new warnings, don't fail + for param in args.lit_param: + lit_params += ['--param', param] if args.spec_dir is not None: lit_params += ['--param', f'spec_dir={args.spec_dir}'] cmd = [args.git_repo / 'libcxx/utils/test-at-commit', @@ -218,7 +229,7 @@ def main(argv): '--build-dir', artifacts / 'benchmarks-build', '--test-suite-commit', args.test_suite_commit, '--tmp-src-dir', artifacts / 'benchmarks-src', - '--test-config', args.git_repo / 'libcxx/test/configs/installed-libc++.cfg.in', + '--test-config', test_config, '--compiler', args.compiler, '--', '-j1', '--time-tests', '--test-output=failed', >From 398d316a3df70026c5d085efa636cd8855f8ba21 Mon Sep 17 00:00:00 2001 From: Louis Dionne <[email protected]> Date: Fri, 18 Sep 2026 08:26:36 -0700 Subject: [PATCH 3/7] Comments --- .github/workflows/libcxx/build-at-commit/action.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/libcxx/build-at-commit/action.yml b/.github/workflows/libcxx/build-at-commit/action.yml index 1613cd4429548..25a82e0481441 100644 --- a/.github/workflows/libcxx/build-at-commit/action.yml +++ b/.github/workflows/libcxx/build-at-commit/action.yml @@ -1,12 +1,7 @@ -# This composite action builds libc++ as of an arbitrary commit and installs it, so that the -# benchmark suite can then be run against that installation. +# This composite action builds libc++ as of an arbitrary commit and installs it into the +# specified directory. # -# It is shared by the workflow that benchmarks a single commit (libcxx-benchmark-commit.yml) -# and the one that compares a pull request against its merge base (libcxx-pr-benchmark.yml), -# which builds twice. -# -# Machines that benchmark a Standard Library we don't build ourselves (e.g. libstdc++) simply -# don't use this action: see the `build` key in libcxx/utils/ci/lnt/machines.json. +# This allows other workflows to reuse this job for various purposes. # # This action assumes that the monorepo has been checked out with its full history (arbitrary # historical commits must be reachable) and that CMake, Ninja and the compiler are available. >From 802eeb9ffc46f450c62ce126aa2e9bf3d775ac85 Mon Sep 17 00:00:00 2001 From: Louis Dionne <[email protected]> Date: Fri, 18 Sep 2026 08:34:05 -0700 Subject: [PATCH 4/7] Simplify --- .github/workflows/libcxx-pr-benchmark.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml index 40bd24bc6fbd1..5a018cfb94e9e 100644 --- a/.github/workflows/libcxx-pr-benchmark.yml +++ b/.github/workflows/libcxx-pr-benchmark.yml @@ -182,17 +182,10 @@ jobs: # TODO: The invocations below should actually become calls to run-benchmarks, which would simply include # benchmark consolidation for _ in $(seq 1 5); do - for which in baseline candidate; do - ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" \ - -B "benchmarks/${which}" \ - --compiler "${COMPILER}" \ - -- \ - -sv -j1 \ - --param optimization=speed \ - --param libcxx_installation="${PWD}/install/${which}" \ - "${BENCHMARKS}" - ./libcxx/utils/consolidate-benchmarks "benchmarks/${which}" | tee -a "${which}.lnt" - done + ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/baseline" "${BENCHMARKS}" + ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt + ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/candidate" "${BENCHMARKS}" + ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt done - name: Compare baseline and candidate runs >From 4a71dc317422280ab480671e5e55c3814fa40ef0 Mon Sep 17 00:00:00 2001 From: Louis Dionne <[email protected]> Date: Fri, 18 Sep 2026 08:40:32 -0700 Subject: [PATCH 5/7] More comments --- .github/workflows/libcxx/build-at-commit/action.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/libcxx/build-at-commit/action.yml b/.github/workflows/libcxx/build-at-commit/action.yml index 25a82e0481441..33f9c35df16c3 100644 --- a/.github/workflows/libcxx/build-at-commit/action.yml +++ b/.github/workflows/libcxx/build-at-commit/action.yml @@ -7,16 +7,14 @@ # historical commits must be reachable) and that CMake, Ninja and the compiler are available. name: 'Build libc++ at a commit' -description: 'Build and install libc++ as of the given commit, for benchmarking purposes.' +description: 'Build and install libc++ as of the given commit.' inputs: commit: description: 'The libc++ commit to build.' required: true install-dir: - description: >- - Directory to install the library into. This is the directory that the benchmarks are then - run against, via the `libcxx_installation` Lit parameter. + description: 'Directory to install the library into.' required: true compiler: description: 'The C++ compiler to build the library with.' >From e3f3ca966e316a125aaec3bc991e5b5e0da84e68 Mon Sep 17 00:00:00 2001 From: Louis Dionne <[email protected]> Date: Fri, 18 Sep 2026 08:43:30 -0700 Subject: [PATCH 6/7] More comments --- .github/workflows/libcxx-benchmark-commit.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml index a383921259aeb..f800826be82c0 100644 --- a/.github/workflows/libcxx-benchmark-commit.yml +++ b/.github/workflows/libcxx-benchmark-commit.yml @@ -143,9 +143,8 @@ jobs: "${COMPILER}" --version python3 --version - # A build failure is tolerated on purpose: the library doesn't build at every historical commit, - # and we still want to submit an (empty) run for such commits. `plan-benchmarks` counts runs - # rather than results, so that is what stops the cron from requesting a commit forever. + # A build failure is tolerated on purpose, since the library doesn't build at every historical commit. + # We still submit an empty run for such commits. - name: Build libc++ at ${{ inputs.commit }} continue-on-error: true uses: ./.github/workflows/libcxx/build-at-commit >From f35d7c0b63456d32eb3cfa1bbc6fe2347d37910b Mon Sep 17 00:00:00 2001 From: Louis Dionne <[email protected]> Date: Fri, 18 Sep 2026 07:33:52 -0700 Subject: [PATCH 7/7] Allow passing lit parameters --- .github/workflows/libcxx-benchmark-commit.yml | 20 +++++++++++++++++-- .github/workflows/libcxx-pr-benchmark.yml | 16 +++++++++++---- libcxx/utils/ci/lnt/machines.json | 12 ++++++++--- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/workflows/libcxx-benchmark-commit.yml b/.github/workflows/libcxx-benchmark-commit.yml index f800826be82c0..133c7ae85b24c 100644 --- a/.github/workflows/libcxx-benchmark-commit.yml +++ b/.github/workflows/libcxx-benchmark-commit.yml @@ -143,16 +143,19 @@ jobs: "${COMPILER}" --version python3 --version + # Optionally, build the library. + # # A build failure is tolerated on purpose, since the library doesn't build at every historical commit. # We still submit an empty run for such commits. - name: Build libc++ at ${{ inputs.commit }} + if: ${{ matrix.build.cmake-cache != '' }} continue-on-error: true uses: ./.github/workflows/libcxx/build-at-commit with: commit: ${{ inputs.commit }} install-dir: ${{ env.INSTALL_DIR }} compiler: ${{ matrix.cxx }} - cmake-cache: ${{ matrix.cmake-cache }} + cmake-cache: ${{ matrix.build.cmake-cache }} - name: Setup virtual environment run: | @@ -163,8 +166,10 @@ jobs: - name: Run the benchmarks env: BENCHMARK_SUITE_VERSION: ${{ matrix.benchmark-suite-version }} + BUILT_LIBCXX: ${{ matrix.build.cmake-cache != '' }} COMMIT: ${{ inputs.commit }} FILTER: ${{ inputs.filter }} + LIT_PARAMS: ${{ matrix.lit-params }} LNT_MACHINE: ${{ matrix.lnt-machine }} TEST_CONFIG: ${{ matrix.test-config }} run: | @@ -173,13 +178,24 @@ jobs: if [ -n "${FILTER}" ]; then filter_arg=(--filter "${FILTER}") fi + + # When we built the library ourselves, point the testing configuration at the installation. + # Other parameters come from machines.json. + lit_params=() + if [ "${BUILT_LIBCXX}" = "true" ]; then + lit_params+=(--lit-param "libcxx_installation=${INSTALL_DIR}") + fi + for param in ${LIT_PARAMS}; do + lit_params+=(--lit-param "${param}") + done + libcxx/utils/ci/lnt/run-benchmarks \ --test-suite-commit "${BENCHMARK_SUITE_VERSION}" \ --machine "${LNT_MACHINE}" \ --compiler "${COMPILER}" \ --benchmark-commit "${COMMIT}" \ --test-config "${TEST_CONFIG}" \ - --lit-param "libcxx_installation=${INSTALL_DIR}" \ + "${lit_params[@]}" \ "${filter_arg[@]}" \ --output "${COMMIT}.json" diff --git a/.github/workflows/libcxx-pr-benchmark.yml b/.github/workflows/libcxx-pr-benchmark.yml index 5a018cfb94e9e..e1120ee3d7f90 100644 --- a/.github/workflows/libcxx-pr-benchmark.yml +++ b/.github/workflows/libcxx-pr-benchmark.yml @@ -111,6 +111,7 @@ jobs: env: BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }} COMPILER: ${{ matrix.cxx }} + LIT_PARAMS: ${{ matrix.lit-params }} LNT_MACHINE: ${{ matrix.lnt-machine }} PR_HEAD: ${{ needs.extract-info.outputs.pr_head }} PR_BASE: ${{ needs.extract-info.outputs.pr_base }} @@ -163,7 +164,7 @@ jobs: commit: ${{ steps.baseline.outputs.commit }} install-dir: ${{ github.workspace }}/install/baseline compiler: ${{ matrix.cxx }} - cmake-cache: ${{ matrix.cmake-cache }} + cmake-cache: ${{ matrix.build.cmake-cache }} - name: Build the candidate uses: ./.github/workflows/libcxx/build-at-commit @@ -171,20 +172,27 @@ jobs: commit: ${{ needs.extract-info.outputs.pr_head }} install-dir: ${{ github.workspace }}/install/candidate compiler: ${{ matrix.cxx }} - cmake-cache: ${{ matrix.cmake-cache }} + cmake-cache: ${{ matrix.build.cmake-cache }} - name: Run baseline and candidate interleaved run: | source .venv/bin/activate + + # Any parameter the testing configuration needs on top of the ones below comes from machines.json. + lit_params=() + for param in ${LIT_PARAMS}; do + lit_params+=(--param "${param}") + done + # Run 5 times so we can pick the median, and interleave baseline and candidate to mitigate the impact of # environmental noise # # TODO: The invocations below should actually become calls to run-benchmarks, which would simply include # benchmark consolidation for _ in $(seq 1 5); do - ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/baseline" "${BENCHMARKS}" + ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/baseline" "${lit_params[@]}" "${BENCHMARKS}" ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt - ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/candidate" "${BENCHMARKS}" + ./libcxx/utils/test-at-commit --test-config "${PWD}/${TEST_CONFIG}" -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed --param libcxx_installation="${PWD}/install/candidate" "${lit_params[@]}" "${BENCHMARKS}" ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt done diff --git a/libcxx/utils/ci/lnt/machines.json b/libcxx/utils/ci/lnt/machines.json index cd0d633fc013e..39c9a76d97930 100644 --- a/libcxx/utils/ci/lnt/machines.json +++ b/libcxx/utils/ci/lnt/machines.json @@ -5,7 +5,9 @@ "cxx": "clang++", "xcode-version": "26.5", "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0", - "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake", + "build": { + "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake" + }, "test-config": "libcxx/test/configs/installed-libc++.cfg.in", "coverage": { "since": "2023-01-01", @@ -21,7 +23,9 @@ "cxx": "clang++", "xcode-version": "26.5", "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0", - "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake", + "build": { + "cmake-cache": "libcxx/utils/ci/lnt/cmake/hardened-fast.cmake" + }, "test-config": "libcxx/test/configs/installed-libc++.cfg.in", "coverage": { "since": "2023-12-01", @@ -36,7 +40,9 @@ "runner": "llvm-premerge-libcxx-runners", "cxx": "clang++-22", "benchmark-suite-version": "8bb5e216937e6b541f351aa1637c67e85a43ada0", - "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake", + "build": { + "cmake-cache": "libcxx/utils/ci/lnt/cmake/generic.cmake" + }, "test-config": "libcxx/test/configs/installed-libc++.cfg.in", "coverage": { "since": "2023-01-01", _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
