ldionne updated this revision to Diff 554420. ldionne marked an inline comment as done. ldionne added a comment.
Address review comments. I did some testing using GH PRs and this should work, but some tweaks might be necessary. After discussing with Mikhail, I'll merge this now and we can make tweaks and fixes on top of it. Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D158863/new/ https://reviews.llvm.org/D158863 Files: .ci/generate-buildkite-pipeline-premerge .ci/generate-buildkite-pipeline-scheduled .ci/monolithic-linux.sh .ci/monolithic-windows.sh
Index: .ci/monolithic-windows.sh =================================================================== --- /dev/null +++ .ci/monolithic-windows.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This script performs a monolithic build of the monorepo and runs the tests of +# most projects on Windows. This should be replaced by per-project scripts that +# run only the relevant tests. +# + +set -ex +set -o pipefail + +MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" +BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-windows}" + +rm -rf ${BUILD_DIR} + +sccache --zero-stats +function show-stats { + sccache --show-stats +} +trap show-stats EXIT + +projects="${1}" +targets="${2}" + +echo "--- cmake" +pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt +cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ + -D LLVM_ENABLE_PROJECTS="${projects}" \ + -G Ninja \ + -D CMAKE_BUILD_TYPE=Release \ + -D LLVM_ENABLE_ASSERTIONS=ON \ + -D LLVM_BUILD_EXAMPLES=ON \ + -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D COMPILER_RT_BUILD_ORC=OFF \ + -D CMAKE_C_COMPILER_LAUNCHER=sccache \ + -D CMAKE_CXX_COMPILER_LAUNCHER=sccache + +echo "--- ninja" +ninja -C ${BUILD_DIR} ${targets} Index: .ci/monolithic-linux.sh =================================================================== --- /dev/null +++ .ci/monolithic-linux.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This script performs a monolithic build of the monorepo and runs the tests of +# most projects on Linux. This should be replaced by per-project scripts that +# run only the relevant tests. +# + +set -ex +set -o pipefail + +MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" +BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build/monolithic-linux}" + +rm -rf ${BUILD_DIR} + +ccache --zero-stats +ccache --show-config +function show-stats { + ccache --print-stats +} +trap show-stats EXIT + +projects="${1}" +targets="${2}" + +echo "--- cmake" +pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt +cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ + -D LLVM_ENABLE_PROJECTS="${projects}" \ + -G Ninja \ + -D CMAKE_BUILD_TYPE=Release \ + -D LLVM_ENABLE_ASSERTIONS=ON \ + -D LLVM_BUILD_EXAMPLES=ON \ + -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D LLVM_ENABLE_LLD=ON \ + -D CMAKE_CXX_FLAGS=-gmlt \ + -D BOLT_CLANG_EXE=/usr/bin/clang \ + -D LLVM_CCACHE_BUILD=ON + +echo "--- ninja" +ninja -C ${BUILD_DIR} ${targets} Index: .ci/generate-buildkite-pipeline-scheduled =================================================================== --- .ci/generate-buildkite-pipeline-scheduled +++ .ci/generate-buildkite-pipeline-scheduled @@ -27,4 +27,43 @@ message: "${BUILDKITE_MESSAGE}" commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" -EOF + + - label: ':linux: x64 Debian' + artifact_paths: + - '*_result.json' + - 'build/monolithic-linux/test-results.xml' + agents: + queue: 'linux' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 120 + env: + CC: 'clang' + CXX: 'clang++' + commands: + - './.ci/monolithic-linux.sh "bolt;clang-tools-extra;compiler-rt;flang;libc;libclc;lld;llvm;mlir;polly;pstl" "check-all"' + + - label: ':windows: x64 Windows' + artifact_paths: + - '*_result.json' + - 'build/monolithic-windows/test-results.xml' + agents: + queue: 'windows' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 150 + env: + CC: 'cl' + CXX: 'cl' + LD: 'link' + commands: + - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64' + - 'bash .ci/monolithic-windows.sh "clang-tools-extra;flang;libclc;lld;llvm;mlir;polly;pstl" "check-all"' Index: .ci/generate-buildkite-pipeline-premerge =================================================================== --- .ci/generate-buildkite-pipeline-premerge +++ .ci/generate-buildkite-pipeline-premerge @@ -14,11 +14,6 @@ # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. # -if ! git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/|^clang/"; then - # libcxx/, libcxxabi/, libunwind/, runtimes/, cmake/ or clang/ are not affected - exit 0 -fi - reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')" if [[ "${reviewID}" != "" ]]; then buildMessage="https://llvm.org/${reviewID}"; @@ -51,3 +46,213 @@ branch: "${BUILDKITE_BRANCH}" EOF fi + +# +# If we're not running a more specific pipeline, generate a legacy monolithic pipeline +# based on which subdirectories have been modified. We have some heuristics for this +# to be reasonable. +# +# Individual projects should instead define the pre-commit CI tests that suits their +# needs while letting them run on the infrastructure provided by LLVM. +# +function compute-projects-to-test() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + lld) + for p in bolt cross-project-tests; do + echo $p + done + ;; + llvm) + for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do + echo $p + done + ;; + clang) + for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do + echo $p + done + ;; + clang-tools-extra) + echo libc + ;; + mlir) + echo flang + ;; + *) + # Nothing to do + ;; + esac + done +} + +function add-dependencies() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + bolt) + for p in lld llvm; do + echo $p + done + ;; + cross-project-tests) + for p in lld clang; do + echo $p + done + ;; + clang-tools-extra) + for p in llvm clang; do + echo $p + done + ;; + compiler-rt|libc|openmp) + echo clang + ;; + flang|lldb) + for p in llvm clang; do + echo $p + done + ;; + lld|mlir|polly) + echo llvm + ;; + *) + # Nothing to do + ;; + esac + done +} + +function exclude-linux() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + lldb) ;; # tests failing + openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410 + *) + echo "${project}" + ;; + esac + done +} + +function exclude-windows() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + compiler-rt) ;; # tests taking too long + openmp) ;; # TODO: having trouble with the Perl installation + libc) ;; # no Windows support + lldb) ;; # tests failing + bolt) ;; # tests are not supported yet + *) + echo "${project}" + ;; + esac + done +} + +function keep-modified-projects() { + projects=${@} + git_diff="$(git diff --name-only HEAD~1)" + for project in ${projects}; do + if echo "${git_diff}" | grep -q -E "^${project}/"; then + echo "${project}" + fi + done +} + +function check-targets() { + projects=${@} + for project in ${projects}; do + case ${project} in + clang-tools-extra) + echo "check-clang-tools" + ;; + compiler-rt) + echo "check-all" + ;; + cross-project-tests) + echo "check-cross-project" + ;; + lldb) + echo "check-all" # TODO: check-lldb may not include all the LLDB tests? + ;; + pstl) + echo "check-all" + ;; + libclc) + echo "check-all" + ;; + *) + echo "check-${project}" + ;; + esac + done +} + +# Figure out which projects need to be built on each platform +all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" +modified_projects="$(keep-modified-projects ${all_projects})" + +linux_projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects})) +linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq) +linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq) + +windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects})) +windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq) +windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq) + +# Generate the appropriate pipeline +if [[ "${linux_projects}" != "" ]]; then + cat <<EOF +- label: ':linux: x64 Debian' + artifact_paths: + - '*_result.json' + - 'build/monolithic-linux/test-results.xml' + agents: + queue: 'linux' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 120 + env: + CC: 'clang' + CXX: 'clang++' + commands: + - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"' +EOF +fi + +if [[ "${windows_projects}" != "" ]]; then + cat <<EOF +- label: ':windows: x64 Windows' + artifact_paths: + - '*_result.json' + - 'build/monolithic-windows/test-results.xml' + agents: + queue: 'windows' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 + limit: 2 # Forced agent shutdown + timeout_in_minutes: 150 + env: + CC: 'cl' + CXX: 'cl' + LD: 'link' + commands: + - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64' + - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"' +EOF +fi
_______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
