This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tirx-standalone-root in repository https://gitbox.apache.org/repos/asf/tvm.git
commit e228c9e39c42c274c8d176d6e4f4e9e968ac2534 Author: tqchen <[email protected]> AuthorDate: Fri Oct 16 02:31:09 2026 +0000 [CI] Add standalone TIRx build, wheel, and docs pipelines Build pinned FFI and TIRx wheels from the standalone root and run CPU, GPU, ARM, Windows, and macOS jobs. Build the Python API documentation alongside GPU tests using the built wheels, and publish only the tirx site subtree from an explicitly enabled main pipeline. Validate the archive revision and publication scope. Run the existing lint workflow on standalone branch pushes. Share environment setup, CPU detection, and release wheel builds through composite actions adapted to the standalone repository layout. --- .github/actions/build-wheel-for-publish/action.yml | 148 +++++ .../detect-env-vars/action.yml} | 39 +- .github/actions/setup/action.yml | 62 ++ .github/workflows/lint.yml | 2 +- .github/workflows/main.yml | 139 +++++ .github/workflows/publish_wheel.yml | 218 +++++++ ci/jenkins/.gitignore | 1 + .../task_build_arm.sh => ci/jenkins/artifacts.py | 21 +- .../jenkins/docker-images.ini | 14 +- ci/jenkins/generated/arm_jenkinsfile.groovy | 46 ++ ci/jenkins/generated/common.groovy | 386 ++++++++++++ ci/jenkins/generated/cpu_jenkinsfile.groovy | 64 ++ ci/jenkins/generated/gpu_jenkinsfile.groovy | 144 +++++ ci/jenkins/scripts/determine_docker_images.py | 156 +++++ ci/jenkins/scripts/git_skip_ci.py | 100 +++ ci/jenkins/scripts/git_skip_ci_globs.py | 76 +++ .../lint.yml => ci/jenkins/scripts/http_utils.py | 35 +- .../jenkins/scripts/retry.sh | 26 +- ci/jenkins/scripts/s3.py | 243 ++++++++ .../jenkins/scripts/show_node_info.sh | 29 +- .../lint.yml => ci/wheel/build-environment.yaml | 51 +- ci/wheel/manylinux_build_libtvm_runtime_cuda.sh | 64 ++ ci/wheel/windows_build_libtvm_runtime_cuda.bat | 99 +++ docker/bash.sh | 669 +++++++++++++++++++++ tests/scripts/task_build_arm.sh | 5 +- tests/scripts/task_build_cpu.sh | 5 +- tests/scripts/task_build_gpu.sh | 5 +- .../{task_build_arm.sh => task_build_gpu_other.sh} | 13 +- ...d_arm.sh => task_install_build_dependencies.sh} | 12 +- .../{task_build_arm.sh => task_python_docs.sh} | 12 +- .../{task_build_arm.sh => task_python_unittest.sh} | 10 +- ...uild_arm.sh => task_python_unittest_gpuonly.sh} | 21 +- ...ild_arm.sh => test_task_install_dist_wheels.sh} | 11 +- 33 files changed, 2800 insertions(+), 126 deletions(-) diff --git a/.github/actions/build-wheel-for-publish/action.yml b/.github/actions/build-wheel-for-publish/action.yml new file mode 100644 index 0000000000..3207675deb --- /dev/null +++ b/.github/actions/build-wheel-for-publish/action.yml @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build TIRx Wheel +description: > + Build and test the standalone LLVM-enabled apache-tvm-tirx wheel for a given OS/architecture + combination using cibuildwheel. + +inputs: + arch: + description: "Target architecture for cibuildwheel (e.g., x86_64, aarch64, arm64, AMD64)" + required: true + build: + description: "cibuildwheel build selector (e.g., cp310-manylinux_x86_64)" + required: true + cmake_defines: + description: "Feature CMake defines that vary by wheel (e.g. -DTVM_TIRX_USE_METAL=ON on macOS)" + required: false + default: "" + include_cuda_runtime: + description: "Set to 1 to build and inject the CUDA runtime library (Linux only)" + required: false + default: "0" + +runs: + using: "composite" + steps: + # Single source of truth for the LLVM toolchain version, shared by the cache + # key and the conda install steps below. + - name: Set LLVM version + shell: bash + run: echo "LLVM_VERSION=22.1.0" >> "$GITHUB_ENV" + + - name: Prepare LLVM cache path (Unix) + if: runner.os != 'Windows' + shell: bash + run: | + set -eux + sudo mkdir -p /opt/llvm + sudo chown -R "$(whoami)" /opt/llvm + + # ---- Cache LLVM prefix ---- + - name: Cache LLVM + uses: actions/[email protected] + id: llvm-cache + with: + path: ${{ runner.os == 'Windows' && 'C:/opt/llvm' || '/opt/llvm' }} + key: tirx-wheel-llvm-${{ env.LLVM_VERSION }}-${{ runner.os }}-${{ inputs.arch }}-v6 + + # ---- Install LLVM via conda (cache miss only) ---- + - name: Setup conda + if: steps.llvm-cache.outputs.cache-hit != 'true' + uses: conda-incubator/[email protected] + continue-on-error: true + id: conda1 + with: + miniforge-version: latest + conda-remove-defaults: true + + - name: Setup conda (retry with tar.bz2) + if: steps.llvm-cache.outputs.cache-hit != 'true' && steps.conda1.outcome == 'failure' + uses: conda-incubator/[email protected] + with: + miniforge-version: latest + use-only-tar-bz2: true + conda-remove-defaults: true + + - name: Install LLVM (Unix) + if: steps.llvm-cache.outputs.cache-hit != 'true' && runner.os != 'Windows' + shell: bash -l {0} + run: | + set -eux + if [[ "${RUNNER_OS}" == "Linux" ]]; then + sudo mkdir -p /opt/llvm + sudo chown -R "$(whoami)" /opt/llvm + fi + conda create -q -p /opt/llvm -c conda-forge \ + "llvmdev=${LLVM_VERSION}" "clangdev=${LLVM_VERSION}" "compiler-rt=${LLVM_VERSION}" zlib zstd-static libxml2-devel \ + -y + + - name: Install LLVM (Windows) + if: steps.llvm-cache.outputs.cache-hit != 'true' && runner.os == 'Windows' + shell: cmd /C call {0} + run: | + call conda create -q -p C:\opt\llvm -c conda-forge llvmdev=%LLVM_VERSION% zlib zstd-static libxml2-devel -y + + # The {project} placeholder is not expanded inside CIBW_ENVIRONMENT values, so + # compute the forward-slash host path here (the Windows analog of the Linux + # /project hardcode). + - name: Compute CUDA sidecar path (Windows) + if: runner.os == 'Windows' && inputs.include_cuda_runtime == '1' + shell: bash + run: echo "TVM_CUDA_EXTRA_LIB=$(cygpath -m "$(pwd)")/build-wheel-cuda/lib/tvm_runtime_cuda.dll" >> "$GITHUB_ENV" + + # ---- Build and test wheels ---- + - name: Build and test TIRx core wheel + uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0 + with: + package-dir: . + output-dir: wheelhouse + env: + CIBW_BUILD: ${{ inputs.build }} + CIBW_ARCHS_LINUX: ${{ inputs.arch }} + CIBW_ARCHS_MACOS: ${{ inputs.arch }} + CIBW_ARCHS_WINDOWS: ${{ inputs.arch }} + # Linux builds run in cibuildwheel's default manylinux_2_28 container; + # bind-mount the cached LLVM prefix into it. Ignored on macOS/Windows, + # which build without a container. + CIBW_CONTAINER_ENGINE: "docker; create_args: --volume /opt/llvm:/opt/llvm:ro" + # Each per-OS block carries the toolchain location (conda LLVM prefix + + # llvm-config path), the universal static-link policy (TVM_TIRX_USE_LLVM --link-static + # and ZLIB_USE_STATIC_LIBS), and the CUDA sidecar. Feature defines that vary by + # wheel (e.g. TVM_TIRX_USE_METAL on macOS) come from the matrix via inputs.cmake_defines. + # One explicit block per build platform (macOS / Linux / Windows); cibuildwheel + # env overrides replace rather than merge, so there is no shared base block. + CIBW_ENVIRONMENT_MACOS: >- + CMAKE_PREFIX_PATH="/opt/llvm" + CMAKE_ARGS="-DTVM_TIRX_USE_LLVM='/opt/llvm/bin/llvm-config --link-static' -DZLIB_USE_STATIC_LIBS=ON -DCMAKE_PREFIX_PATH=/opt/llvm ${{ inputs.cmake_defines }}" + CIBW_ENVIRONMENT_LINUX: >- + CMAKE_PREFIX_PATH="/opt/llvm" + LIBRARY_PATH="/opt/llvm/lib" + CMAKE_ARGS="-DTVM_TIRX_USE_LLVM='/opt/llvm/bin/llvm-config --link-static' -DZLIB_USE_STATIC_LIBS=ON -DCMAKE_PREFIX_PATH=/opt/llvm ${{ inputs.cmake_defines }} -DTVM_TIRX_PACKAGE_EXTRA_LIBS=/project/build-wheel-cuda/lib/libtvm_runtime_cuda.so" + CIBW_ENVIRONMENT_WINDOWS: >- + CMAKE_PREFIX_PATH="C:/opt/llvm/Library" + PATH="C:/opt/llvm/Library/bin;$PATH" + CMAKE_ARGS="-DTVM_TIRX_USE_LLVM='C:/opt/llvm/Library/bin/llvm-config.exe --link-static' -DZLIB_USE_STATIC_LIBS=ON -DCMAKE_PREFIX_PATH=C:/opt/llvm/Library ${{ inputs.cmake_defines }} -DTVM_TIRX_PACKAGE_EXTRA_LIBS=${{ env.TVM_CUDA_EXTRA_LIB }}" + # Turns the wheel-specific assertions in + # tests/python/runtime/test_validate_wheel_library.py + # ON (they skip unless these are set). Every published wheel is LLVM-enabled, + # so TVM_WHEEL_EXPECT_LLVM is always "1"; the CUDA runtime is only bundled on + # the CUDA wheels, so TVM_WHEEL_EXPECT_CUDA_RUNTIME is "0" for CPU wheels. + CIBW_TEST_ENVIRONMENT: >- + TVM_WHEEL_EXPECT_LLVM="1" + TVM_WHEEL_EXPECT_CUDA_RUNTIME="${{ inputs.include_cuda_runtime }}" diff --git a/.github/workflows/lint.yml b/.github/actions/detect-env-vars/action.yml similarity index 53% copy from .github/workflows/lint.yml copy to .github/actions/detect-env-vars/action.yml index a9bff51f3b..afd9efeaad 100644 --- a/.github/workflows/lint.yml +++ b/.github/actions/detect-env-vars/action.yml @@ -15,27 +15,26 @@ # specific language governing permissions and limitations # under the License. -name: Lint +name: Detect Environment Variables +description: Detects environment variables such as CPU count and sets them as outputs. +runs: + using: "composite" + steps: + - name: Run Python to detect environment variables + shell: python + id: detect + run: | + import multiprocessing, os -on: - push: - branches: [main] - pull_request: - branches: [main] + output_file = open(os.environ.get("GITHUB_OUTPUT"), "a") -permissions: - contents: read + def write_env_var(name, value): + output_file.write(f"{name}={value}\n") + print(f"Detected environment variable: {name}={value}") -concurrency: - group: Lint-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true + write_env_var("cpu_count", multiprocessing.cpu_count()) -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/[email protected] - with: - fetch-depth: 0 - fetch-tags: true - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 +outputs: + cpu_count: + description: "The number of CPU cores" + value: ${{ steps.detect.outputs.cpu_count }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000000..5ca6414215 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Setup TVM CI environment +description: Set up the cached conda build environment and install tvm-ffi. + +runs: + using: "composite" + steps: + - uses: actions/[email protected] + env: + CACHE_NUMBER: 2 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/wheel/build-environment.yaml') }} + - uses: conda-incubator/[email protected] + continue-on-error: true + id: conda1 + with: + activate-environment: tvm-build + channel-priority: strict + environment-file: ci/wheel/build-environment.yaml + auto-activate-base: false + miniforge-version: latest + python-version: "3.10" + conda-remove-defaults: true + - uses: conda-incubator/[email protected] + if: steps.conda1.outcome == 'failure' + with: + activate-environment: tvm-build + channel-priority: strict + environment-file: ci/wheel/build-environment.yaml + auto-activate-base: false + miniforge-version: latest + use-only-tar-bz2: true + python-version: "3.10" + conda-remove-defaults: true + - name: Conda info + shell: pwsh + run: | + mamba info + mamba list + mamba info --envs + mamba list --name base + - name: Install tvm-ffi pip package + shell: bash -l {0} + run: | + pip install -v ./3rdparty/tvm-ffi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a9bff51f3b..8ac5f2f9f5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ name: Lint on: push: - branches: [main] + branches: [main, tirx-standalone-root] pull_request: branches: [main] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..ec387f6e1c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# GH actions. +# We use it to cover windows and mac builds +# Jenkins is still the primary CI + +name: CI + +on: + push: + branches: + - main + - tirx-standalone-root # Temporary standalone branch validation. + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: CI-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + MacOS: + if: >- + ${{ github.repository == 'apache/tvm' || + (github.repository == 'tqchen/tvm-dev' && github.ref == 'refs/heads/tirx-standalone-root') }} + runs-on: macOS-latest + steps: + - uses: actions/[email protected] + with: + submodules: 'recursive' + - uses: ./.github/actions/setup + - name: Detect environment variables + id: env_vars + uses: ./.github/actions/detect-env-vars + - name: Build TIRx wheels + shell: bash -l {0} + env: + CMAKE_BUILD_PARALLEL_LEVEL: ${{ steps.env_vars.outputs.cpu_count }} + run: | + set -euxo pipefail + pip install "scikit-build-core>=0.11" "setuptools-scm>=8" + export CMAKE_ARGS="-DTVM_TIRX_USE_LLVM=ON -DTVM_TIRX_USE_METAL=ON -DTVM_TIRX_BUILD_TESTS=OFF" + pip wheel --no-deps -w dist . -v + pip install --no-deps --force-reinstall dist/apache_tvm_tirx-*.whl + - name: Install TIRx from wheels + shell: bash -l {0} + run: | + set -euxo pipefail + export TVM_FFI_VERSION="$(python -c 'from importlib.metadata import version; print(version("apache-tvm-ffi"))')" + pip install psutil cloudpickle ml_dtypes numpy packaging scipy typing_extensions + # --no-deps: this package requires apache-tvm-ffi>=0.1.14.post1, and the source-built core + # versions as 0.1.dev1+g<sha> from the shallow checkout, which is < 0.1.0 under PEP 440. + # Without it pip replaces the source-matched core with a PyPI build. + pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 + # Keep the source-matched tvm-ffi installed by the setup steps. + pip install --no-deps dist/*.whl + python - <<'PY' + import os + from importlib.metadata import requires, version + + assert version("apache-tvm-ffi") == os.environ["TVM_FFI_VERSION"] + assert version("apache-tvm-tirx") == "0.1.0a1" + assert "apache-tvm-ffi>=0.1.15rc0" in requires("apache-tvm-tirx") + assert "apache-tvm-ffi-orcjit==0.1.2" in requires("apache-tvm-tirx") + PY + + - name: Test + shell: bash -l {0} + run: >- + python -m pytest -v tests/python/runtime/test_validate_wheel_library.py tests/python/script + - name: Minimal Metal Compile-Only + shell: bash -l {0} + run: | + set -euxo pipefail + python -m pytest -v -s 'tests/python/codegen/gpu_common/test_allreduce.py::test_allreduce_sum_compile' + python -m pytest -v -s 'tests/python/backend/metal/codegen/test_basic.py::test_func_with_trailing_pod_params' + - name: Minimal Metal Compile-and-Run + shell: bash -l {0} + run: | + set -euxo pipefail + python -m pytest -v -s tests/python/backend/metal + python -m pytest -v -s 'tests/python/codegen/gpu_common/test_intrinsics.py' + python -m pytest -v -s 'tests/python/codegen/gpu_common/test_allreduce.py::test_allreduce_sum[dims0-metal]' + + Windows: + if: >- + ${{ github.repository == 'apache/tvm' || + (github.repository == 'tqchen/tvm-dev' && github.ref == 'refs/heads/tirx-standalone-root') }} + runs-on: windows-latest + steps: + - uses: actions/[email protected] + with: + submodules: 'recursive' + - uses: ./.github/actions/setup + - name: Detect environment variables + id: env_vars + uses: ./.github/actions/detect-env-vars + - name: Build and install TIRx + shell: cmd /C call {0} + env: + CMAKE_BUILD_PARALLEL_LEVEL: ${{ steps.env_vars.outputs.cpu_count }} + run: | + pip install "scikit-build-core>=0.11" "setuptools-scm>=8" || exit /b 1 + set CMAKE_ARGS=-DTVM_TIRX_USE_LLVM=ON -DTVM_TIRX_BUILD_TESTS=OFF + pip wheel --no-deps -w dist . -v || exit /b 1 + for %%w in (dist\apache_tvm_tirx-*.whl) do ( + pip install --no-deps --force-reinstall "%%w" || exit /b 1 + ) + - name: Install test dependencies + shell: cmd /C call {0} + run: | + pip install psutil cloudpickle ml_dtypes numpy packaging scipy typing_extensions || exit /b 1 + rem --no-deps keeps the source-built apache-tvm-ffi (versioned below 0.1.14.post1 from the + rem shallow checkout) from being replaced by a PyPI build. + pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 || exit /b 1 + - name: Test + shell: cmd /C call {0} + run: >- + python -m pytest -v tests/python/runtime/test_validate_wheel_library.py tests/python/script diff --git a/.github/workflows/publish_wheel.yml b/.github/workflows/publish_wheel.yml new file mode 100644 index 0000000000..3f97192c29 --- /dev/null +++ b/.github/workflows/publish_wheel.yml @@ -0,0 +1,218 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Publish TIRx wheels + +on: + workflow_dispatch: + inputs: + tag: + description: "Tag, branch, or SHA to build" + required: true + type: string + publish_repository: + description: "Where to publish after the wheel build succeeds" + required: true + default: "none" + type: choice + options: + - none + - pypi + +permissions: + contents: read + +# CI runners are ephemeral, so pip's HTTP cache buys nothing and a stale +# preinstalled cache (e.g. on the macOS image) only produces noisy +# "Cache entry deserialization failed" warnings. Disable it everywhere. +env: + PIP_NO_CACHE_DIR: "1" + +jobs: + # Build the CUDA runtime sidecar once per arch and upload it as an artifact that + # build_wheels bundles. The Linux legs build inside the official manylinux_2_28 + # CUDA image (CUDA toolkit preinstalled, see pypa/manylinux) -- the same glibc + # baseline cibuildwheel targets for the wheel -- so the sidecar stays + # ABI-compatible with the wheel's libtvm_runtime.so regardless of the exact tag. + build_cuda_runtime: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + strategy: + fail-fast: false + matrix: + include: + - name: "CUDA runtime sidecar (Linux x86_64, manylinux_2_28)" + os: ubuntu-latest + container: quay.io/manylinux_cuda/manylinux_2_28_x86_64_cuda13_1:latest + arch: x86_64 + script: ci/wheel/manylinux_build_libtvm_runtime_cuda.sh + lib: build-wheel-cuda/lib/libtvm_runtime_cuda.so + - name: "CUDA runtime sidecar (Linux aarch64, manylinux_2_28)" + os: ubuntu-24.04-arm + container: quay.io/manylinux_cuda/manylinux_2_28_aarch64_cuda13_1:latest + arch: aarch64 + script: ci/wheel/manylinux_build_libtvm_runtime_cuda.sh + lib: build-wheel-cuda/lib/libtvm_runtime_cuda.so + - name: "CUDA runtime sidecar (Windows AMD64)" + os: windows-2022 + container: "" + arch: AMD64 + script: ci/wheel/windows_build_libtvm_runtime_cuda.bat + lib: build-wheel-cuda/lib/tvm_runtime_cuda.dll + steps: + # The containerized Linux legs check out as root; mark the tree safe so git + # (submodule init in checkout) does not bail on "dubious ownership". + - name: Mark workspace safe for git + if: runner.os == 'Linux' + shell: bash + run: git config --global --add safe.directory '*' + + - name: Checkout source + uses: actions/[email protected] + with: + ref: ${{ inputs.tag }} + submodules: recursive + fetch-depth: 1 + fetch-tags: true + + # Windows has no manylinux interpreter; the script's pip install needs one. + - name: Set up Python (Windows host) + if: runner.os == 'Windows' + uses: actions/[email protected] + with: + python-version: "3.10" + + - name: Build CUDA runtime sidecar (Unix) + if: runner.os != 'Windows' + shell: bash + run: bash ${{ matrix.script }} + + - name: Build CUDA runtime sidecar (Windows) + if: runner.os == 'Windows' + shell: cmd + run: call ${{ matrix.script }} + + - name: Upload CUDA runtime sidecar + uses: actions/[email protected] + with: + name: tirx-cuda-runtime-${{ matrix.arch }} + path: ${{ matrix.lib }} + if-no-files-found: error + + build_wheels: + name: ${{ matrix.name }} + # All-or-nothing: a failed sidecar leg blocks the whole wheel matrix (a publish + # needs the complete 4-wheel set anyway). + needs: [build_cuda_runtime] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: "Linux x86_64 wheel with CUDA runtime (manylinux_2_28)" + os: ubuntu-latest + arch: x86_64 + build: cp310-manylinux_x86_64 + include_cuda_runtime: "1" + artifact_suffix: linux-x86_64-manylinux_2_28 + - name: "Linux aarch64 wheel with CUDA runtime (manylinux_2_28)" + os: ubuntu-24.04-arm + arch: aarch64 + build: cp310-manylinux_aarch64 + include_cuda_runtime: "1" + artifact_suffix: linux-aarch64-manylinux_2_28 + - name: "macOS arm64 wheel with Metal" + os: macos-14 + arch: arm64 + build: cp310-macosx_arm64 + include_cuda_runtime: "0" + artifact_suffix: macos-arm64 + cmake_defines: "-DTVM_TIRX_USE_METAL=ON" + - name: "Windows AMD64 wheel with CUDA runtime" + os: windows-2022 + arch: AMD64 + build: cp310-win_amd64 + include_cuda_runtime: "1" + artifact_suffix: windows-amd64 + steps: + - name: Checkout source + uses: actions/[email protected] + with: + ref: ${{ inputs.tag }} + submodules: recursive + # Preserve the source history and tags for reproducible release builds. + fetch-depth: 0 + fetch-tags: true + + # Land the sidecar where -DTVM_TIRX_PACKAGE_EXTRA_LIBS / cibuildwheel's /project + # mount expects it. Skipped on CPU-only rows (macOS). + - name: Download CUDA runtime sidecar + if: ${{ matrix.include_cuda_runtime == '1' }} + uses: actions/[email protected] + with: + name: tirx-cuda-runtime-${{ matrix.arch }} + path: build-wheel-cuda/lib + + - name: Build and test TIRx core wheel + uses: ./.github/actions/build-wheel-for-publish + with: + arch: ${{ matrix.arch }} + build: ${{ matrix.build }} + cmake_defines: ${{ matrix.cmake_defines }} + include_cuda_runtime: ${{ matrix.include_cuda_runtime }} + + - name: Upload wheel artifact + uses: actions/[email protected] + with: + name: tirx-wheel-${{ matrix.artifact_suffix }} + path: wheelhouse/*.whl + if-no-files-found: error + + upload_pypi: + name: Upload package distributions + needs: [build_wheels] + if: ${{ inputs.publish_repository != 'none' }} + runs-on: ubuntu-latest + environment: ${{ inputs.publish_repository }} + permissions: + id-token: write + attestations: write + steps: + - uses: actions/[email protected] + with: + pattern: tirx-wheel-* + path: dist + merge-multiple: true + + # Print wheel sizes for visibility only. We do not fail on the PyPI 100 MB + # limit here -- the publish step's upload would surface that error directly. + - name: Print wheel sizes + shell: bash + run: ls -alh dist/*.whl + + - name: Generate artifact attestation for wheels + uses: actions/[email protected] + with: + subject-path: dist/* + + - name: Publish package distributions to PyPI + if: ${{ inputs.publish_repository == 'pypi' }} + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + attestations: true + verbose: true diff --git a/ci/jenkins/.gitignore b/ci/jenkins/.gitignore new file mode 100644 index 0000000000..3d2dbd4b63 --- /dev/null +++ b/ci/jenkins/.gitignore @@ -0,0 +1 @@ +/_venv diff --git a/tests/scripts/task_build_arm.sh b/ci/jenkins/artifacts.py old mode 100755 new mode 100644 similarity index 66% copy from tests/scripts/task_build_arm.sh copy to ci/jenkins/artifacts.py index cb85fea9ea..7ad2a1287e --- a/tests/scripts/task_build_arm.sh +++ b/ci/jenkins/artifacts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,11 +15,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Named standalone CI artifacts; uploads expand globs.""" -set -euxo pipefail +import sys -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +ARTIFACTS = {"wheels": ["dist/*.whl"]} + +if __name__ == "__main__": + paths = [] + for name in sys.argv[1:]: + if name not in ARTIFACTS: + print(f"unknown bundle: {name}", file=sys.stderr) + sys.exit(1) + paths.extend(ARTIFACTS[name]) + for p in paths: + print(p) diff --git a/tests/scripts/task_build_arm.sh b/ci/jenkins/docker-images.ini old mode 100755 new mode 100644 similarity index 77% copy from tests/scripts/task_build_arm.sh copy to ci/jenkins/docker-images.ini index cb85fea9ea..fbd96a1770 --- a/tests/scripts/task_build_arm.sh +++ b/ci/jenkins/docker-images.ini @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,10 +15,9 @@ # specific language governing permissions and limitations # under the License. -set -euxo pipefail - -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +# This data file is read during when Jenkins runs job to determine docker images. +[jenkins] +ci_tag: 20260705-142349-cfb98e93 +ci_arm: tlcpack/ci-arm:%(ci_tag)s +ci_cpu: tlcpack/ci-cpu:%(ci_tag)s +ci_gpu: tlcpack/ci-gpu:%(ci_tag)s diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy new file mode 100644 index 0000000000..19703571e7 --- /dev/null +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +def helper_source +try { + helper_source = readTrusted('ci/jenkins/generated/common.groovy') +} catch (Throwable ex) { + echo 'Jenkinsfile changes must be pushed to an upstream branch. ' + + 'arm/pr-head will fail for these changes; check arm/branch status instead.' + throw ex +} +ci = evaluate(helper_source) +ci.start() + +stage('Build') { + ci.with_node(instance: 'ARM-GRAVITON3-SPOT', retry_instance: 'ARM-GRAVITON3', + name: 'BUILD: arm', folder: 'tvm/build-arm', variables: ['PLATFORM=arm']) { + ci.docker_init(images.ci_arm) + withEnv(ci.build_environment()) { + sh( + script: """${docker_run} ${images.ci_arm} bash -euo pipefail -c ' + source ./tests/scripts/task_install_build_dependencies.sh + ./tests/scripts/task_build_arm.sh + '""", + label: 'Build arm wheels and C++ tests', + ) + } + ci.upload_artifacts('arm', '--path "dist/*.whl"') + + } +} diff --git a/ci/jenkins/generated/common.groovy b/ci/jenkins/generated/common.groovy new file mode 100644 index 0000000000..f994d6cae3 --- /dev/null +++ b/ci/jenkins/generated/common.groovy @@ -0,0 +1,386 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.jenkinsci.plugins.pipeline.modeldefinition.Utils + +// ------ Config ------ +// Keep image names aligned with docker-images.ini. Tags are resolved in prepare. +image_platforms = [ci_arm: 'ARM', ci_cpu: 'CPU', ci_gpu: 'GPU'] +// Resolve Hub inputs through the caller account ECR cache; false keeps public names. +enable_docker_hub_cache = true +max_time = 180 +// Temporarily bypass image sccache wrappers and inherited compiler launchers. +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TEST_STEP_NAME' + + ' --env CC=/usr/bin/gcc --env CXX=/usr/bin/g++' + + ' --env CMAKE_C_COMPILER_LAUNCHER= --env CMAKE_CXX_COMPILER_LAUNCHER=' +s3_bucket = 'tvm-jenkins-artifacts-prod' +jenkins_scripts_root = 'ci/jenkins/scripts' + +// ------ Runtime state ------ +images = image_platforms.collectEntries { name, platform -> [(name): ''] } +source_images = [:] +upstream_revision = null +s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" + +properties([parameters(image_platforms.collect { name, platform -> + string(name: "${name}_param", defaultValue: '') +})]) + +// Shared utilities; platform Jenkinsfiles own stages, parallel groups and commands. + +def per_exec_ws(folder) { + return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder +} + +def update_upstream_revision(git_ref) { + if (upstream_revision == null) { + upstream_revision = sh( + script: "git log -1 ${git_ref} --format=\'%H\'", + label: 'Determine upstream revision', + returnStdout: true, + ).trim() + } +} + +def merge_with_target() { + def target = env.CHANGE_TARGET + sh ( + script: "git fetch origin ${target}", + label: "Fetch target branch ${target}", + ) + update_upstream_revision("FETCH_HEAD") + sh ( + script: "git -c user.name=TVM-Jenkins -c [email protected] merge ${upstream_revision}", + label: "Merge to origin/${target}" + ) +} + +def checkout_trusted_files() { + // trust everything from branch builds + if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { + return; + } + + // Host helpers can access credentials; always use the pinned target version. + sh( + script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/. ci/jenkins/artifacts.py", + label: 'Check out trusted files', + ) +} + +def init_git() { + retry(5) { + checkout scm + } + + // Add more info about job node + sh ( + script: './ci/jenkins/scripts/show_node_info.sh', + label: 'Show executor node info', + ) + + // Determine merge commit to use for all stages + if (env.CHANGE_TARGET) { + // This is a PR build, so merge with the latest of the PR's target branch + // (e.g. main or a release branch like v0.25.0). + merge_with_target() + } else { + // This is a branch build (main or a release branch). Only set + // upstream_revision to HEAD and skip merging to avoid a race with another + // commit merged to the branch. + update_upstream_revision("HEAD") + } + + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0 + """, + label: 'Update git submodules', + ) + checkout_trusted_files() +} + +// Apply only after Hub selection and job overrides; inputs stay public image names. +def docker_hub_image(image, cache) { + if (!cache || !image) return image + + def path = image + if (image.contains('/')) { + def host = image.substring(0, image.indexOf('/')) + if (host in ['docker.io', 'index.docker.io', 'registry-1.docker.io']) { + path = image.substring(image.indexOf('/') + 1) + } else if (host.contains('.') || host.contains(':') || host == 'localhost') { + return image + } + } + if (!path.contains('/')) path = "library/${path}" + return "${cache}/${path}" +} + +def ecr_registry(image) { + def registry = image.tokenize('/')[0] + return registry ==~ /[0-9]{12}\.dkr\.ecr\.[a-z0-9-]+\.amazonaws\.com(\.cn)?/ ? registry : null +} + +def ecr_pull(full_name) { + def registry = ecr_registry(full_name) + def region = registry.tokenize('.')[3] + withEnv(["AWS_DEFAULT_REGION=${region}", "AWS_ECR_REPO=${registry}"]) { + try { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } finally { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + +def cancel_previous_build() { + // cancel previous build if it is not on main. + if (env.BRANCH_NAME != 'main') { + def buildNumber = env.BUILD_NUMBER as int + // Milestone API allows us to cancel previous build + // with the same milestone number + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) + } +} + +def is_last_build() { + // check whether it is last build + try { + return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number + } catch (Throwable ex) { + echo 'Error during check is_last_build ' + ex.toString() + return false + } +} + +def should_skip_ci(pr_number) { + if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { + // never skip CI on build sourced from a branch + return false + } + glob_skip_ci_code = sh ( + returnStatus: true, + script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", + label: 'Check if CI should be skipped due to changed files', + ) + if (glob_skip_ci_code == 0) { + return true + } + withCredentials([string( + credentialsId: 'tvm-bot-jenkins-reader', + variable: 'GITHUB_TOKEN', + )]) { + // Exit code of 1 means run full CI (or the script had an error, so run + // full CI just in case). Exit code of 0 means skip CI. + git_skip_ci_code = sh ( + returnStatus: true, + script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", + label: 'Check if CI should be skipped', + ) + } + return git_skip_ci_code == 0 +} + +def prepare(node_type) { + stage('Prepare') { + node(node_type) { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { + init_git() + + images.keySet().each { name -> + images[name] = sh( + script: "python3 ${jenkins_scripts_root}/determine_docker_images.py --lookup-only ${name}", + returnStdout: true, + label: "Read image tag for ${name}", + ).trim() + } + if (env.DETERMINE_DOCKER_IMAGES == 'yes') { + sh( + script: "./${jenkins_scripts_root}/determine_docker_images.py ${images.keySet().join(' ')}", + label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', + ) + images.keySet().each { name -> + images[name] = sh( + script: "cat .docker-image-names/${name}", + label: "Find docker image name for ${name}", + returnStdout: true, + ).trim() + } + } + def cache = '' + if (enable_docker_hub_cache) { + def aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity --query Account --output text', + label: 'Get AWS ID' + ).trim() + cache = "${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com/docker-hub" + } + images.keySet().each { name -> + source_images[name] = params["${name}_param"] ?: images[name] + images[name] = docker_hub_image(source_images[name], cache) + } + sh( + script: 'echo "Docker images being used in this build:"\n' + + images.collect { name, image -> "echo \" ${name} = ${image}\"" }.join('\n'), + label: 'Docker image names', + ) + + skip_ci = should_skip_ci(env.CHANGE_ID) + } + } + } +} + +def retry_latest(ex, body) { + echo 'Exception during SPOT run ' + ex.toString() + if (!is_last_build()) { + echo 'Exit since it is not last build' + throw ex + } + echo 'Retry on-demand given it is last build' + currentBuild.result = 'SUCCESS' + // Do not catch failure of the on-demand retry. + body() +} + +// Public entry points. + +// with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', name: 'Build', +// folder: 'tvm/build', variables: ['PLATFORM=cpu']) { docker_init(image); ... } +// name marks skipped stages; folder isolates executor workspaces; variables is optional. +// Skip before allocation/setup. Each attempt checks out sources, then times the body. +// Keep docker_init explicit in the body. Omit retry_instance for a single attempt; +// otherwise failures, including interruptions, retry only for the newest build. +def with_node(options, body) { + if (skip_ci) { + Utils.markStageSkippedForConditional(options.name) + return + } + def run = { instance -> + node(instance) { + ws(per_exec_ws(options.folder)) { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv(options.variables ?: []) { body() } + } + } + } + } + try { + run(options.instance) + } catch (Throwable ex) { + if (!options.retry_instance) throw ex + // Preserve the original newest-build retry policy, including interruptions. + // Failures from the single on-demand attempt propagate without another retry. + retry_latest(ex) { run(options.retry_instance) } + } +} + +// upload_artifacts('cpu', '--artifact wheels') +// Or select paths: '--path "build/*.whl"'. Options pass unchanged to s3.py. +// Registry artifacts and explicit paths are mutually exclusive; repeat one-value flags. +// Upload globs are sorted/deduplicated; download_artifacts(tag) copies the whole saved prefix. +def upload_artifacts(tag, options) { + sh( + script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/${tag} ${options}", + label: 'Upload artifacts to S3', + ) +} + +def download_artifacts(tag) { + sh( + script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/${tag}", + label: 'Download artifacts from S3', + ) +} + +def docker_init(image) { + // Forget compiler wrappers cached before the temporary sccache bypass. + sh 'rm -f build/CMakeCache.txt 3rdparty/tvm-ffi/build/CMakeCache.txt' + // Clear out all Docker images that aren't going to be used + sh( + script: """ + set -eux + docker image ls --all + IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}') + + echo -e "Found images:\\n\$IMAGES" + echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } + + docker image ls --all + """, + label: 'Clean old Docker images', + ) + + if (ecr_registry(image)) { + ecr_pull(image) + } else { + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${image} + """, + label: 'Pull docker image', + ) + } +} + +// Preparation retries Exceptions unconditionally; Errors and cancellation of the +// previous build are deliberately outside this fallback. +def start() { + cancel_previous_build() + try { + prepare('CPU-SMALL-SPOT') + } catch (Exception ex) { + prepare('CPU-SMALL') + } +} + +// Preserve caller limits, otherwise share the node among its executors. +def build_environment() { + def parallel_limit = env.CMAKE_BUILD_PARALLEL_LEVEL ?: sh( + script: 'n=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} )); if [ "$n" -lt 1 ]; then n=1; fi; echo "$n"', + returnStdout: true, + label: 'Determine build parallelism', + ).trim() + return ["CMAKE_BUILD_PARALLEL_LEVEL=${parallel_limit}"] +} + +return this diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy new file mode 100644 index 0000000000..d487846a0b --- /dev/null +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +def helper_source +try { + helper_source = readTrusted('ci/jenkins/generated/common.groovy') +} catch (Throwable ex) { + echo 'Jenkinsfile changes must be pushed to an upstream branch. ' + + 'cpu/pr-head will fail for these changes; check cpu/branch status instead.' + throw ex +} +ci = evaluate(helper_source) +ci.start() + +stage('Build') { + ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', + name: 'BUILD: CPU', folder: 'tvm/build-cpu', variables: ['PLATFORM=cpu']) { + ci.docker_init(images.ci_cpu) + withEnv(ci.build_environment()) { + sh( + script: """${docker_run} ${images.ci_cpu} bash -euo pipefail -c ' + source ./tests/scripts/task_install_build_dependencies.sh + ./tests/scripts/task_build_cpu.sh + '""", + label: 'Build cpu wheels and C++ tests', + ) + } + ci.upload_artifacts('cpu', '--path "dist/*.whl"') + + } +} + +stage('Test') { + parallel('unittest: CPU': { + ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', + name: 'unittest: CPU', folder: 'tvm/ut-python-cpu', + variables: ['PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU']) { + ci.docker_init(images.ci_cpu) + ci.download_artifacts('cpu') + sh ( + script: """${docker_run} ${images.ci_cpu} bash -euo pipefail -c ' + source ./tests/scripts/test_task_install_dist_wheels.sh + ./tests/scripts/task_python_unittest.sh + '""", + label: 'Run Python unit tests', + ) + } + }) +} diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy new file mode 100644 index 0000000000..c5854e010b --- /dev/null +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reject untrusted helper edits before allocating nodes or running pipeline steps. +def helper_source +try { + helper_source = readTrusted('ci/jenkins/generated/common.groovy') +} catch (Throwable ex) { + echo 'Jenkinsfile changes must be pushed to an upstream branch. ' + + 'gpu/pr-head will fail for these changes; check gpu/branch status instead.' + throw ex +} +ci = evaluate(helper_source) +ci.start() + +stage('Build') { + ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', + name: 'BUILD: GPU', folder: 'tvm/build-gpu', variables: ['PLATFORM=gpu']) { + ci.docker_init(images.ci_gpu) + withEnv(ci.build_environment()) { + sh( + script: """${docker_run} --no-gpu ${images.ci_gpu} bash -euo pipefail -c ' + source ./tests/scripts/task_install_build_dependencies.sh + ./tests/scripts/task_build_gpu.sh + '""", + label: 'Build gpu', + ) + } + ci.upload_artifacts('gpu', '--path "dist/*.whl"') + + // Check the compiler-only GPU configuration after publishing the test build. + sh 'rm -rf build' + withEnv(ci.build_environment()) { + sh( + script: "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_build_gpu_other.sh", + label: 'Build gpu_other', + ) + } + } +} + +parallel( + 'Test': { + stage('Test') { + ci.with_node(instance: 'GPU-SPOT', retry_instance: 'GPU', + name: 'unittest: GPU', folder: 'tvm/ut-python-gpu', + variables: ['PLATFORM=gpu', 'TEST_STEP_NAME=unittest: GPU']) { + ci.docker_init(images.ci_gpu) + ci.download_artifacts('gpu') + sh( + script: """${docker_run} ${images.ci_gpu} bash -euo pipefail -c ' + source ./tests/scripts/test_task_install_dist_wheels.sh + ./tests/scripts/task_python_unittest_gpuonly.sh + '""", + label: 'Run Python GPU unit tests', + ) + } + } + }, + 'Docs': { + stage('Docs') { + ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', + name: 'Docs', folder: 'tvm/tirx-docs', variables: ['PLATFORM=gpu']) { + ci.docker_init(images.ci_gpu) + ci.download_artifacts('gpu') + sh( + script: "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_python_docs.sh", + label: 'Build TIRx API documentation', + ) + ci.upload_artifacts('docs', '--path docs.tgz') + } + } + }, +) + +stage('Deploy Docs') { + if (env.BRANCH_NAME == 'main' && !env.CHANGE_ID && env.DOCS_DEPLOY_ENABLED == 'yes') { + ci.with_node(instance: 'CPU', name: 'Deploy Docs', folder: 'tvm/deploy-tirx-docs') { + ci.download_artifacts('docs') + // Keep credential-bearing deployment in the trusted Jenkinsfile. + sh( + script: ''' + set -eu + SOURCE_COMMIT=$(git rev-parse HEAD) + rm -rf tvm-site + git clone --branch asf-site --depth=1 https://github.com/apache/tvm-site.git tvm-site + cd tvm-site + git log -1 --format='Site base: %H' + rm -rf tirx + mkdir tirx + tar -xzf ../docs.tgz -C tirx + test -s tirx/index.html + test "$(cat tirx/commit_hash)" = "$SOURCE_COMMIT" + git config user.name tvm-bot + git config user.email [email protected] + git add -f -- tirx + test -z "$(git diff --cached --name-only -- . ':!:tirx')" + git diff --cached --shortstat + git diff --cached --name-only | sed -n '1,20p' + if ! git diff --cached --quiet; then + git commit --quiet -m "deploying tirx docs ($(cat tirx/commit_hash))" -- tirx + fi + git log -1 --format='Docs preview commit: %H' + ''', + label: 'Update tvm-site/tirx', + ) + withCredentials([string(credentialsId: 'docs-push-token', variable: 'GITHUB_TOKEN')]) { + sh( + script: ''' + set +x + set -eu + cd tvm-site + auth=$(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 | tr -d '\\n') + for attempt in 1 2 3; do + if git -c http.extraHeader="Authorization: Basic $auth" push origin HEAD:asf-site; then + exit 0 + fi + if [ "$attempt" = 3 ]; then + exit 1 + fi + sleep 2 + git pull --rebase origin asf-site + done + ''', + label: 'Publish TIRx documentation to apache/tvm-site', + ) + } + } + } +} diff --git a/ci/jenkins/scripts/determine_docker_images.py b/ci/jenkins/scripts/determine_docker_images.py new file mode 100755 index 0000000000..9802ea784f --- /dev/null +++ b/ci/jenkins/scripts/determine_docker_images.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ruff: noqa: E501 +import argparse +import configparser +import datetime +import json +import logging +import urllib.error +from pathlib import Path +from typing import Any + +from http_utils import get + +REPO_ROOT = Path(__file__).resolve().parents[3] + +DOCKER_API_BASE = "https://hub.docker.com/v2/"; +PAGE_SIZE = 25 +TEST_DATA = None +IMAGE_TAGS_FILE = REPO_ROOT / "ci" / "jenkins" / "docker-images.ini" +TVM_CI_ECR = "477529581014.dkr.ecr.us-west-2.amazonaws.com" + + +def docker_api(url: str, use_pagination: bool = False) -> dict[str, Any]: + """ + Run a paginated fetch from the public Docker Hub API + """ + if TEST_DATA is not None: + if url not in TEST_DATA: + raise urllib.error.HTTPError(url, 404, "Not found", {}, None) + return TEST_DATA[url] + pagination = "" + if use_pagination: + pagination = f"?page_size={PAGE_SIZE}&page=1" + url = DOCKER_API_BASE + url + pagination + r, headers = get(url) + reset = headers.get("x-ratelimit-reset") + if reset is not None: + reset = datetime.datetime.fromtimestamp(int(reset)) + reset = reset.isoformat() + logging.info( + f"Docker API Rate Limit: {headers.get('x-ratelimit-remaining')} / {headers.get('x-ratelimit-limit')} (reset at {reset})" + ) + return r + + +def image_exists(spec: str) -> bool: + name, tag = spec.split(":") + try: + r = docker_api(f"repositories/{name}/tags/{tag}") + logging.debug(f"Image exists, got response: {json.dumps(r, indent=2)}") + return True + except urllib.error.HTTPError as e: + # Image was not found + logging.debug(e) + return False + + +def lookup_image_tag(name: str) -> str: + """Resolve image ``name`` (e.g. ``ci_cpu``) to its tag string from the ini. + + Pure ini read — no Docker Hub query, no tlcpackstaging fallback. Used by + ``docker/bash.sh`` for local-dev image-name shortcuts where the + full Hub-existence check is unnecessary. + """ + config = configparser.ConfigParser() + config.read(IMAGE_TAGS_FILE) + return config.get("jenkins", name) + + +if __name__ == "__main__": + logging.basicConfig( + format="[%(filename)s:%(lineno)d %(levelname)s] %(message)s", level=logging.INFO + ) + parser = argparse.ArgumentParser( + description="Writes out Docker images names to be used to .docker-image-names/" + ) + parser.add_argument( + "--lookup-only", + metavar="NAME", + help="Print the tag for NAME from the ini and exit (no Docker Hub query, no fallback).", + ) + parser.add_argument( + "--testing-docker-data", + help="(testing only) JSON data to mock response from Docker Hub API", + ) + parser.add_argument( + "--testing-images-data", + help=f"(testing only) JSON data to mock contents of {IMAGE_TAGS_FILE}", + ) + parser.add_argument( + "--base-dir", + default=".docker-image-names", + help="(testing only) Folder to write image names to", + ) + args, other = parser.parse_known_args() + + if args.lookup_only: + print(lookup_image_tag(args.lookup_only)) + raise SystemExit(0) + + name_dir = Path(args.base_dir) + + if args.testing_images_data: + repo_image_tags = json.loads(args.testing_images_data) + else: + config = configparser.ConfigParser() + config.read(IMAGE_TAGS_FILE) + repo_image_tags = {} + for name in other: + repo_image_tags[name] = config.get("jenkins", name) + + images = {} + for name in other: + images[name] = repo_image_tags[name] + + if args.testing_docker_data is not None: + TEST_DATA = json.loads(args.testing_docker_data) + + logging.info(f"Checking if these images exist in tlcpack: {images}") + + name_dir.mkdir(exist_ok=True) + images_to_use = {} + for filename, spec in images.items(): + if spec.startswith(TVM_CI_ECR): + logging.info(f"{spec} is from ECR") + images_to_use[filename] = spec + elif image_exists(spec): + logging.info(f"{spec} found in tlcpack") + images_to_use[filename] = spec + else: + logging.info(f"{spec} not found in tlcpack, using tlcpackstaging") + part, tag = spec.split(":") + user, repo = part.split("/") + tlcpackstaging_tag = f"tlcpackstaging/{repo.replace('-', '_')}:{tag}" + images_to_use[filename] = tlcpackstaging_tag + + for filename, image in images_to_use.items(): + logging.info(f"Writing image {image} to {name_dir / filename}") + with open(name_dir / filename, "w") as f: + f.write(image) diff --git a/ci/jenkins/scripts/git_skip_ci.py b/ci/jenkins/scripts/git_skip_ci.py new file mode 100755 index 0000000000..f7c84923f9 --- /dev/null +++ b/ci/jenkins/scripts/git_skip_ci.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import json +import logging +import os +import re +import subprocess +from urllib import request + + +def parse_remote(remote: str) -> tuple[str, str]: + """ + Get a GitHub (user, repo) pair out of a git remote + """ + if remote.startswith("https://";): + # Parse HTTP remote + parts = remote.split("/") + if len(parts) < 2: + raise RuntimeError(f"Unable to parse remote '{remote}'") + user, repo = parts[-2], parts[-1].replace(".git", "") + else: + # Parse SSH remote + m = re.search(r":(.*)/(.*)\.git", remote) + if m is None or len(m.groups()) != 2: + raise RuntimeError(f"Unable to parse remote '{remote}'") + user, repo = m.groups() + + user = os.getenv("DEBUG_USER", user) + repo = os.getenv("DEBUG_REPO", repo) + return user, repo + + +def git(command, **kwargs): + command = ["git", *command] + logging.info(f"Running {command}") + proc = subprocess.run(command, stdout=subprocess.PIPE, encoding="utf-8", **kwargs) + if proc.returncode != 0: + raise RuntimeError(f"Command failed {command}:\nstdout:\n{proc.stdout}") + return proc.stdout.strip() + + +if __name__ == "__main__": + help = "Exits with 0 if CI should be skipped, 1 otherwise" + parser = argparse.ArgumentParser(description=help) + parser.add_argument("--pr", required=True) + parser.add_argument("--remote", default="origin", help="ssh remote to parse") + parser.add_argument( + "--pr-title", help="(testing) PR title to use instead of fetching from GitHub" + ) + args = parser.parse_args() + logging.basicConfig( + format="[%(filename)s:%(lineno)d %(levelname)s] %(message)s", level=logging.INFO + ) + + branch = git(["rev-parse", "--abbrev-ref", "HEAD"]) + log = git(["log", "--format=%s", "-1"]) + + # Check the PR's title (don't check this until everything else passes first) + def check_pr_title(): + remote = git(["config", "--get", f"remote.{args.remote}.url"]) + user, repo = parse_remote(remote) + + if args.pr_title: + title = args.pr_title + else: + req = request.Request( + f"https://api.github.com/repos/{user}/{repo}/pulls/{args.pr}";, + headers={"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"}, + ) + with request.urlopen(req) as response: + pr = json.loads(response.read()) + title = pr["title"] + logging.info(f"pr title: {title}") + tags = [tag.strip() for tag in re.findall(r"\[(.*?)\]", title)] + logging.info(f"Found title tags: {tags}") + return "skip ci" in tags + + if args.pr != "null" and args.pr.strip() != "" and branch != "main" and check_pr_title(): + logging.info("PR title starts with '[skip ci]', skipping...") + exit(0) + else: + logging.info(f"Not skipping CI:\nargs.pr: {args.pr}\nbranch: {branch}\ncommit: {log}") + exit(1) diff --git a/ci/jenkins/scripts/git_skip_ci_globs.py b/ci/jenkins/scripts/git_skip_ci_globs.py new file mode 100755 index 0000000000..f7fc0860ec --- /dev/null +++ b/ci/jenkins/scripts/git_skip_ci_globs.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ruff: noqa: E501 + +import argparse +import fnmatch + +from git_skip_ci import git + +globs = [ + "*.md", + ".github/*", + ".asf.yaml", + ".gitignore", + "LICENSE", + "NOTICE", + "KEYS", + "tests/lint/*", + ".pre-commit-config.yaml", +] + + +def match_any(f: str) -> str | None: + for glob in globs: + if fnmatch.fnmatch(f, glob): + return glob + return None + + +if __name__ == "__main__": + help = "Exits with code 1 if a change only touched files, indicating that CI could be skipped for this changeset" + parser = argparse.ArgumentParser(description=help) + parser.add_argument("--files", help="(testing only) comma separated list of files to check") + args = parser.parse_args() + print(args) + if args.files is not None: + diff = [x for x in args.files.split(",") if x.strip() != ""] + else: + diff = git(["diff", "--no-commit-id", "--name-only", "-r", "origin/main"]) + diff = diff.split("\n") + diff = [d.strip() for d in diff] + diff = [d for d in diff if d != ""] + + print(f"Changed files:\n{diff}") + + if len(diff) == 0: + print("Found no changed files, skipping CI") + exit(0) + + print(f"Checking with globs:\n{globs}") + + for file in diff: + match = match_any(file) + if match is None: + print(f"{file} did not match any globs, running CI") + exit(1) + else: + print(f"{file} matched glob {match}") + + print("All files matched a glob, skipping CI") + exit(0) diff --git a/.github/workflows/lint.yml b/ci/jenkins/scripts/http_utils.py similarity index 60% copy from .github/workflows/lint.yml copy to ci/jenkins/scripts/http_utils.py index a9bff51f3b..05e1b71f24 100644 --- a/.github/workflows/lint.yml +++ b/ci/jenkins/scripts/http_utils.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,27 +16,19 @@ # specific language governing permissions and limitations # under the License. -name: Lint +import json +import logging +from typing import Any +from urllib import request -on: - push: - branches: [main] - pull_request: - branches: [main] -permissions: - contents: read +def get(url: str, headers: dict[str, str] | None = None) -> dict[str, Any]: + logging.info(f"Requesting GET to {url}") + if headers is None: + headers = {} + req = request.Request(url, headers=headers) + with request.urlopen(req) as response: + response_headers = {k: v for k, v in response.getheaders()} + response = json.loads(response.read()) -concurrency: - group: Lint-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/[email protected] - with: - fetch-depth: 0 - fetch-tags: true - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + return response, response_headers diff --git a/tests/scripts/task_build_arm.sh b/ci/jenkins/scripts/retry.sh old mode 100755 new mode 100644 similarity index 63% copy from tests/scripts/task_build_arm.sh copy to ci/jenkins/scripts/retry.sh index cb85fea9ea..933eb1aba6 --- a/tests/scripts/task_build_arm.sh +++ b/ci/jenkins/scripts/retry.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,10 +17,23 @@ # specific language governing permissions and limitations # under the License. -set -euxo pipefail +set -eux + +retry() { + local max_retries=$1 + shift + local n=0 + until [ "$n" -ge "$max_retries" ] + do + "$@" && break + n=$((n+1)) + if [ "$n" -eq "$max_retries" ]; then + echo "failed to update after attempt $n / $max_retries, giving up" + exit 1 + fi -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx + WAIT=$(python3 -c 'import random; print(random.randint(30, 200))') + echo "failed to update $n / $max_retries, waiting $WAIT to try again" + sleep "$WAIT" + done +} diff --git a/ci/jenkins/scripts/s3.py b/ci/jenkins/scripts/s3.py new file mode 100755 index 0000000000..25ae3f24af --- /dev/null +++ b/ci/jenkins/scripts/s3.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import glob +import importlib.util +import logging +import re +import subprocess +from enum import Enum +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] + +ARTIFACTS_PY = REPO_ROOT / "ci" / "jenkins" / "artifacts.py" + + +def load_artifacts(): + """Load the ARTIFACTS dict from ci/jenkins/artifacts.py.""" + spec = importlib.util.spec_from_file_location("artifacts", ARTIFACTS_PY) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.ARTIFACTS + + +def resolve_artifacts(artifact_names: list[str]) -> list[str]: + """Resolve a list of artifact names to a flat list of file paths.""" + artifacts = load_artifacts() + items = [] + for name in artifact_names: + if name not in artifacts: + known = list(artifacts.keys()) + logging.error(f"Unknown artifact '{name}'. Known artifacts: {known}") + raise SystemExit(1) + items.extend(artifacts[name]) + return items + + +def resolve_upload_items(items: list[str]) -> list[str]: + """Expand local artifact patterns once, preserving literal path handling.""" + paths = set() + for item in items: + path = Path(item) + if glob.has_magic(item): + matches = glob.glob(item, root_dir=REPO_ROOT, recursive=True) + if not matches: + logging.warning("No artifacts matched pattern: %s", item) + paths.update(Path(match).as_posix() for match in matches) + else: + # Keep missing literal paths for the existing optional-artifact warning. + paths.add(path.as_posix()) + directories = {Path(path) for path in paths if Path(path).is_dir()} + # Recursive directory copies already include descendants selected by other + # entries or overlapping patterns. Transfer each destination only once. + return sorted( + path for path in paths if not any(parent in directories for parent in Path(path).parents) + ) + + +RETRY_SCRIPT = REPO_ROOT / "ci" / "jenkins" / "scripts" / "retry.sh" +S3_DOWNLOAD_REGEX = re.compile(r"download: s3://.* to (.*)") + + +class Action(Enum): + UPLOAD = 1 + DOWNLOAD = 2 + + +def show_md5(item: str) -> None: + if not Path(item).is_dir(): + subprocess.run(["md5sum", item], check=True) + + +def parse_output_files(stdout: str) -> list[str]: + """ + Grab the list of downloaded files from the output of 'aws s3 cp'. Lines look + like: + + download: s3://some/prefix/a_file.txt to a_file.txt + """ + files = [] + for line in stdout.split("\n"): + line = line.strip() + if line == "": + continue + m = S3_DOWNLOAD_REGEX.match(line) + if m: + files.append(m.groups()[0]) + + return files + + +def chmod(files: list[str]) -> None: + """ + S3 has no concept of file permissions so add them back in here to every file + """ + # Add execute bit for downloads + to_chmod = [str(f) for f in files] + logging.info(f"Adding execute bit for files: {to_chmod}") + if len(to_chmod) > 0: + subprocess.run(["chmod", "+x", *to_chmod], check=True) + + +def s3(source: str, destination: str, recursive: bool) -> list[str]: + """ + Send or download the source to the destination in S3 + """ + # Retry each transfer independently; retain retry.sh's bounded random backoff. + command = [ + "bash", + "-c", + '. "$1"; shift; retry 3 "$@"', + "s3", + str(RETRY_SCRIPT), + "aws", + "s3", + "cp", + "--no-progress", + ] + if recursive: + command.append("--recursive") + command.extend([source, destination]) + logging.info("Copying %s to %s", source, destination) + with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process: + output = [] + for line in process.stdout: + print(line, end="", flush=True) + output.append(line) + stdout = "".join(output) + if process.wait(): + raise subprocess.CalledProcessError(process.returncode, command, stdout) + return stdout + + +if __name__ == "__main__": + logging.basicConfig( + format="[%(filename)s:%(lineno)d %(levelname)s] %(message)s", level=logging.INFO + ) + help = "Uploads or downloads files from S3" + parser = argparse.ArgumentParser(description=help) + parser.add_argument("--action", help="either 'upload' or 'download'", required=True) + parser.add_argument("--bucket", help="s3 bucket", required=True) + parser.add_argument( + "--prefix", help="s3 bucket + tag (e.g. s3://tvm-ci-prod/PR-1234/cpu", required=True + ) + parser.add_argument( + "--path", + dest="paths", + action="append", + help="repository-relative file, folder or upload glob pattern (repeatable)", + metavar="PATH", + ) + parser.add_argument( + "--artifact", + "--bundle", + dest="artifacts", + action="append", + help="artifact name from ci/jenkins/artifacts.py (repeatable)", + metavar="NAME", + ) + # Legacy --items accepts several paths and retains the last occurrence. + parser.add_argument("--items", nargs="+", help="files and folders to transfer") + + args = parser.parse_args() + if args.artifacts is not None and (args.paths is not None or args.items is not None): + parser.error("--artifact/--bundle and --path/--items are mutually exclusive") + + logging.info(args) + + if Path.cwd() != REPO_ROOT: + logging.error(f"s3.py can only be executed from the repo root, instead was in {Path.cwd()}") + exit(1) + + prefix = args.prefix.strip("/") + s3_path = f"s3://{args.bucket}/{prefix}" + logging.info(f"Using s3 path: {s3_path}") + + if args.action == "upload": + action = Action.UPLOAD + elif args.action == "download": + action = Action.DOWNLOAD + else: + logging.error(f"Unsupported action: {args.action}") + exit(1) + + if args.artifacts is not None: + items = resolve_artifacts(args.artifacts) + elif args.paths is not None or args.items is not None: + items = (args.paths or []) + (args.items or []) + else: + if args.action == "upload": + logging.error("Cannot upload without --path or --artifact") + exit(1) + else: + # Download the whole prefix + items = ["."] + + if action == Action.UPLOAD: + items = resolve_upload_items(items) + + for item in items: + if action == Action.DOWNLOAD: + source = s3_path + recursive = True + if item != ".": + source = s3_path + "/" + item + recursive = False + try: + stdout = s3(source=source, destination=item, recursive=recursive) + except Exception: + # Optional artifacts (e.g. per-backend device runtime DSOs) may not + # exist in S3 when the build config didn't produce them. Skip silently. + logging.warning(f"Download failed for {item}, skipping (may be optional)") + continue + files = parse_output_files(stdout) + chmod(files) + for file in files: + # Show md5 after downloading + show_md5(file) + elif action == Action.UPLOAD: + if not Path(item).exists(): + logging.warning(f"The path doesn't exist: {item}") + continue + show_md5(item) + if Path(item).is_dir(): + if len(list(Path(item).glob("**/*"))) == 0: + raise RuntimeError(f"Cannot upload empty folder with name: {item}") + s3(item, s3_path + "/" + item, recursive=Path(item).is_dir()) diff --git a/tests/scripts/task_build_arm.sh b/ci/jenkins/scripts/show_node_info.sh similarity index 55% copy from tests/scripts/task_build_arm.sh copy to ci/jenkins/scripts/show_node_info.sh index cb85fea9ea..af095ed453 100755 --- a/tests/scripts/task_build_arm.sh +++ b/ci/jenkins/scripts/show_node_info.sh @@ -18,8 +18,27 @@ set -euxo pipefail -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +echo "===== JENKINS INFO =====" +echo "NODE_NAME=$NODE_NAME" +echo "EXECUTOR_NUMBER=$EXECUTOR_NUMBER" +echo "WORKSPACE=$WORKSPACE" +echo "BUILD_NUMBER=$BUILD_NUMBER" +echo "WORKSPACE=$WORKSPACE" + +echo "===== EC2 INFO =====" +function ec2_metadata() { + # See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html + curl -w '\n' -fsSL "http://169.254.169.254/latest/meta-data/$1"; || echo failed +} + +ec2_metadata ami-id +ec2_metadata instance-id +ec2_metadata instance-type +ec2_metadata hostname +ec2_metadata public-hostname + +echo "===== RUNNER INFO =====" +df --human-readable +lscpu +free +nvidia-smi 2>/dev/null || echo "CUDA not found" diff --git a/.github/workflows/lint.yml b/ci/wheel/build-environment.yaml similarity index 51% copy from .github/workflows/lint.yml copy to ci/wheel/build-environment.yaml index a9bff51f3b..3b2c4dd167 100644 --- a/.github/workflows/lint.yml +++ b/ci/wheel/build-environment.yaml @@ -11,31 +11,32 @@ # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. -name: Lint +# Build environment for TVM wheel building. +# This environment provides the necessary dependencies for building TVM wheels. +name: tvm-wheel-build -on: - push: - branches: [main] - pull_request: - branches: [main] +# The conda channels to lookup the dependencies +channels: + - conda-forge -permissions: - contents: read - -concurrency: - group: Lint-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/[email protected] - with: - fetch-depth: 0 - fetch-tags: true - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 +# The packages to install to the environment +dependencies: + # Core build tools + - cmake >=3.24 + - ninja + - make + - llvmdev >=11 + - python >=3.10 + - pip + - git + - bzip2 + - zlib + - zstd-static + - pytest + - numpy + - scipy + - cython + - libxml2-devel diff --git a/ci/wheel/manylinux_build_libtvm_runtime_cuda.sh b/ci/wheel/manylinux_build_libtvm_runtime_cuda.sh new file mode 100755 index 0000000000..7701a12f13 --- /dev/null +++ b/ci/wheel/manylinux_build_libtvm_runtime_cuda.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Build libtvm_runtime_cuda.so inside a manylinux CUDA container, run by the +# build_cuda_runtime CI job. The official quay.io/manylinux_cuda images ship +# the CUDA toolkit preinstalled under /usr/local/cuda, but omit the libcuda +# driver stub, so we install just cuda-driver-devel from the image's CUDA repo. +# Builds the sidecar into build-wheel-cuda/lib/ for the wheel build to bundle. +# +# Usage: manylinux_build_libtvm_runtime_cuda.sh +set -euxo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +build_dir="${repo_root}/build-wheel-cuda" +parallel="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)" + +# The image ships the toolkit but not the libcuda driver stub; install it from +# the image's CUDA repo so the sidecar links libcuda.so.1 (sync version with tag). +dnf -y install cuda-driver-devel-13-1 + +# Build the CUDA runtime sidecar with CUDA on and LLVM off, so it does not need +# the LLVM prefix; the main CPU wheel links LLVM statically. The manylinux CUDA +# image already ships cmake and make, and the build uses the default Makefiles +# generator (no Ninja), so no build tools are installed here. Put the bundled +# CPython and CUDA toolchain on PATH for the CMake configure and nvcc. +export PATH="/opt/python/cp310-cp310/bin:/usr/local/cuda/bin:${PATH}" +python -m pip install "apache-tvm-ffi>=0.1.15rc0" +nvcc --version + +rm -rf "${build_dir}" +# CMAKE_CUDA_COMPILER only tells CMake which nvcc to use; it does not affect the +# resulting libtvm_runtime_cuda.so, which is built only from .cc host sources (no +# .cu device code, so nvcc is never invoked for it). CMAKE_CUDA_ARCHITECTURES is +# intentionally not set: it would be a no-op here for the same reason (verified -- +# the .so is byte-identical across arch values and carries no device code), and +# modern CMake fills in a default so configure does not fail without it. +cmake -S "${repo_root}" -B "${build_dir}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DTVM_TIRX_BUILD_TESTS=OFF \ + -DTVM_TIRX_BUILD_PYTHON_MODULE=ON \ + -DTVM_TIRX_USE_CUDA=/usr/local/cuda \ + -DTVM_TIRX_USE_LLVM=OFF \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc +cmake --build "${build_dir}" --target tvm_runtime tvm_runtime_cuda --parallel "${parallel}" + +cuda_lib="${build_dir}/lib/libtvm_runtime_cuda.so" +test -f "${cuda_lib}" +patchelf --set-rpath '$ORIGIN' "${cuda_lib}" +echo "CUDA runtime: ${cuda_lib}" diff --git a/ci/wheel/windows_build_libtvm_runtime_cuda.bat b/ci/wheel/windows_build_libtvm_runtime_cuda.bat new file mode 100644 index 0000000000..d4fa9f38e9 --- /dev/null +++ b/ci/wheel/windows_build_libtvm_runtime_cuda.bat @@ -0,0 +1,99 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one +rem or more contributor license agreements. See the NOTICE file +rem distributed with this work for additional information +rem regarding copyright ownership. The ASF licenses this file +rem to you under the Apache License, Version 2.0 (the +rem "License"); you may not use this file except in compliance +rem with the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, +rem software distributed under the License is distributed on an +rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +rem KIND, either express or implied. See the License for the +rem specific language governing permissions and limitations +rem under the License. +rem +rem Build tvm_runtime_cuda.dll on a Windows runner, run by the build_cuda_runtime +rem CI job (on the host; unlike Linux there is no build container on Windows). +rem Installs the pinned CUDA toolkit via conda and builds the sidecar into +rem build-wheel-cuda\lib\ for the wheel build to bundle. Windows mirror of +rem manylinux_build_libtvm_runtime_cuda.sh. Run with: shell: cmd. +setlocal enableextensions + +rem repo root = this script's directory / ..\.. (native Windows paths; no cygpath). +pushd "%~dp0..\.." || exit /b 1 +set "repo_root=%CD%" +popd +set "build_dir=%repo_root%\build-wheel-cuda" +set "cuda_prefix=C:\opt\cuda" + +rem Locate conda: the runner ships Miniconda (exposed via %CONDA%) but it may not be +rem on PATH in this shell. +set "conda_exe=conda" +where conda >nul 2>nul || set "conda_exe=%CONDA%\Scripts\conda.exe" + +rem Install the pinned CUDA toolkit via conda from the nvidia channel, mirroring the +rem LLVM-via-conda install used elsewhere. The win-64 channel caps at 13.0.x, so this +rem pins 13.0.2 -- slightly behind the Linux image's CUDA 13.1, which is harmless: the +rem sidecar has no device code and links the CUDA runtime by soname only. The nvidia +rem CDN occasionally returns a transient HTTP 5xx, so retry once; a half-finished first +rem attempt can leave the prefix partially populated, so wipe it before retrying. +if not exist "%cuda_prefix%\Library\bin\nvcc.exe" ( + call "%conda_exe%" create -q -p "%cuda_prefix%" -c nvidia/label/cuda-13.0.2 cuda-toolkit -y + if errorlevel 1 ( + if exist "%cuda_prefix%" rmdir /s /q "%cuda_prefix%" + call "%conda_exe%" create -q -p "%cuda_prefix%" -c nvidia/label/cuda-13.0.2 cuda-toolkit -y || exit /b 1 + ) +) + +rem conda lays the Windows toolkit out under <prefix>\Library (bin\nvcc.exe, +rem lib\x64\cudart.lib, include\...). Discover the root from nvcc.exe so TVM's +rem FindCUDA MSVC branch resolves against the real layout instead of a hardcode. +set "nvcc_exe=" +for /f "delims=" %%i in ('dir /s /b "%cuda_prefix%\nvcc.exe" 2^>nul') do if not defined nvcc_exe set "nvcc_exe=%%i" +if not defined nvcc_exe ( echo nvcc.exe not found under %cuda_prefix% & exit /b 1 ) +rem cuda_root = dirname(dirname(nvcc)) = <prefix>\Library +for %%i in ("%nvcc_exe%") do set "nvcc_bin=%%~dpi" +pushd "%nvcc_bin%.." || exit /b 1 +set "cuda_root=%CD%" +popd +set "CUDA_PATH=%cuda_root%" + +python -m pip install -U pip cmake ninja "apache-tvm-ffi>=0.1.15rc0" || exit /b 1 +"%nvcc_exe%" --version || exit /b 1 + +rem nvcc needs the MSVC host compiler (cl.exe), so locate VS via vswhere and run the +rem cmake configure+build inside vcvars64 (this shell is not a VS Developer prompt). +set "vswhere=C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" +set "vs_path=" +for /f "usebackq delims=" %%i in (`"%vswhere%" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set "vs_path=%%i" +if not defined vs_path ( echo Visual Studio with VC tools not found & exit /b 1 ) +set "vcvars=%vs_path%\VC\Auxiliary\Build\vcvars64.bat" + +if exist "%build_dir%" rmdir /s /q "%build_dir%" + +rem CMAKE_CUDA_COMPILER only tells CMake which nvcc to use (load-bearing: the conda +rem nvcc is not on PATH); it does not affect the resulting tvm_runtime_cuda.dll, which +rem is built only from .cc host sources (no .cu device code). CMAKE_CUDA_ARCHITECTURES +rem is intentionally not set -- a no-op for the same reason, and modern CMake fills a +rem default. -allow-unsupported-compiler guards against the runner's MSVC being newer +rem than the CUDA toolkit officially supports. The cmake command is kept on one line: +rem `^` continuations in a batch file break on any trailing whitespace. +rem CMake parses backslashes in string values as escapes (e.g. C:\opt -> invalid \o), +rem so hand cmake forward-slash paths. cmd builtins (rmdir / if exist) keep backslashes. +set "repo_root_fwd=%repo_root:\=/%" +set "build_dir_fwd=%build_dir:\=/%" +set "cuda_root_fwd=%cuda_root:\=/%" +set "nvcc_fwd=%nvcc_exe:\=/%" + +call "%vcvars%" || exit /b 1 +cmake -S "%repo_root_fwd%" -B "%build_dir_fwd%" -G Ninja -DCMAKE_BUILD_TYPE=Release -DTVM_TIRX_BUILD_TESTS=OFF -DTVM_TIRX_BUILD_PYTHON_MODULE=ON -DTVM_TIRX_USE_CUDA="%cuda_root_fwd%" -DTVM_TIRX_USE_LLVM=OFF -DCMAKE_CUDA_COMPILER="%nvcc_fwd%" -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler" || exit /b 1 +cmake --build "%build_dir_fwd%" --target tvm_runtime tvm_runtime_cuda --config Release || exit /b 1 + +if not exist "%build_dir%\lib\tvm_runtime_cuda.dll" ( echo tvm_runtime_cuda.dll was not produced & exit /b 1 ) +rem No patchelf/rpath step on Windows; delvewheel vendors dependencies at repair time. +echo CUDA runtime: %build_dir%\lib\tvm_runtime_cuda.dll +endlocal diff --git a/docker/bash.sh b/docker/bash.sh new file mode 100755 index 0000000000..ccd79a6379 --- /dev/null +++ b/docker/bash.sh @@ -0,0 +1,669 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# +# Start a bash, mount REPO_MOUNT_POINT to be current directory. +# +# Usage: docker/bash.sh [-i|--interactive] [--net=host] [-t|--tty] +# [--shell SHELL] [--mount MOUNT_DIR] +# [--repo-mount-point REPO_MOUNT_POINT] +# [--dry-run] [--name NAME] [--privileged] +# <DOCKER_IMAGE_NAME> [--] [COMMAND] +# +# Usage: docker/bash.sh <CONTAINER_NAME> +# Starts an interactive session +# +# Usage2: docker/bash.sh [-i] <CONTAINER_NAME> [COMMAND] +# Execute command in the docker image, default non-interactive +# With -i, execute interactively. +# + +set -euo pipefail + +############################################## +### Section 1: Constants and defaults ### +############################################## + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_DIR="$(dirname "${SCRIPT_DIR}")" + +DRY_RUN=false +INTERACTIVE=false +TTY=false +USE_NET_HOST=false +USE_GPU=true +DOCKER_IMAGE_NAME= +COMMAND=bash +MOUNT_DIRS=( ) +CONTAINER_NAME= +USER_SHELL= + +# TODO(Lunderberg): Remove this if statement and always set to +# "${REPO_DIR}". The consistent directory for Jenkins is currently +# necessary to allow CMake build commands to run in CI after the build +# steps. +# TODO(https://github.com/apache/tvm/issues/11952): +# Figure out a better way to keep the same path +# between build and testing stages. +if [[ -n "${JENKINS_HOME:-}" ]]; then + REPO_MOUNT_POINT=/workspace +else + REPO_MOUNT_POINT="${REPO_DIR}" +fi + +############################################## +### Section 2: Helper functions ### +############################################## + +function show_usage() { + cat <<EOF +Usage: docker/bash.sh [-i|--interactive] [--net=host] [-t|--tty] + [--shell SHELL] [--cpus NUM_CPUS] [--mount MOUNT_DIR] + [--repo-mount-point REPO_MOUNT_POINT] + [--dry-run] [--name NAME] + <DOCKER_IMAGE_NAME> [--] [COMMAND] + + +-h, --help + + Display this help message. + +-i, --interactive + + Start the docker session in interactive mode. + +-t, --tty + + Start the docker session with a pseudo terminal (tty). + +--shell SHELL + + Specify a shell (e.g. --shell zsh) to use instead of bash + when entering auto-interactive mode (no command or command + is "bash"). + +--cpus NUM_CPUS + + Limit the number of CPU cores to be used. + +--net=host + + Expose servers run into the container to the host, passing the + "--net=host" argument through to docker. On MacOS, this is + instead passed as "-p 8888:8888" since the host networking driver + isn't supported. + +--mount MOUNT_DIR + + Expose MOUNT_DIR as an additional mount point inside the docker + container. The mount point inside the container is the same as + the folder location outside the container. This option can be + specified multiple times. + +--repo-mount-point REPO_MOUNT_POINT + + The directory inside the docker container at which the TVM + repository should be mounted, and is used as the workspace inside + the docker container. + + If unspecified, the mount location depends on the environment. If + running inside Jenkins, the mount location will be /workspace. + Otherwise, the mount location of the repository will be the same + as the external location of the repository, to maintain + compatibility with git-worktree. + +--no-gpu + + Do not use GPU device drivers even if using an CUDA Docker image + +--dry-run + + Print the docker command to be run, but do not execute it. + +--build-dir BUILD_DIR + + The build directory of TVM. This is appended to LD_LIBRARY_PATH + +--env + + Pass an environment variable through to the container. + +--name + + Set the name of the docker container, and the hostname that will + appear inside the container. + +--privileged + + Give extended privileges to this container. + +DOCKER_IMAGE_NAME + + The name of the docker container to be run. This can be an + explicit name of a docker image (e.g. "tlcpack/ci-gpu:v0.76") or + can be a shortcut as defined in the TVM Jenkinsfile + (e.g. "ci_gpu"). + +COMMAND + + The command to be run inside the docker container. If this is set + to "bash", the --interactive, --tty and --net=host flags are set. + If no command is specified, defaults to "bash". If the command + contains dash-prefixed arguments, the command should be preceded + by -- to indicate arguments that are not intended for bash.sh. + +Environment Variables: + +TVM_DEV_DOCKER_MOUNTS + + Space-separated list of additional mount specifications. + Each entry is either: + /path/to/dir - bind mount at same path inside container + /src:/dst - bind mount src to dst inside container + Source paths are validated; missing sources are skipped with a warning. + +EOF +} + +function parse_error() { + echo "$@" >&2 + show_usage >&2 + exit 1 +} + +# Detect if Docker daemon is running in rootless mode. +# Uses DOCKER_IS_ROOTLESS env var as override (for CI), otherwise auto-detects. +# Result is cached after first call. +_ROOTLESS_CACHED= +_ROOTLESS_RESULT=1 +function detect_rootless() { + if [ -n "${_ROOTLESS_CACHED}" ]; then + return $_ROOTLESS_RESULT + fi + _ROOTLESS_CACHED=1 + if [ -n "${DOCKER_IS_ROOTLESS:-}" ]; then + # env var override for CI + _ROOTLESS_RESULT=0 + return 0 + fi + if docker info -f '{{.SecurityOptions}}' 2>/dev/null | grep -q "rootless"; then + _ROOTLESS_RESULT=0 + return 0 + fi + _ROOTLESS_RESULT=1 + return 1 +} + +# Validate a mount source path exists, warn if missing. +# Usage: validate_mount /path/to/source +# Returns 0 if source exists, 1 if missing. +function validate_mount() { + local src="$1" + if [ -e "$src" ]; then + return 0 + else + echo "Warning: Mount source '$src' not found on host, skipping." >&2 + return 1 + fi +} + + +############################################## +### Section 3: Argument parsing ### +############################################## + +# Handle joined flags, such as interpreting -ih as -i -h. Either rewrites +# the current argument if it is a joined argument, or shifts all arguments +# otherwise. Should be called as "eval $break_joined_flag" where joined +# flags are possible. Can't use a function definition, because it needs +# to overwrite the parent scope's behavior. +break_joined_flag='if (( ${#1} == 2 )); then shift; else set -- -"${1#-i}" "${@:2}"; fi' + +DOCKER_ENV=( ) +# Forward build settings selected by the caller without overriding explicit values. +BUILD_ENV_VARS=(SCCACHE_BUCKET SCCACHE_REGION CMAKE_BUILD_PARALLEL_LEVEL + CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER) +for build_var in "${BUILD_ENV_VARS[@]}"; do + if [ "${!build_var+x}" = "x" ]; then + DOCKER_ENV+=( --env "$build_var=${!build_var}" ) + fi +done +DOCKER_FLAGS=( ) + +while (( $# )); do + case "$1" in + -h|--help) + show_usage + exit 0 + ;; + + -i*|--interactive) + INTERACTIVE=true + eval $break_joined_flag + ;; + + -t*|--tty) + TTY=true + if [[ "$1" == "--tty" ]]; then + shift + else + # Short form: -t or combined like -th + eval $break_joined_flag + fi + ;; + + --shell) + if (( $# >= 2 )) && [[ -n "$2" ]]; then + USER_SHELL="$2" + shift 2 + else + parse_error 'ERROR: --shell requires a non-empty argument' + fi + ;; + + --cpus) + DOCKER_FLAGS+=(--cpus "$2") + shift 2 + ;; + + --net=host) + USE_NET_HOST=true + shift + ;; + + --net) + DOCKER_FLAGS+=( --net "$2" ) + shift 2 + ;; + + --mount) + if [[ -n "$2" ]]; then + MOUNT_DIRS+=("$2") + shift 2 + else + parse_error 'ERROR: --mount requires a non-empty argument' + fi + ;; + + --mount=?*) + MOUNT_DIRS+=("${1#*=}") + shift + ;; + + --name) + if [[ -n "$2" ]]; then + CONTAINER_NAME="$2" + shift 2 + else + parse_error 'ERROR: --name requires a non empty argument' + fi + ;; + + --privileged) + DOCKER_FLAGS+=( "--privileged" ) + shift 1 + ;; + + --env) + DOCKER_ENV+=( --env "$2" ) + shift 2 + ;; + + --volume) + DOCKER_FLAGS+=( --volume "$2" ) + shift 2 + ;; + + -e) + DOCKER_ENV+=( --env "$2" ) + shift 2 + ;; + + -v) + DOCKER_FLAGS+=( --volume "$2" ) + shift 2 + ;; + + --dry-run) + DRY_RUN=true + shift + ;; + + --no-gpu) + USE_GPU=false + shift + ;; + + --repo-mount-point) + if [[ -n "$2" ]]; then + REPO_MOUNT_POINT="$2" + shift 2 + else + parse_error 'ERROR: --repo-mount-point requires a non-empty argument' + fi + ;; + + --repo-mount-point=?*) + REPO_MOUNT_POINT="${1#*=}" + shift + ;; + + --build-dir) + DOCKER_ENV+=( --env LD_LIBRARY_PATH=${REPO_MOUNT_POINT}/${2}) + shift 2 + ;; + + --) + shift + COMMAND=( "$@" ) + break + ;; + + -*|--*) + echo "Error: Unknown flag: $1" >&2 + echo " If this flag is intended to be passed to the" >&2 + echo " docker command, please add -- before the docker" >&2 + echo " command (e.g. docker/bash.sh ci_gpu -- build -j2)" >&2 + show_usage >&2 + exit 1 + ;; + + *) + # First positional argument is the image name, all + # remaining below to the COMMAND. + if [[ -z "${DOCKER_IMAGE_NAME}" ]]; then + DOCKER_IMAGE_NAME=$1 + shift + else + COMMAND=( "$@" ) + break + fi + ;; + esac +done + +if [[ -z "${DOCKER_IMAGE_NAME}" ]]; then + echo "Error: Missing DOCKER_IMAGE_NAME" >&2 + show_usage >&2 +fi + +# Smart defaults: "bash" command triggers interactive mode +if [[ ${COMMAND[@]+"${COMMAND[@]}"} = bash ]]; then + INTERACTIVE=true + TTY=true +fi + + +############################################## +### Section 4: Image resolution ### +############################################## + +function lookup_image_spec() { + local image_spec + image_spec=$(python3 "${REPO_DIR}/ci/jenkins/scripts/determine_docker_images.py" \ + --lookup-only "$1" 2>/dev/null) || return 0 + printf '%s\n' "$image_spec" +} + +DOCKER_MOUNT=( ) +DOCKER_DEVICES=( ) +# If the user gave a shortcut defined in the Jenkinsfile, use it. +EXPANDED_SHORTCUT=$(lookup_image_spec "${DOCKER_IMAGE_NAME}") +if [ -n "${EXPANDED_SHORTCUT}" ]; then + if [ "${CI+x}" == "x" ]; then + DOCKER_IMAGE_NAME="${EXPANDED_SHORTCUT}" + else + python3 "${REPO_DIR}/ci/jenkins/scripts/determine_docker_images.py" \ + --base-dir "${REPO_DIR}/.docker-image-names" "$DOCKER_IMAGE_NAME" 2> /dev/null + DOCKER_IMAGE_NAME=$(cat "${REPO_DIR}/.docker-image-names/$DOCKER_IMAGE_NAME") + if [[ "$DOCKER_IMAGE_NAME" == *"tlcpackstaging"* ]]; then + echo "WARNING: resolved docker image to fallback tag in tlcpackstaging" >&2 + fi + fi +fi + + +############################################## +### Section 5: Mount construction ### +############################################## + +# Working directories +DOCKER_FLAGS+=( --workdir "${REPO_MOUNT_POINT}" ) +DOCKER_MOUNT+=( --volume "${REPO_DIR}":"${REPO_MOUNT_POINT}" + ) + +# Expose external directories from --mount flags +for MOUNT_DIR in ${MOUNT_DIRS[@]+"${MOUNT_DIRS[@]}"}; do + DOCKER_MOUNT+=( --volume "${MOUNT_DIR}:${MOUNT_DIR}" ) +done + +# Parse TVM_DEV_DOCKER_MOUNTS env var: space-separated mount specs +# Each entry is either /path (same path) or /src:/dst +if [[ -n "${TVM_DEV_DOCKER_MOUNTS:-}" ]]; then + for mount_spec in ${TVM_DEV_DOCKER_MOUNTS}; do + if [[ "$mount_spec" == *:* ]]; then + # src:dst format + mount_src="${mount_spec%%:*}" + if validate_mount "$mount_src"; then + DOCKER_MOUNT+=( --volume "$mount_spec" ) + fi + else + # Same path format + if validate_mount "$mount_spec"; then + DOCKER_MOUNT+=( --volume "$mount_spec:$mount_spec" ) + fi + fi + done +fi + +# When running from a git worktree, also mount the original git dir. +if [ -f "${REPO_DIR}/.git" ]; then + git_dir=$(cd ${REPO_DIR} && git rev-parse --git-common-dir) + if [ "${git_dir}" != "${REPO_DIR}/.git" ]; then + DOCKER_MOUNT+=( --volume "${git_dir}:${git_dir}" ) + fi +fi + + +############################################## +### Section 6: Environment construction ### +############################################## + +# CI-specific environment variables +DOCKER_ENV+=( --env CI_BUILD_HOME="${REPO_MOUNT_POINT}" + --env CI_BUILD_USER="$(id -u -n)" + --env CI_BUILD_UID="$(id -u)" + --env CI_BUILD_GROUP="$(id -g -n)" + --env CI_BUILD_GID="$(id -g)" + --env CI_PYTEST_ADD_OPTIONS="${CI_PYTEST_ADD_OPTIONS:-}" + --env CI_IMAGE_NAME="${DOCKER_IMAGE_NAME}" + ) + +# Remove the container once it finishes running (--rm). +DOCKER_FLAGS+=(--rm) + +# Share the PID namespace (--pid=host). The process inside does not +# have pid 1 and SIGKILL is propagated to the process inside, allowing +# jenkins to kill it if needed. This is only necessary for docker +# daemons running as root. +if ! detect_rootless; then + DOCKER_FLAGS+=(--pid=host) +fi + +# Expose services running in container to the host. +if $USE_NET_HOST; then + if [[ $(uname) == "Darwin" ]]; then + # Docker's host networking driver isn't supported on macOS. + # Use default bridge network and expose port for jupyter notebook. + DOCKER_FLAGS+=( -p 8888:8888 ) + else + DOCKER_FLAGS+=(--net=host) + fi +fi + +# Set up interactive sessions +if ${INTERACTIVE}; then + DOCKER_FLAGS+=( --interactive ) +fi + +if ${TTY}; then + DOCKER_FLAGS+=( --tty ) +fi + +# Setup the docker name and the hostname inside the container +if [[ ! -z "${CONTAINER_NAME}" ]]; then + DOCKER_FLAGS+=( --name ${CONTAINER_NAME} --hostname ${CONTAINER_NAME}) +fi + +# Pass any restrictions of allowed CUDA devices from the host to the +# docker container. +if [[ -n ${CUDA_VISIBLE_DEVICES:-} ]]; then + DOCKER_ENV+=( --env CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" ) +fi + +# Set TVM import path inside the docker image +if [[ "${DOCKER_IMAGE_NAME}" == *"ci"* ]]; then + DOCKER_ENV+=( --env PYTHONPATH="${REPO_MOUNT_POINT}/python" ) +fi + + +############################################## +### Section 7: GPU/device setup ### +############################################## + +DOCKER_BINARY=docker + +# Use --gpus all for GPU/CUDA containers +if [[ "$USE_GPU" == "true" ]] && [[ "${DOCKER_IMAGE_NAME}" == *"gpu"* || "${DOCKER_IMAGE_NAME}" == *"cuda"* ]]; then + DOCKER_FLAGS+=( --gpus all ) + DOCKER_ENV+=( --env NVIDIA_DRIVER_CAPABILITIES=compute,graphics,utility ) +fi + +# Add ROCm devices and set ROCM_ENABLED=1 which is used in the with_the_same_user script +# to add the user to the video group +if [[ "${DOCKER_IMAGE_NAME}" == *"rocm"* && -d "/dev/dri" ]]; then + DOCKER_DEVICES+=( --device=/dev/kfd --device=/dev/dri ) + DOCKER_ENV+=( --env ROCM_ENABLED=1 ) +fi + + +############################################## +### Section 8: Entry command ### +############################################## + +# Apply --shell override: when command is "bash" (auto-interactive) +# and a custom shell was requested via --shell <name>, use that shell +if [[ ${COMMAND[@]+"${COMMAND[@]}"} = bash ]] && [[ -n "$USER_SHELL" ]]; then + COMMAND=( "$USER_SHELL" ) +fi + +# Build the entry command +if ! detect_rootless; then + # Prefer the image helper, with an inline fallback for images that previously + # obtained it from the host docker/ bind mount. + SAME_USER_SCRIPT=$(cat <<'SAME_USER_EOF' +set -e +if [ -f /docker/with_the_same_user ]; then + exec bash --login /docker/with_the_same_user "$@" +fi + +# NOTE: sudo uses the env_reset option to reset environment variables to a secure bare minimum. +# The --preserve-env option below passes those variables through to the invoked process; however, +# this appears not to affect the environment used with execve, so we resolve the binary to run +# in this file using the $PATH specified in the Dockerfile. +COMMAND=( "$(which "$1")" ) +shift +COMMAND=( "${COMMAND[@]}" "$@" ) + +if ! touch /this_is_writable_file_system; then + echo "You can't write to your filesystem!" + echo "If you are in Docker you should check you do not have too many images" \ + "with too many files in them. Docker has some issue with it." + exit 1 +else + rm /this_is_writable_file_system +fi + +getent group "${CI_BUILD_GID}" || ( + # Ensure "${CI_BUILD_GROUP}" is not already some other gid inside container. + if grep -q "^${CI_BUILD_GROUP}:" /etc/group; then + CI_BUILD_GROUP="${CI_BUILD_GROUP}2" + fi + addgroup --force-badname --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}" >/dev/null) + +getent group tvm-venv || (addgroup tvm-venv >/dev/null) +getent passwd "${CI_BUILD_UID}" || adduser --force-badname --gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \ + --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \ + --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}" +usermod -a -G sudo -G tvm-venv "${CI_BUILD_USER}" +usermod -a -G sudo -G dialout "${CI_BUILD_USER}" + +# Add user to video group for ROCm +if [[ ! -z "${ROCM_ENABLED-}" ]]; then + usermod -a -G video "${CI_BUILD_USER}" +fi + +# This is a grotesque hack to get PYTEST_ADD_OPTS available to all task scripts. +echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo + +if [[ ! -z "${CUDA_VISIBLE_DEVICES-}" ]]; then + CUDA_ENV="CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" +else + CUDA_ENV="" +fi + +sudo -u "#${CI_BUILD_UID}" --preserve-env \ +${CUDA_ENV} \ +PATH=${PATH} \ +LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}" \ +PYTHONPATH="${PYTHONPATH-}" \ +CI_IMAGE_NAME="${CI_IMAGE_NAME-}" \ +HOME="${CI_BUILD_HOME-}" \ +"${COMMAND[@]}" +SAME_USER_EOF + ) + COMMAND=(bash --login -c "${SAME_USER_SCRIPT}" -- "${COMMAND[@]}") +fi + + +############################################## +### Section 9: Final docker run ### +############################################## + +# Print arguments. +echo "REPO_DIR: ${REPO_DIR}" +echo "DOCKER CONTAINER NAME: ${DOCKER_IMAGE_NAME}" +echo "" + +echo Running \'${COMMAND[@]+"${COMMAND[@]}"}\' inside ${DOCKER_IMAGE_NAME}... + +DOCKER_CMD=(${DOCKER_BINARY} run + ${DOCKER_FLAGS[@]+"${DOCKER_FLAGS[@]}"} + ${DOCKER_ENV[@]+"${DOCKER_ENV[@]}"} + ${DOCKER_MOUNT[@]+"${DOCKER_MOUNT[@]}"} + ${DOCKER_DEVICES[@]+"${DOCKER_DEVICES[@]}"} + "${DOCKER_IMAGE_NAME}" + ${COMMAND[@]+"${COMMAND[@]}"} + ) + +if ${DRY_RUN}; then + echo ${DOCKER_CMD[@]+"${DOCKER_CMD[@]}"} +else + ${DOCKER_CMD[@]+"${DOCKER_CMD[@]}"} +fi diff --git a/tests/scripts/task_build_arm.sh b/tests/scripts/task_build_arm.sh index cb85fea9ea..576687813f 100755 --- a/tests/scripts/task_build_arm.sh +++ b/tests/scripts/task_build_arm.sh @@ -19,7 +19,10 @@ set -euxo pipefail export CMAKE_ARGS=" + -DTVM_TIRX_BUILD_TESTS=ON -DTVM_TIRX_USE_LLVM=llvm-config-17 -DCMAKE_CXX_FLAGS=-Werror " -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist . + +./build/tirx_cpptest diff --git a/tests/scripts/task_build_cpu.sh b/tests/scripts/task_build_cpu.sh index 3583a06391..2e5e540097 100755 --- a/tests/scripts/task_build_cpu.sh +++ b/tests/scripts/task_build_cpu.sh @@ -19,7 +19,10 @@ set -euxo pipefail export CMAKE_ARGS=" + -DTVM_TIRX_BUILD_TESTS=ON -DTVM_TIRX_USE_LLVM='/usr/bin/llvm-config-17 --link-static' -DCMAKE_CXX_FLAGS='-Wno-error=range-loop-construct -Wno-error=comment' " -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist . + +./build/tirx_cpptest diff --git a/tests/scripts/task_build_gpu.sh b/tests/scripts/task_build_gpu.sh index 4eb7a17385..eb7129baba 100755 --- a/tests/scripts/task_build_gpu.sh +++ b/tests/scripts/task_build_gpu.sh @@ -19,6 +19,7 @@ set -euxo pipefail export CMAKE_ARGS=" + -DTVM_TIRX_BUILD_TESTS=ON -DTVM_TIRX_USE_CUDA=ON -DTVM_TIRX_USE_VULKAN=ON -DTVM_TIRX_USE_OPENCL=ON @@ -27,4 +28,6 @@ export CMAKE_ARGS=" -DCMAKE_CXX_FLAGS=-Werror -DCMAKE_CUDA_ARCHITECTURES=75 " -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist . + +./build/tirx_cpptest diff --git a/tests/scripts/task_build_arm.sh b/tests/scripts/task_build_gpu_other.sh similarity index 61% copy from tests/scripts/task_build_arm.sh copy to tests/scripts/task_build_gpu_other.sh index cb85fea9ea..2ed4f1302c 100755 --- a/tests/scripts/task_build_arm.sh +++ b/tests/scripts/task_build_gpu_other.sh @@ -16,10 +16,13 @@ # specific language governing permissions and limitations # under the License. +# This file is a compiler test to ensure that runtimes can compile +# correctly, even if they aren't actively tested in the CI. set -euxo pipefail -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +# Preserve the separate runtime-only GPU compile check. +uv pip install --python python3 --no-deps --target build/python ./3rdparty/tvm-ffi +export PYTHONPATH="$PWD/build/python" +cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DTVM_TIRX_USE_OPENCL=ON -DTVM_TIRX_USE_ROCM=ON -DCMAKE_CXX_FLAGS=-Werror +cmake --build build --target tvm_runtime tvm_runtime_opencl tvm_runtime_rocm diff --git a/tests/scripts/task_build_arm.sh b/tests/scripts/task_install_build_dependencies.sh similarity index 71% copy from tests/scripts/task_build_arm.sh copy to tests/scripts/task_install_build_dependencies.sh index cb85fea9ea..2b84c57c7a 100755 --- a/tests/scripts/task_build_arm.sh +++ b/tests/scripts/task_install_build_dependencies.sh @@ -18,8 +18,10 @@ set -euxo pipefail -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +rm -rf dist +uv pip install --python python3 --target .local/python \ + "scikit-build-core>=0.11,!=1.1.0" "setuptools-scm>=8" "cython>=3.2.8" "mlc-z3-static>=4.16.0" +export PYTHONPATH="$PWD/.local/python" + +uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./3rdparty/tvm-ffi +uv pip install --python python3 --target .local/python --no-deps --offline --reinstall dist/apache_tvm_ffi-*.whl diff --git a/tests/scripts/task_build_arm.sh b/tests/scripts/task_python_docs.sh similarity index 65% copy from tests/scripts/task_build_arm.sh copy to tests/scripts/task_python_docs.sh index cb85fea9ea..21d205d26d 100755 --- a/tests/scripts/task_build_arm.sh +++ b/tests/scripts/task_python_docs.sh @@ -18,8 +18,10 @@ set -euxo pipefail -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +source ./tests/scripts/test_task_install_dist_wheels.sh +uv pip install --python python3 --target .local/python -r docs/requirements.txt +export TVM_TIRX_LIBRARY_PATH="$PWD/.local/python/tvm_tirx/lib" +rm -rf docs/_build docs/reference/python/generated +python3 -m sphinx -E -a -b html -d docs/_build/doctrees -W --keep-going docs docs/_build/html +git rev-parse HEAD > docs/_build/html/commit_hash +tar -czf docs.tgz -C docs/_build/html . diff --git a/tests/scripts/task_build_arm.sh b/tests/scripts/task_python_unittest.sh similarity index 79% copy from tests/scripts/task_build_arm.sh copy to tests/scripts/task_python_unittest.sh index cb85fea9ea..e71f603388 100755 --- a/tests/scripts/task_build_arm.sh +++ b/tests/scripts/task_python_unittest.sh @@ -17,9 +17,9 @@ # under the License. set -euxo pipefail +export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +if [ "$#" -eq 0 ]; then + set -- tests/python +fi +python3 -m pytest -vvs -n auto -o faulthandler_timeout=120 -m "${TVM_TEST_MARKER:-not gpu}" "$@" diff --git a/tests/scripts/task_build_arm.sh b/tests/scripts/task_python_unittest_gpuonly.sh similarity index 56% copy from tests/scripts/task_build_arm.sh copy to tests/scripts/task_python_unittest_gpuonly.sh index cb85fea9ea..f283c8e8c8 100755 --- a/tests/scripts/task_build_arm.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@ -18,8 +18,19 @@ set -euxo pipefail -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +# Every GPU test carries the `gpu` marker; the specific backend is gated by skipif. +export TVM_TEST_MARKER="gpu" + +# Test most of the enabled runtimes here. +# TODO: disabled opencl tests due to segmentation fault. +export TVM_TEST_TARGETS='cuda;metal;rocm;nvptx' + +./tests/scripts/task_python_unittest.sh + +# Kept separate to avoid increasing time needed to run CI, testing +# only minimal functionality of Vulkan runtime. +export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}' + +./tests/scripts/task_python_unittest.sh \ + tests/python/backend/vulkan/codegen/test_basic.py \ + tests/python/backend/vulkan/test_module_export.py diff --git a/tests/scripts/task_build_arm.sh b/tests/scripts/test_task_install_dist_wheels.sh similarity index 68% copy from tests/scripts/task_build_arm.sh copy to tests/scripts/test_task_install_dist_wheels.sh index cb85fea9ea..2754924c83 100755 --- a/tests/scripts/task_build_arm.sh +++ b/tests/scripts/test_task_install_dist_wheels.sh @@ -18,8 +18,9 @@ set -euxo pipefail -export CMAKE_ARGS=" - -DTVM_TIRX_USE_LLVM=llvm-config-17 - -DCMAKE_CXX_FLAGS=-Werror -" -uv build --wheel --no-build-isolation --offline --python python3 --out-dir dist ./tirx +unset PYTHONPATH TVM_LIBRARY_PATH TVM_FFI_LIBRARY_PATH TVM_TIRX_LIBRARY_PATH +export PYTHONNOUSERSITE=1 +uv pip install --python python3 --target .local/python --no-deps apache-tvm-ffi-orcjit==0.1.2 +uv pip install --python python3 --target .local/python --no-deps --offline --reinstall \ + dist/apache_tvm_ffi-*.whl dist/apache_tvm_tirx-*.whl +export PYTHONPATH="$PWD/.local/python"
