This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tirx/standalone-package-cleanup in repository https://gitbox.apache.org/repos/asf/tvm.git
commit fc53ebd4b7cadb983d7ee52776bf4a8635fdfe04 Author: Tianqi Chen <[email protected]> AuthorDate: Thu Oct 15 02:10:47 2026 +0000 [FIX][CI] Build both native partitions with the pinned FFI dependency --- ci/jenkins/data.py | 4 ++- tests/python/ci/test_jenkins_helpers.py | 12 ++++++++- tests/scripts/task_build.py | 44 +++++++++++++++++++++++++++++---- tests/scripts/task_cpp_unittest.sh | 6 +++++ tests/scripts/task_python_unittest.sh | 4 +-- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/ci/jenkins/data.py b/ci/jenkins/data.py index 74f7ccc96d..d6ddd98990 100644 --- a/ci/jenkins/data.py +++ b/ci/jenkins/data.py @@ -29,6 +29,7 @@ data-driven artifact lists. """ import sys +from glob import glob files_to_stash = { # Executables and build files needed to run c++ tests @@ -45,7 +46,8 @@ files_to_stash = { "build/lib/libtvm_compiler.so", "build/lib/libtvm_tirx.so", "build/lib/libtvm_runtime.so", - "build/lib/libtvm_ffi.so", + "python/tvm_ffi", + *glob("python/apache_tvm_ffi-*.dist-info"), "build/lib/libtvm_runtime_cuda.so", "build/lib/libtvm_runtime_vulkan.so", "build/lib/libtvm_runtime_opencl.so", diff --git a/tests/python/ci/test_jenkins_helpers.py b/tests/python/ci/test_jenkins_helpers.py index cfc0c69ced..ef3a0a3445 100644 --- a/tests/python/ci/test_jenkins_helpers.py +++ b/tests/python/ci/test_jenkins_helpers.py @@ -25,7 +25,7 @@ import sys import pytest -from .test_utils import JENKINS_SCRIPT_ROOT, REPO_ROOT +from .test_utils import JENKINS_SCRIPT_ROOT, REPO_ROOT, TempGit def load_s3(script_root=JENKINS_SCRIPT_ROOT): @@ -93,6 +93,16 @@ def test_build_environment_and_target(tmp_path): script.parent.mkdir(parents=True) shutil.copy(REPO_ROOT / "tests/scripts/task_build.py", script) (tmp_path / "build").mkdir() + ffi_path = tmp_path / "3rdparty/tvm-ffi" + ffi_path.mkdir(parents=True) + ffi_repo = TempGit(ffi_path) + ffi_repo.run("config", "user.name", "CI Test") + ffi_repo.run("config", "user.email", "[email protected]") + ffi_repo.run("commit", "--allow-empty", "-m", "FFI fixture") + ffi_commit = ffi_repo.run("rev-parse", "HEAD", stdout=subprocess.PIPE).stdout.strip() + ffi_marker = tmp_path / "python/tvm_ffi/.source-commit" + ffi_marker.parent.mkdir(parents=True) + ffi_marker.write_text(f"{ffi_commit} {sys.implementation.cache_tag}\n") executable( tmp_path / "cmake", """ diff --git a/tests/scripts/task_build.py b/tests/scripts/task_build.py index c240ec51c0..3160870aaf 100755 --- a/tests/scripts/task_build.py +++ b/tests/scripts/task_build.py @@ -21,6 +21,7 @@ import multiprocessing import os import shutil import subprocess +import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent.parent @@ -34,12 +35,19 @@ if __name__ == "__main__": parser.add_argument("--sccache-bucket", required=False, help="sccache bucket name") parser.add_argument("--sccache-region", required=False, help="sccache region") parser.add_argument("--build-dir", default="build", help="build folder") - parser.add_argument("--cmake-target", help="optional build target") + parser.add_argument( + "--cmake-target", action="append", default=[], help="build target (repeatable)" + ) + parser.add_argument( + "--ffi-only", action="store_true", help="prepare the source-pinned FFI package" + ) parser.add_argument("--debug", required=False, action="store_true", help="build in debug mode") args = parser.parse_args() sccache_exe = shutil.which("sccache") - if args.cmake_target in ["standalone_crt", "crttest"]: + if args.cmake_target and all( + target in {"standalone_crt", "crttest"} for target in args.cmake_target + ): logging.info("Skipping standalone_crt build") exit(0) @@ -47,6 +55,13 @@ if __name__ == "__main__": build_dir = Path(os.getcwd()) / args.build_dir build_dir = build_dir.relative_to(REPO_ROOT) env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join( + [ + str(REPO_ROOT / "tirx" / "python"), + str(REPO_ROOT / "python"), + os.environ.get("PYTHONPATH", ""), + ] + ) if use_sccache: if args.sccache_bucket and "AWS_ACCESS_KEY_ID" in os.environ: @@ -68,6 +83,27 @@ if __name__ == "__main__": reason = "<unknown>" logging.info(f"Not using sccache, reason: {reason}") + # Build from this checkout's pinned submodule once, then transfer the exact + # Python/native package together with the TVM libraries to test workers. + ffi_commit = subprocess.check_output( + ["git", "-C", str(REPO_ROOT / "3rdparty" / "tvm-ffi"), "rev-parse", "HEAD"], text=True + ).strip() + ffi_identity = f"{ffi_commit} {sys.implementation.cache_tag}" + ffi_marker = REPO_ROOT / "python" / "tvm_ffi" / ".source-commit" + if not ffi_marker.is_file() or ffi_marker.read_text().strip() != ffi_identity: + subprocess.run( + [ + "uv", "pip", "install", "--python", sys.executable, + "-v", "--no-deps", "--reinstall", "--target=python", "./3rdparty/tvm-ffi/", + ], + cwd=REPO_ROOT, + env=env, + check=True, + ) + ffi_marker.write_text(ffi_identity + "\n") + if args.ffi_only: + sys.exit(0) + if use_sccache: subprocess.run(["sccache", "--start-server"], env=env, check=False) logging.info("===== sccache stats =====") @@ -89,9 +125,7 @@ if __name__ == "__main__": check=True, ) - ninja_args = [f"-j{num_cpus}"] - if args.cmake_target: - ninja_args.insert(0, args.cmake_target) + ninja_args = [*args.cmake_target, f"-j{num_cpus}"] if os.environ.get("VERBOSE", "true").lower() in {"1", "true", "yes"}: ninja_args.append("-v") subprocess.run(["cmake", "--build", ".", "--", *ninja_args], cwd=build_dir, env=env, check=True) diff --git a/tests/scripts/task_cpp_unittest.sh b/tests/scripts/task_cpp_unittest.sh index 424642673b..efe6417733 100755 --- a/tests/scripts/task_cpp_unittest.sh +++ b/tests/scripts/task_cpp_unittest.sh @@ -32,6 +32,12 @@ fi export TVM_BIND_THREADS=0 export OMP_NUM_THREADS=1 +# Test workers receive the exact FFI package in their own checkout. Native +# binaries must find that copy even when their build workspace had another path. +ffi_lib_dir="$(pwd)/python/tvm_ffi/lib" +export LD_LIBRARY_PATH="${ffi_lib_dir}:${LD_LIBRARY_PATH:-}" +export DYLD_LIBRARY_PATH="${ffi_lib_dir}:${DYLD_LIBRARY_PATH:-}" + pushd "${BUILD_DIR}" # run cpp test executable ./cpptest diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index c6c578cc5b..85b5d692f0 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -21,8 +21,8 @@ set -euxo pipefail export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ +# Reuse the exact source-pinned FFI package transferred by the native build. +python3 tests/scripts/task_build.py --ffi-only # LLVMModule JIT execution. Installed here as well as in the CI image because a docker/ # change does not reach the test containers in the same run (they use the pinned tag).
