This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-wheel-test-artifacts in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 01f05952fc964e67821500c2fa099cffbb999904 Author: Tianqi Chen <[email protected]> AuthorDate: Thu Oct 15 12:34:58 2026 +0000 [CI] Transfer TVM and pinned FFI wheels to Python and Java consumers --- ci/jenkins/README.md | 34 +-- ci/jenkins/artifacts.py | 4 +- ci/jenkins/generated/common.groovy | 3 +- ci/jenkins/generated/cpu_jenkinsfile.groovy | 2 +- ci/jenkins/generated/gpu_jenkinsfile.groovy | 2 +- .../linux-x86_64/src/main/assembly/assembly.xml | 7 +- .../osx-x86_64/src/main/assembly/assembly.xml | 7 +- jvm/core/pom.xml | 2 +- jvm/native/linux-x86_64/pom.xml | 6 +- jvm/native/osx-x86_64/pom.xml | 6 +- jvm/pom.xml | 4 + tests/python/ci/test_wheel_artifacts.py | 309 --------------------- tests/scripts/ci.py | 1 + tests/scripts/task_java_unittest.sh | 8 +- tests/scripts/task_jvm_build.sh | 22 +- tests/scripts/task_python_install_wheels.sh | 55 +++- 16 files changed, 118 insertions(+), 354 deletions(-) diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index 5121cb5a67..dbb68032b7 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -127,14 +127,17 @@ changing any retained script interface. ## Wheel build and test artifacts Normal builds use `uv build --wheel --no-build-isolation --offline` with the -existing platform CMake configuration. `uv build` builds only TVM; it does not -resolve runtime dependencies. CPU, ARM and Wasm host builds request `all;cpptest` +existing platform CMake configuration. `uv build` does not resolve runtime dependencies. The build stage also builds +`apache-tvm-ffi` from the pinned submodule, using its declared isolated build +requirements. CPU, ARM and Wasm host builds request `all;cpptest` and run C++ tests in that same native tree before uploading anything. Python -consumers install the exact TVM wheel with `--no-deps --offline`, clear checkout -path overrides, and verify the package and native-library paths. FFI comes from -the checked-out submodule. GPU Java tests also receive their existing native -runtime libraries; the separate runtime-only compile check and Emscripten runtime -build remain. +consumers install the exact TVM and FFI wheels with `--no-deps --offline`, clear +checkout path overrides, and verify artifact hashes and package/native-library +paths. Only `build/wheels/*.whl` is uploaded. GPU Java consumers discover headers, +linker paths and packaged native libraries through `tvm.config` and +`tvm_ffi.config`; no separate native files are transferred. The configuration +commands avoid loading TVM's Python compiler, backends or native libraries. +The separate runtime-only compile check and Emscripten runtime build remain. Immediately before building, the same container and Python install the declared `pyproject.toml` build requirements (scikit-build-core, setuptools-scm and @@ -146,17 +149,16 @@ modifying the image environment or exposing the build overlay to test consumers. `uv build` then runs offline without isolation. The selected images must already provide compatible runtime and test dependencies, -apart from core FFI and its ORCJIT addon. Consumers install the checkout's pinned -FFI source into the writable test environment with `--no-deps`, using its declared -isolated build -requirements, plus the explicit `apache-tvm-ffi-orcjit==0.1.2 --no-deps` prerequisite -used by legacy Python tests. They then install the transferred TVM wheel with -`--no-deps --offline`. +apart from core FFI and its ORCJIT addon. Consumers install the explicit +`apache-tvm-ffi-orcjit==0.1.2 --no-deps` prerequisite used by legacy Python tests, +then install exactly one transferred wheel for each of TVM and FFI with +`--no-deps --offline`. Consumers do not compile FFI. The checkout remains available for tests and resources through `TVM_HOME`, including Wasm assets under `web/dist`, without adding its Python package ahead of the -wheel. Provenance checks verify both TVM/native libraries and source-built FFI -load from the test environment; other missing runtime dependencies -remain errors rather than permission to fetch replacements. +wheel. Provenance checks bind installed distributions to the transferred wheel +filenames and the loaded files' SHA-256 hashes from each wheel RECORD, and verify +TVM/FFI Python and native libraries load +from the test environment. Other missing runtime dependencies remain errors. `CMAKE_BUILD_PARALLEL_LEVEL` is passed through Docker as an environment variable. An explicit value wins; otherwise builds use at least one CPU and divide the diff --git a/ci/jenkins/artifacts.py b/ci/jenkins/artifacts.py index b222d0dd4d..4f468c5cc8 100644 --- a/ci/jenkins/artifacts.py +++ b/ci/jenkins/artifacts.py @@ -34,8 +34,8 @@ artifact prefix. import sys ARTIFACTS = { - # New pipelines install the exact TVM wheel and retain only JVM native files. - "wheels": ["build/wheels"], + # New pipelines transfer only wheels; legacy raw-library bundles stay available. + "wheels": ["build/wheels/*.whl"], "jvm_runtime": ["build/lib/libtvm_runtime.so", "build/lib/libtvm_ffi.so"], # Executables and build files needed to run c++ tests "cpptest": ["build/cpptest", "build/build.ninja", "build/CMakeFiles/rules.ninja"], diff --git a/ci/jenkins/generated/common.groovy b/ci/jenkins/generated/common.groovy index 933bb11f0a..0eb042c301 100644 --- a/ci/jenkins/generated/common.groovy +++ b/ci/jenkins/generated/common.groovy @@ -391,9 +391,10 @@ def build_docker() { def wheel_build(image, cpp_tests = false) { // Overlay build requirements without modifying the image Python environment. - // The build itself stays offline; runtime dependencies belong to the image. + // FFI uses its declared build requirements; neither build resolves runtime dependencies. def commands = build_environment() + ''' rm -rf build/wheels build/wheel-build-deps + uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi uv pip install --python python3 --target build/wheel-build-deps "scikit-build-core>=0.11" "setuptools-scm>=8" "mlc-z3-static>=4.16.0" PYTHONPATH="$PWD/build/wheel-build-deps${PYTHONPATH:+:$PYTHONPATH}" \ uv build --wheel --no-build-isolation --offline --python python3 --out-dir build/wheels . \ diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 8be385ac42..14d3daae2d 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -41,7 +41,7 @@ stage('Test') { variables: ['PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU']) { ci.docker_init(images.ci_cpu) ci.download('cpu') - ci.python_unittest(images.ci_cpu) + ci.python_unittest(images.ci_cpu) } }) } diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 7f2be47292..e8a3b3d76d 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -27,7 +27,7 @@ stage('Build') { ci.docker_init(images.ci_gpu) sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" ci.wheel_build("--no-gpu ${images.ci_gpu}") - ci.upload_artifacts('gpu', '--artifacts wheels --artifacts jvm_runtime') + ci.upload_artifacts('gpu', '--artifacts wheels') // Check the compiler-only GPU configuration after publishing the test build. sh 'rm -rf build' diff --git a/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml b/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml index 7345c496e9..ea7396f8dc 100644 --- a/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml +++ b/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml @@ -26,7 +26,12 @@ under the License. <includeBaseDirectory>false</includeBaseDirectory> <files> <file> - <source>../../../build/lib/libtvm_runtime.so</source> + <source>${tvm.lib.dir}/libtvm_runtime.so</source> + <outputDirectory>lib/native</outputDirectory> + <fileMode>0644</fileMode> + </file> + <file> + <source>${tvm.ffi.lib.dir}/libtvm_ffi.so</source> <outputDirectory>lib/native</outputDirectory> <fileMode>0644</fileMode> </file> diff --git a/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml b/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml index 739852e474..70f5754983 100644 --- a/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml +++ b/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml @@ -26,7 +26,12 @@ under the License. <includeBaseDirectory>false</includeBaseDirectory> <files> <file> - <source>../../../build/lib/libtvm_runtime.dylib</source> + <source>${tvm.lib.dir}/libtvm_runtime.dylib</source> + <outputDirectory>lib/native</outputDirectory> + <fileMode>0644</fileMode> + </file> + <file> + <source>${tvm.ffi.lib.dir}/libtvm_ffi.dylib</source> <outputDirectory>lib/native</outputDirectory> <fileMode>0644</fileMode> </file> diff --git a/jvm/core/pom.xml b/jvm/core/pom.xml index eeb55f5cfd..26ba6dcd78 100644 --- a/jvm/core/pom.xml +++ b/jvm/core/pom.xml @@ -104,7 +104,7 @@ under the License. <threadCount>1</threadCount> <argLine> -Djava.library.path=${project.parent.basedir}/native/${platform}/target - -Dlibtvm.so.path=${project.parent.basedir}/../build/lib/${libtvm.so.filename} + -Dlibtvm.so.path=${tvm.lib.dir}/${libtvm.so.filename} -Dlog4j.configuration=file://${project.parent.basedir}/conf/log4j.properties </argLine> </configuration> diff --git a/jvm/native/linux-x86_64/pom.xml b/jvm/native/linux-x86_64/pom.xml index 31e120bc58..dcda4c6ce1 100644 --- a/jvm/native/linux-x86_64/pom.xml +++ b/jvm/native/linux-x86_64/pom.xml @@ -117,8 +117,8 @@ under the License. <compilerStartOption>-std=c++17</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I../../../include</compilerEndOption> - <compilerEndOption>-I../../../3rdparty/tvm-ffi/include</compilerEndOption> + <compilerEndOption>-I${tvm.include.dir}</compilerEndOption> + <compilerEndOption>-I${tvm.ffi.include.dir}</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include/linux</compilerEndOption> <compilerEndOption>${cflags}</compilerEndOption> @@ -127,7 +127,7 @@ under the License. <linkerStartOption>-shared</linkerStartOption> </linkerStartOptions> <linkerEndOptions> - <linkerEndOption>-L${project.parent.basedir}/../../build/lib/</linkerEndOption> + <linkerEndOption>-L${tvm.lib.dir}</linkerEndOption> <linkerEndOption>-ltvm_runtime</linkerEndOption> <linkerEndOption>${ldflags}</linkerEndOption> </linkerEndOptions> diff --git a/jvm/native/osx-x86_64/pom.xml b/jvm/native/osx-x86_64/pom.xml index 4f9d70d60d..ccb79f5bbb 100644 --- a/jvm/native/osx-x86_64/pom.xml +++ b/jvm/native/osx-x86_64/pom.xml @@ -118,8 +118,8 @@ under the License. <compilerStartOption>-std=c++17</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I../../../include</compilerEndOption> - <compilerEndOption>-I../../../3rdparty/tvm-ffi/include</compilerEndOption> + <compilerEndOption>-I${tvm.include.dir}</compilerEndOption> + <compilerEndOption>-I${tvm.ffi.include.dir}</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include/darwin</compilerEndOption> <compilerEndOption>${cflags}</compilerEndOption> @@ -134,7 +134,7 @@ under the License. <linkerMiddleOption>-Wl,-x</linkerMiddleOption> </linkerMiddleOptions> <linkerEndOptions> - <linkerEndOption>-L${project.parent.basedir}/../../build/lib/</linkerEndOption> + <linkerEndOption>-L${tvm.lib.dir}</linkerEndOption> <linkerEndOption>-ltvm_runtime</linkerEndOption> <linkerEndOption>${ldflags}</linkerEndOption> </linkerEndOptions> diff --git a/jvm/pom.xml b/jvm/pom.xml index e5efce0bf5..2a2f53cd73 100644 --- a/jvm/pom.xml +++ b/jvm/pom.xml @@ -44,6 +44,10 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <tvm.include.dir>${maven.multiModuleProjectDirectory}/../include</tvm.include.dir> + <tvm.ffi.include.dir>${maven.multiModuleProjectDirectory}/../3rdparty/tvm-ffi/include</tvm.ffi.include.dir> + <tvm.lib.dir>${maven.multiModuleProjectDirectory}/../build/lib</tvm.lib.dir> + <tvm.ffi.lib.dir>${tvm.lib.dir}</tvm.ffi.lib.dir> </properties> <packaging>pom</packaging> diff --git a/tests/python/ci/test_wheel_artifacts.py b/tests/python/ci/test_wheel_artifacts.py deleted file mode 100644 index a8adeef068..0000000000 --- a/tests/python/ci/test_wheel_artifacts.py +++ /dev/null @@ -1,309 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Wheel artifact transport and local CI command contracts, without a TVM import.""" - -import ast -import importlib.util -import json -import os -import re -import shutil -import subprocess -import sys -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[3] - - -def load_local_ci(): - spec = importlib.util.spec_from_file_location("local_ci", ROOT / "tests/scripts/ci.py") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - [email protected]("platform", ["cpu", "gpu", "arm", "wasm"]) -def test_local_wheel_build(monkeypatch, platform): - ci = load_local_ci() - calls = [] - monkeypatch.setattr(ci, "docker", lambda **kwargs: calls.append(kwargs)) - command, _, _ = ci.generate_command(platform, {}, "wheel build") - command(cpp=True) - scripts = calls[0]["scripts"] - wheel = next(script for script in scripts if "-Cbuild-dir=" in script) - assert "-Ccmake.build-type=RelWithDebInfo" in wheel - assert "all;cpptest" in wheel - assert not any("./3rdparty/tvm-ffi" in script for script in scripts) - assert "--no-build-isolation --offline" in wheel - assert any("task_python_install_wheels.sh" in script for script in scripts) - calls.clear() - command(skip_build=True, tests=["tests/python/example.py"]) - assert not any("uv build" in script for script in calls[0]["scripts"]) - - [email protected]("succeed", [True, False]) -def test_wheel_directory_upload_retries_each_transfer(tmp_path, succeed): - # Exercise the existing S3 CLI/retry boundary with a directory bundle. - for relative in [ - "ci/jenkins/scripts/s3.py", - "ci/jenkins/scripts/retry.sh", - "ci/jenkins/artifacts.py", - ]: - target = tmp_path / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(ROOT / relative, target) - wheels = tmp_path / "build/wheels" - wheels.mkdir(parents=True) - (wheels / "apache_tvm.whl").write_text("tvm") - binaries = tmp_path / "bin" - binaries.mkdir() - aws = binaries / "aws" - aws.write_text( - f"#!{sys.executable}\n" - "import json, pathlib, sys\n" - "path = pathlib.Path('attempts.json')\n" - "calls = json.loads(path.read_text()) if path.exists() else []\n" - "calls.append(sys.argv[1:])\n" - "path.write_text(json.dumps(calls))\n" - f"sys.exit(0 if {succeed!r} and len(calls) == 3 else 1)\n" - ) - aws.chmod(0o755) - sleep = binaries / "sleep" - sleep.write_text('#!/bin/sh\necho "$1" >> delays.txt\n') - sleep.chmod(0o755) - result = subprocess.run( - [ - sys.executable, - "ci/jenkins/scripts/s3.py", - "--action", - "upload", - "--bucket", - "test", - "--prefix", - "cpu", - "--bundle", - "wheels", - ], - cwd=tmp_path, - env={**os.environ, "PATH": f"{binaries}:{os.environ['PATH']}"}, - capture_output=True, - text=True, - ) - assert (result.returncode == 0) == succeed, result.stdout + result.stderr - attempts = json.loads((tmp_path / "attempts.json").read_text()) - assert len(attempts) == 3 - assert all( - call[-3:] == ["--recursive", "build/wheels", "s3://test/cpu/build/wheels"] - for call in attempts - ) - delays = [int(line) for line in (tmp_path / "delays.txt").read_text().splitlines()] - assert len(delays) == 2 and all(30 <= delay <= 200 for delay in delays) - - [email protected]("fail", [False, True]) -def test_local_interactive_keeps_cache_stats(tmp_path, monkeypatch, fail): - ci = load_local_ci() - monkeypatch.setattr(ci, "check_docker", lambda: None) - monkeypatch.setattr(ci, "cmd", lambda *args, **kwargs: None) - monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts") - monkeypatch.setattr(ci, "REPO_ROOT", tmp_path) - monkeypatch.delenv("USE_SCCACHE", raising=False) - monkeypatch.setenv("DEBUG", "1") - binaries = tmp_path / "bin" - binaries.mkdir() - for name in ["sccache", "bash"]: - binary = binaries / name - binary.write_text(f'#!/bin/sh\necho "{name} $*" >> "$CALL_LOG"\n') - binary.chmod(0o755) - ci.docker("interactive", "ci_cpu", ["false" if fail else "true"], {}, True) - result = subprocess.run( - [shutil.which("bash"), str(ci.SCRIPT_DIR / "interactive.sh")], - env={ - **os.environ, - "PATH": f"{binaries}:{os.environ['PATH']}", - "CALL_LOG": str(tmp_path / "calls"), - }, - capture_output=True, - text=True, - ) - assert (result.returncode != 0) == fail - calls = (tmp_path / "calls").read_text().splitlines() - assert calls == [ - "sccache --start-server", - "sccache --show-stats", - "sccache --show-stats", - "bash ", - ] - - [email protected]("wheel_docs", [False, True]) -def test_docs_import_path(tmp_path, monkeypatch, wheel_docs): - # Run the real Sphinx path bootstrap without requiring Sphinx extensions. - # A stale --target=python FFI package must not precede Jenkins' wheel install. - monkeypatch.chdir(ROOT / "docs") - source_python = str(ROOT / "python") - monkeypatch.setattr(sys, "path", [path for path in sys.path if path != source_python]) - monkeypatch.setenv("TVM_DOCS_USE_WHEEL", "1" if wheel_docs else "0") - config = ROOT / "docs/conf.py" - bootstrap = config.read_text().split("# -- General configuration")[0] - exec(compile(bootstrap, str(config), "exec"), {"__file__": str(config)}) - assert (source_python in sys.path) != wheel_docs - - [email protected]( - "host_limit,caller_limit,expected", [(None, None, "2"), ("7", None, "7"), ("7", "9", "9")] -) -def test_local_parallel_limit_crosses_docker_boundary( - tmp_path, monkeypatch, host_limit, caller_limit, expected -): - ci = load_local_ci() - calls = [] - monkeypatch.setattr(ci, "check_docker", lambda: None) - monkeypatch.setattr(ci, "cmd", lambda command: calls.append(command)) - monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts") - monkeypatch.setattr(ci, "REPO_ROOT", tmp_path) - monkeypatch.setattr(ci, "NPROC", 8) - monkeypatch.setenv("CI_NUM_EXECUTORS", "3") - if host_limit is None: - monkeypatch.delenv("CMAKE_BUILD_PARALLEL_LEVEL", raising=False) - else: - monkeypatch.setenv("CMAKE_BUILD_PARALLEL_LEVEL", host_limit) - env = {} if caller_limit is None else {"CMAKE_BUILD_PARALLEL_LEVEL": caller_limit} - ci.docker("build", "ci_cpu", ["true"], env, False) - args = calls[0] - index = args.index(f"CMAKE_BUILD_PARALLEL_LEVEL={expected}") - assert args[index - 1] == "--env" - - [email protected]( - "script", ["task_python_unittest.sh", "task_python_docs.sh", "task_web_wasm.sh"] -) [email protected]("wheel_mode", [False, True]) -def test_shared_scripts_preserve_legacy_and_wheel_modes(tmp_path, script, wheel_mode): - scripts = tmp_path / "tests/scripts" - scripts.mkdir(parents=True) - shutil.copy(ROOT / "tests/scripts" / script, scripts / script) - # Substitute only the install boundary; execute the real caller's mode routing. - (scripts / "task_python_install_wheels.sh").write_text('echo WHEEL_SETUP >> "$CALL_LOG"\n') - for directory in ["docs", "web", "bin"]: - (tmp_path / directory).mkdir() - for tool in ["uv", "python3", "make"]: - binary = tmp_path / "bin" / tool - binary.write_text( - f'#!/bin/sh\necho "{tool}:$*" >> "$CALL_LOG"\n' - + ("exit 77\n" if tool == "make" else "") - ) - binary.chmod(0o755) - result = subprocess.run( - [shutil.which("bash"), str(scripts / script)], - cwd=tmp_path, - env={ - **os.environ, - "PATH": f"{tmp_path / 'bin'}:{os.environ['PATH']}", - "CALL_LOG": str(tmp_path / "calls"), - "TVM_TEST_USE_WHEEL": "1" if wheel_mode else "0", - "IS_LOCAL": "0", - }, - capture_output=True, - text=True, - ) - assert result.returncode == (0 if script == "task_python_unittest.sh" else 77), result.stderr - calls = (tmp_path / "calls").read_text().splitlines() - assert ("WHEEL_SETUP" in calls) == wheel_mode - dependency_calls = [call for call in calls if call.startswith("uv:")] - if wheel_mode: - assert not dependency_calls - else: - assert "uv:pip install -v --target=python ./3rdparty/tvm-ffi/" in dependency_calls - - -def test_wheel_mode_does_not_fall_back_when_install_fails(tmp_path): - scripts = tmp_path / "tests/scripts" - scripts.mkdir(parents=True) - shutil.copy(ROOT / "tests/scripts/task_python_unittest.sh", scripts) - (scripts / "task_python_install_wheels.sh").write_text("return 37\n") - result = subprocess.run( - [shutil.which("bash"), str(scripts / "task_python_unittest.sh")], - cwd=tmp_path, - env={**os.environ, "TVM_TEST_USE_WHEEL": "1"}, - capture_output=True, - text=True, - ) - assert result.returncode == 37 - assert "uv pip install" not in result.stderr - - [email protected]("bootstrap_fails", [False, True]) -def test_build_bootstrap_precedes_offline_build(tmp_path, monkeypatch, bootstrap_fails): - import shlex - - ci = load_local_ci() - calls = [] - monkeypatch.setattr(ci, "docker", lambda **kwargs: calls.append(kwargs)) - command, _, _ = ci.generate_command("cpu", {}, "wheel build") - command() - commands = [ - script - for script in calls[0]["scripts"] - if "uv pip install" in script or "uv build" in script - ] - bootstrap, build = commands - arguments = shlex.split(bootstrap) - # The declared build requirements are a flat string list; keep this test usable on Python 3.10. - requires = re.search(r"(?ms)^requires = (\[.*?\])", (ROOT / "pyproject.toml").read_text()) - expected = ast.literal_eval(requires[1]) - assert arguments[:5] == ["uv", "pip", "install", "--python", "python3"] - assert arguments[5] == "--target" - assert arguments[7:] == expected - assert "--no-build-isolation --offline --python python3" in build - assert build.startswith(f'PYTHONPATH="$PWD/{arguments[6]}${{PYTHONPATH:+:$PYTHONPATH}}" ') - binary = tmp_path / "uv" - binary.write_text( - f"#!{sys.executable}\n" - "import json, os, pathlib, sys\n" - "with pathlib.Path('calls').open('a') as out: out.write(json.dumps(sys.argv[1:]) + '\\n')\n" - "with pathlib.Path('paths').open('a') as out: out.write(os.environ['PYTHONPATH'] + '\\n')\n" - f"sys.exit(17 if {bootstrap_fails!r} and sys.argv[1] == 'pip' else 0)\n" - ) - binary.chmod(0o755) - result = subprocess.run( - [ - shutil.which("bash"), - "-ec", - "\n".join([*commands, 'printf "%s" "$PYTHONPATH" > after-path']), - ], - cwd=tmp_path, - env={ - **os.environ, - "PATH": f"{tmp_path}:{os.environ['PATH']}", - "PYTHONPATH": "image-marker", - }, - capture_output=True, - text=True, - ) - assert result.returncode == (17 if bootstrap_fails else 0) - recorded = [json.loads(line) for line in (tmp_path / "calls").read_text().splitlines()] - assert [call[0] for call in recorded] == (["pip"] if bootstrap_fails else ["pip", "build"]) - paths = (tmp_path / "paths").read_text().splitlines() - assert paths[0] == "image-marker" - if not bootstrap_fails: - assert paths[1] == f"{tmp_path}/{arguments[6]}:image-marker" - assert (tmp_path / "after-path").read_text() == "image-marker" - else: - assert not (tmp_path / "after-path").exists() diff --git a/tests/scripts/ci.py b/tests/scripts/ci.py index 5c49251f4f..084dc9865b 100755 --- a/tests/scripts/ci.py +++ b/tests/scripts/ci.py @@ -403,6 +403,7 @@ def generate_command( scripts = [ f"./tests/scripts/task_config_build_{name}.sh {build_dir}", f"rm -rf {build_dir}/wheels {build_dir}/wheel-build-deps", + f"uv build --wheel --python python3 --out-dir {build_dir}/wheels ./3rdparty/tvm-ffi", f'uv pip install --python python3 --target {build_dir}/wheel-build-deps "scikit-build-core>=0.11" "setuptools-scm>=8" "mlc-z3-static>=4.16.0"', f'PYTHONPATH="$PWD/{build_dir}/wheel-build-deps${{PYTHONPATH:+:$PYTHONPATH}}" ' f"uv build --wheel --no-build-isolation --offline --python python3 --out-dir {build_dir}/wheels . " diff --git a/tests/scripts/task_java_unittest.sh b/tests/scripts/task_java_unittest.sh index 35621a5a13..07effb03e4 100755 --- a/tests/scripts/task_java_unittest.sh +++ b/tests/scripts/task_java_unittest.sh @@ -18,8 +18,12 @@ set -euxo pipefail -export PYTHONPATH=python -export LD_LIBRARY_PATH="lib:${LD_LIBRARY_PATH:-}" +if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then + source "$(dirname "$0")/task_python_install_wheels.sh" +else + export PYTHONPATH=python + export LD_LIBRARY_PATH="lib:${LD_LIBRARY_PATH:-}" +fi # to avoid CI CPU thread throttling. export TVM_BIND_THREADS=0 diff --git a/tests/scripts/task_jvm_build.sh b/tests/scripts/task_jvm_build.sh index 468fea4ec6..c5f4dac03f 100755 --- a/tests/scripts/task_jvm_build.sh +++ b/tests/scripts/task_jvm_build.sh @@ -33,13 +33,32 @@ fi ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)" TVM_BUILD_PATH="${TVM_BUILD_PATH:-${ROOTDIR}/build}" -TVM_BUILD_PATH="$(realpath "${TVM_BUILD_PATH}")" +if [ "${TVM_TEST_USE_WHEEL:-0}" != "1" ]; then + TVM_BUILD_PATH="$(realpath "${TVM_BUILD_PATH}")" +fi DLPACK_PATH="${DLPACK_PATH:-${ROOTDIR}/3rdparty/tvm-ffi/3rdparty/dlpack}" INCLUDE_FLAGS="-I${ROOTDIR}/include -I${DLPACK_PATH}/include" PKG_CFLAGS="-Wall -O3 ${INCLUDE_FLAGS} -fPIC" PKG_LDFLAGS="" +NATIVE_ARGS=() +if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then + TVM_INCLUDE_DIR="$(python3 -m tvm.config --includedir)" + FFI_INCLUDE_DIR="$(python3 -m tvm_ffi.config --includedir)" + TVM_LIB_DIR="$(python3 -m tvm.config --libdir)" + FFI_LIB_DIR="$(python3 -m tvm_ffi.config --libdir)" + PKG_CFLAGS="-Wall -O3 -fPIC $(python3 -m tvm.config --cflags) $(python3 -m tvm_ffi.config --cflags)" + PKG_LDFLAGS="$(python3 -m tvm.config --ldflags --libs) $(python3 -m tvm_ffi.config --ldflags --libs)" + # Each config flag prints one line; Maven expects one compiler/linker argument list. + PKG_LDFLAGS="${PKG_LDFLAGS//$'\n'/ }" + NATIVE_ARGS=( + "-Dtvm.include.dir=${TVM_INCLUDE_DIR}" + "-Dtvm.ffi.include.dir=${FFI_INCLUDE_DIR}" + "-Dtvm.lib.dir=${TVM_LIB_DIR}" + "-Dtvm.ffi.lib.dir=${FFI_LIB_DIR}" + ) +fi if [ "$(uname -s)" = "Darwin" ]; then JVM_PKG_PROFILE="osx-x86_64" @@ -64,5 +83,6 @@ mvn ${GOAL} \ "-Dcflags=${PKG_CFLAGS}" \ "-Dldflags=${PKG_LDFLAGS}" \ "-Dcurrent_libdir=${TVM_BUILD_PATH}" \ + "${NATIVE_ARGS[@]}" \ ${JVM_TEST_ARGS} \ "$@" diff --git a/tests/scripts/task_python_install_wheels.sh b/tests/scripts/task_python_install_wheels.sh index 9fde7c743c..6791f36780 100644 --- a/tests/scripts/task_python_install_wheels.sh +++ b/tests/scripts/task_python_install_wheels.sh @@ -22,7 +22,7 @@ unset PYTHONPATH TVM_LIBRARY_PATH TVM_FFI_LIBRARY_PATH export PYTHONNOUSERSITE=1 # Installed TVM still needs checkout resources such as web/dist for Wasm linking. export TVM_HOME="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -TVM_WHEEL_DIR="$(realpath "${TVM_WHEEL_DIR:-$(pwd)/build/wheels}")" +export TVM_WHEEL_DIR="$(realpath "${TVM_WHEEL_DIR:-$(pwd)/build/wheels}")" # Preserve the image's active environment (often itself a venv), including when # local ci.py sources this setup again for a named test suite. export TVM_IMAGE_PYTHON="${TVM_IMAGE_PYTHON:-$(command -v python3)}" @@ -34,26 +34,36 @@ uv venv --offline --python "$TVM_IMAGE_PYTHON" --clear "$VIRTUAL_ENV" TVM_WHEEL_SITE="$("$VIRTUAL_ENV/bin/python3" -c 'import sysconfig; print(sysconfig.get_path("purelib"))')" printf '%s\n' "$TVM_IMAGE_SITE_PACKAGES" > "$TVM_WHEEL_SITE/image-dependencies.pth" export PATH="$VIRTUAL_ENV/bin:$PATH" -# Keep the checkout available to tests and build its pinned FFI into this environment. -# FFI declares its own isolated build requirements; do not reuse TVM's build overlay. -export TVM_FFI_SOURCE="$TVM_HOME/3rdparty/tvm-ffi" -uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --reinstall "$TVM_FFI_SOURCE" +# Require one artifact for each distribution before creating an installation. +shopt -s nullglob +tvm_wheels=("$TVM_WHEEL_DIR"/apache_tvm-*.whl) +ffi_wheels=("$TVM_WHEEL_DIR"/apache_tvm_ffi-*.whl) +shopt -u nullglob +if [ "${#tvm_wheels[@]}" -ne 1 ] || [ "${#ffi_wheels[@]}" -ne 1 ]; then + echo "Expected exactly one TVM wheel and one pinned-source FFI wheel" >&2 + return 1 +fi # Match the explicit prerequisite in the legacy Python test setup. uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps apache-tvm-ffi-orcjit==0.1.2 uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --offline --reinstall \ - "$TVM_WHEEL_DIR"/apache_tvm-*.whl + "${tvm_wheels[0]}" "${ffi_wheels[0]}" # Fail before pytest if a source/editable install or native override leaked in. python3 - <<'PYTHON' +import base64 +import csv +import hashlib +import io import json import os import sys from importlib.metadata import distribution, requires, version from pathlib import Path from urllib.parse import unquote, urlparse +from zipfile import ZipFile from packaging.requirements import Requirement -# Core FFI comes from pinned source and ORCJIT is pinned explicitly. +# Core FFI comes from the build-stage wheel and ORCJIT is pinned explicitly. # Check the wheel requirements without invoking a resolver or hardcoding a floor. for text in requires("apache-tvm") or []: requirement = Requirement(text) @@ -69,13 +79,34 @@ prefix = Path(sys.prefix).resolve() path = Path(tvm.__file__).resolve() assert path.is_relative_to(prefix), path print(f"Installed tvm: {path}") -# Verify FFI was installed from this checkout and loads entirely from this environment. -source = json.loads(distribution("apache-tvm-ffi").read_text("direct_url.json")) -assert Path(unquote(urlparse(source["url"]).path)).resolve() == Path(os.environ["TVM_FFI_SOURCE"]).resolve() -assert not source.get("dir_info", {}).get("editable", False), source +# Bind each distribution and its loaded Python/native files to the exact wheel. +def verify_wheel(name, pattern, paths): + wheel, = Path(os.environ["TVM_WHEEL_DIR"]).glob(pattern) + dist = distribution(name) + source = json.loads(dist.read_text("direct_url.json")) + assert Path(unquote(urlparse(source["url"]).path)).resolve() == wheel.resolve(), source + assert "archive_info" in source and "dir_info" not in source, source + with ZipFile(wheel) as archive: + record, = (name for name in archive.namelist() if name.endswith(".dist-info/RECORD")) + hashes = {row[0]: row[1] for row in csv.reader(io.StringIO(archive.read(record).decode()))} + for path in paths: + path = Path(path).resolve() + relative = path.relative_to(Path(dist.locate_file("")).resolve()).as_posix() + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + actual = "sha256=" + base64.urlsafe_b64encode(digest.digest()).decode().rstrip("=") + assert actual == hashes[relative], (name, path) + print(f"Installed {name} wheel: {wheel.name}") + +verify_wheel("apache-tvm", "apache_tvm-*.whl", + [tvm.__file__, *(lib._name for lib in tvm.base._LOADED_LIBS.values())]) +verify_wheel("apache-tvm-ffi", "apache_tvm_ffi-*.whl", + [tvm_ffi.__file__, tvm_ffi.core.__file__, tvm_ffi.LIB._name]) for path in (Path(tvm_ffi.__file__).resolve(), Path(tvm_ffi.LIB._name).resolve()): assert path.is_relative_to(prefix), path - print(f"Source-built FFI: {path}") + print(f"Installed FFI: {path}") assert {"tvm_runtime", "tvm_compiler"} <= tvm.base._LOADED_LIBS.keys() for name, library in tvm.base._LOADED_LIBS.items(): path = Path(library._name).resolve()
