This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-sccache-diagnostics in repository https://gitbox.apache.org/repos/asf/tvm.git
commit d7ed6808067305935ca8f33b06425f302fc2ece6 Author: Tianqi Chen <[email protected]> AuthorDate: Fri Oct 16 01:27:23 2026 +0000 [CI] Add isolated Jenkins sccache diagnostics --- ci/README.md | 56 +++++ ci/jenkins/generated/cpu_jenkinsfile.groovy | 61 ++---- tests/python/ci/test_sccache_diagnostics.py | 150 +++++++++++++ tests/scripts/task_sccache_diagnostics.py | 316 ++++++++++++++++++++++++++++ 4 files changed, 541 insertions(+), 42 deletions(-) diff --git a/ci/README.md b/ci/README.md index e617d7af91..aa9b7a98c0 100644 --- a/ci/README.md +++ b/ci/README.md @@ -93,3 +93,59 @@ Other jobs run in GitHub Actions for Windows and MacOS jobs. The native Groovy entry points in `ci/jenkins/generated/` load shared helpers from `common.groovy`. Edit these sources directly; no generation step is needed. See comments in [common.groovy](jenkins/generated/common.groovy) for shared utility options and trust constraints. + +## Sccache diagnostic branch + +The CPU job on `jenkins-sccache-diagnostics` runs a host IMDS probe and then +runs IMDS plus a small S3 cache roundtrip in the configured CPU image through +`docker/bash.sh`. It uses the wrapper's existing network settings; it does not +change instance metadata options, IAM policies, or Docker networking. + +On the affected worker, the equivalent commands are: + +```bash +python3 tests/scripts/task_sccache_diagnostics.py --scope host +docker/bash.sh <current-ci-cpu-image> python3 tests/scripts/task_sccache_diagnostics.py \ + --scope container --bucket tvm-sccache-prod --region us-west-2 +``` + +The probe requests an IMDSv2 token, lists the instance role, and checks that a +credential response contains the required fields. Only HTTP statuses, elapsed +times, and availability booleans are printed. If the token request fails, an +unauthenticated role-list request records whether IMDS returns an HTTP response +(such as 401) or also times out. No token, role name, credential value, raw +subprocess output, or raw error log is printed or archived. + +The cache probe requires sccache with Unix-domain socket support (the image +installer pins 0.14.0) and `/usr/bin/gcc`. It resets sccache-specific settings, +selects the given S3 bucket and region, and retains the current AWS credential +sources. It does not forward host credentials into Docker. A private temporary +directory and socket isolate its server from other builds. A unique tiny C +source causes a cache write; a second compile after restarting that server must +record an S3 hit. This leaves small cache/probe objects in the bucket, subject to its existing +lifecycle policy. It does not delete cache objects. + +Each metadata request has a three-second socket timeout, each sccache command +has a 30-second process timeout, and the Jenkins stage has a ten-minute bound. +Temporary error logs are reduced to fixed error categories and then removed. +The final `s3_roundtrip` value is true only when the backend is S3, the first +compile writes successfully, and the second server reports a cache hit without +cache read/write errors. Compile success alone does not establish cache access. + +Interpret the host/container results separately from the S3 result: + +- Host token success and container token timeout indicate a difference in the + metadata network path. An IMDSv2 response hop limit is one possible cause; + this probe does not read or establish the configured hop limit. +- HTTP 401/403 responses show that an HTTP path exists. Missing role metadata + or unavailable credential fields are separate from a network timeout. +- Successful metadata credentials followed by an S3 authorization error marker + points toward credential/bucket policy or signing problems. If other AWS + credential sources are present, they may take precedence over IMDS. +- S3 network timeouts and authorization markers are reported separately. Error + categories are recognized markers, not a full root-cause diagnosis; an + unclassified failure remains unresolved. + +A failed probe makes the diagnostic CPU job unstable. The other platform jobs +retain their upstream behavior. A local non-EC2 run cannot establish the +production Jenkins worker's metadata or S3 configuration. diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 44aceb116d..c262483bc1 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -27,49 +27,26 @@ try { ci = evaluate(helper_source) ci.start() -stage('Build') { - ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', - name: 'BUILD: CPU', folder: 'tvm/build-cpu', variables: ['PLATFORM=cpu']) { - ci.docker_init(images.ci_cpu) - sh( - script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", - label: 'Create CPU CMake config', - ) - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_cpu} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir build", - label: 'Run CMake build', - ) - sh ( - script: """ - set -eux - ${docker_run} ${images.ci_cpu} python3 ./tests/scripts/task_build.py \ - --sccache-bucket tvm-sccache-prod \ - --sccache-region us-west-2 \ - --cmake-target cpptest \ - --cmake-target tirx_cpptest \ - --build-dir build - """, - label: 'Make C++ tests', - ) - ci.upload_artifacts('cpu', '--artifact tvm_lib --artifact cpptest') - } -} - -stage('Test') { - parallel('unittest: CPU': { - ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', - name: 'unittest: CPU', folder: 'tvm/ut-python-cpu', - variables: ['PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU']) { - ci.docker_init(images.ci_cpu) - ci.download_artifacts('cpu') - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_cpu} ./tests/scripts/task_cpp_unittest.sh", - label: 'Run C++ tests', +// This diagnostics branch runs only probes in the CPU job. +stage('sccache diagnostics') { + ci.with_node(instance: 'CPU', name: 'sccache diagnostics', + folder: 'tvm/sccache-diagnostics', variables: ['PLATFORM=cpu']) { + timeout(time: 10, unit: 'MINUTES') { + def host_status = sh( + script: 'python3 tests/scripts/task_sccache_diagnostics.py --scope host', + label: 'Host IMDS status (no token or credential output)', + returnStatus: true, ) - sh ( - script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_python_unittest.sh", - label: 'Run Python unit tests', + ci.docker_init(images.ci_cpu) + def container_status = sh( + script: "${docker_run} ${images.ci_cpu} python3 tests/scripts/task_sccache_diagnostics.py --scope container --bucket tvm-sccache-prod --region us-west-2", + label: 'Current Docker network IMDS and isolated S3 cache roundtrip', + returnStatus: true, ) + echo "Diagnostic exit codes: host=${host_status}, container=${container_status}" + if (host_status != 0 || container_status != 0) { + unstable('sccache diagnostic probes did not all pass; inspect the status summaries') + } } - }) + } } diff --git a/tests/python/ci/test_sccache_diagnostics.py b/tests/python/ci/test_sccache_diagnostics.py new file mode 100644 index 0000000000..16fbd63ae5 --- /dev/null +++ b/tests/python/ci/test_sccache_diagnostics.py @@ -0,0 +1,150 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Exercise diagnostic failure paths without contacting metadata or S3.""" + +import importlib.util +import json +import subprocess +import urllib.error +from pathlib import Path +from unittest import mock + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "task_sccache_diagnostics.py" +spec = importlib.util.spec_from_file_location("diagnostics", SCRIPT) +diag = importlib.util.module_from_spec(spec) +spec.loader.exec_module(diag) +SECRET = "sentinel-token-and-credential" + + +def test_metadata_success_never_prints_secrets(capsys): + responses = [ + (200, SECRET.encode()), + (200, b"role-name"), + ( + 200, + json.dumps( + { + "Code": "Success", + "AccessKeyId": SECRET, + "SecretAccessKey": SECRET, + "Token": SECRET, + } + ).encode(), + ), + ] + opener = mock.MagicMock() + opener.open.return_value.__enter__.side_effect = [ + mock.Mock(status=code, read=mock.Mock(return_value=body)) for code, body in responses + ] + with mock.patch.object(diag.urllib.request, "build_opener", return_value=opener): + assert diag.metadata_probe() + requests = [call.args[0] for call in opener.open.call_args_list] + assert requests[0].method == "PUT" + assert requests[-1].get_header("X-aws-ec2-metadata-token") == SECRET + assert SECRET not in capsys.readouterr().out + + +def test_metadata_timeout_is_not_authorization(capsys): + opener = mock.MagicMock() + opener.open.side_effect = [ + TimeoutError(SECRET), + urllib.error.HTTPError(diag.IMDS, 401, SECRET, {}, None), + ] + with mock.patch.object(diag.urllib.request, "build_opener", return_value=opener): + assert not diag.metadata_probe() + output = capsys.readouterr().out + assert "network_timeout" in output + assert '"http_status": 401' in output + assert SECRET not in output + + +def test_stats_and_errors_only_emit_allowlisted_values(): + summary = diag.stats_summary( + json.dumps( + { + "stats": { + "cache_hits": {"counts": {SECRET: 1}}, + "cache_writes": 1, + "cache_read_errors": SECRET, + "not_cached": {SECRET: 4}, + }, + "cache_location": "s3 " + SECRET, + "version": SECRET, + "basedirs": [SECRET], + } + ) + ) + assert summary["cache_hits"] == 1 + assert summary["s3_backend"] + assert SECRET not in json.dumps(summary) + assert diag.error_categories("AccessDenied " + SECRET) == ["authorization"] + assert diag.error_categories("metadata connect error timed out " + SECRET) == [ + "network_timeout", + "network_error", + ] + + +def test_process_timeout_does_not_echo_output(capsys, tmp_path): + with mock.patch.object( + diag.subprocess, + "run", + side_effect=subprocess.TimeoutExpired(SECRET, 30, output=SECRET, stderr=SECRET), + ): + assert diag.run_sccache("sccache", [], {}, tmp_path, "compile")[0] == 1 + output = capsys.readouterr().out + assert "process_timeout" in output + assert SECRET not in output + + [email protected]( + "writes,hits,write_errors,expected", [(1, 1, 0, True), (0, 0, 1, False), (1, 0, 0, False)] +) +def test_s3_roundtrip_restarts_server_and_keeps_credentials_private( + capsys, monkeypatch, writes, hits, write_errors, expected +): + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", SECRET) + monkeypatch.setenv("SCCACHE_SERVER_PORT", "4226") + monkeypatch.setenv("SCCACHE_IGNORE_SERVER_IO_ERROR", "1") + calls = [] + + def run(command, *, env, cwd, **kwargs): + calls.append(command[1:]) + assert env["AWS_SECRET_ACCESS_KEY"] == SECRET + assert "SCCACHE_SERVER_PORT" not in env + assert "SCCACHE_IGNORE_SERVER_IO_ERROR" not in env + assert env["SCCACHE_SERVER_UDS"].startswith(str(cwd)) + assert Path(env["SCCACHE_CONF"]).read_text() == "" + Path(env["SCCACHE_ERROR_LOG"]).write_text(SECRET) + stats = { + "cache_writes": writes, + "cache_hits": {"counts": {"C/C++": hits}}, + "cache_write_errors": write_errors, + } + output = json.dumps({"stats": stats, "cache_location": "s3 " + SECRET}).encode() + return subprocess.CompletedProcess(command, 0, output, SECRET.encode()) + + with ( + mock.patch.object(diag.shutil, "which", return_value="sccache"), + mock.patch.object(diag.subprocess, "run", side_effect=run), + ): + assert diag.cache_probe("test-bucket", "us-west-2", "/usr/bin/gcc") is expected + assert calls[3] == ["--stop-server"] + assert calls[4] == ["--start-server"] + assert calls[-1] == ["--stop-server"] + assert SECRET not in capsys.readouterr().out diff --git a/tests/scripts/task_sccache_diagnostics.py b/tests/scripts/task_sccache_diagnostics.py new file mode 100644 index 0000000000..a9335f93d1 --- /dev/null +++ b/tests/scripts/task_sccache_diagnostics.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Bounded IMDS and S3 probes. Only allowlisted summaries reach the console.""" + +import argparse +import json +import os +import re +import shutil +import subprocess +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +IMDS = "http://169.254.169.254/latest/"; +COUNTERS = ( + "compile_requests", + "requests_executed", + "cache_timeouts", + "cache_read_errors", + "cache_write_errors", + "cache_writes", + "compile_fails", + "non_cacheable_compilations", +) + + +def emit(probe, **fields): + print(json.dumps({"probe": probe, **fields}, sort_keys=True), flush=True) + + +def metadata_request(probe, path, method="GET", headers=None): + # Metadata is link-local: bypass HTTP_PROXY without printing its value. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + request = urllib.request.Request(IMDS + path, method=method, headers=headers or {}) + start = time.monotonic() + body = b"" + try: + with opener.open(request, timeout=3) as response: + body = response.read(16384) + result = {"http_status": response.status, "result": "response"} + except urllib.error.HTTPError as error: + result = {"http_status": error.code, "result": "http_error"} + except TimeoutError: + result = {"result": "network_timeout"} + except urllib.error.URLError as error: + result = { + "result": "network_timeout" + if isinstance(error.reason, TimeoutError) + else "network_error" + } + result["elapsed_ms"] = round((time.monotonic() - start) * 1000) + emit(probe, **result) + return result.get("http_status"), body + + +def metadata_probe(): + status, token = metadata_request( + "imds_token", "api/token", "PUT", {"X-aws-ec2-metadata-token-ttl-seconds": "60"} + ) + if status != 200 or not token: + # A 401 here proves an HTTP path exists even if the token PUT timed out. + metadata_request("imds_without_token", "meta-data/iam/security-credentials/") + return False + headers = {"X-aws-ec2-metadata-token": token.decode("ascii")} + status, roles = metadata_request( + "imds_role_list", "meta-data/iam/security-credentials/", headers=headers + ) + if status != 200 or not roles.strip(): + emit("imds_role", available=False) + return False + role = urllib.parse.quote(roles.decode("utf-8").splitlines()[0], safe="") + status, body = metadata_request( + "imds_credentials_http", "meta-data/iam/security-credentials/" + role, headers=headers + ) + valid = False + if status == 200: + try: + credentials = json.loads(body) + valid = credentials.get("Code") == "Success" and all( + credentials.get(key) for key in ("AccessKeyId", "SecretAccessKey", "Token") + ) + except (ValueError, AttributeError): + pass + emit("imds_credentials", available=bool(valid)) + return bool(valid) + + +def error_categories(output): + # Raw sccache errors may contain credentials, headers, or URLs. Never emit them. + text = output.lower() + categories = [] + for name, patterns in ( + ( + "authorization", + ( + "accessdenied", + "permissiondenied", + "forbidden", + "status: 403", + "invalidaccesskeyid", + "signaturedoesnotmatch", + "expiredtoken", + ), + ), + ( + "credentials_unavailable", + ( + "no credentials", + "no valid credential", + "credentials not found", + "credential not found", + "credentials are empty", + ), + ), + ("network_timeout", ("timed out", "timeout")), + ("network_error", ("connection refused", "connection reset", "dns", "connect error")), + ): + if any(pattern in text for pattern in patterns): + categories.append(name) + return categories + + +def run_sccache(executable, arguments, env, directory, phase): + try: + result = subprocess.run( + [executable, *arguments], + env=env, + cwd=directory, + capture_output=True, + timeout=30, + check=False, + ) + output = (result.stdout + result.stderr).decode("utf-8", errors="replace") + emit( + phase, + exit_code=result.returncode, + error_categories=error_categories(output) if result.returncode else [], + ) + return result.returncode, result.stdout + except subprocess.TimeoutExpired: + emit(phase, result="process_timeout") + except OSError: + emit(phase, result="process_error") + return 1, b"" + + +def stats_summary(output): + info = json.loads(output) + stats = info["stats"] + summary = {key: stats[key] for key in COUNTERS if type(stats.get(key)) is int} + for key in ("cache_hits", "cache_misses", "cache_errors"): + counts = stats.get(key, {}).get("counts", {}) + summary[key] = sum(value for value in counts.values() if type(value) is int) + version = str(info.get("version", "")) + if re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + summary["version"] = version + summary["s3_backend"] = str(info.get("cache_location", "")).lower().startswith("s3") + return summary + + +def cache_probe(bucket, region, compiler): + executable = shutil.which("sccache") + if not executable or not Path(compiler).is_file(): + emit("sccache", result="missing_sccache_or_compiler") + return False + # A private UDS prevents changing/stopping an existing build's cache server. + # Keep AWS credential sources intact; reset only sccache-specific settings. + with tempfile.TemporaryDirectory(prefix="sccache-probe-", dir="/tmp") as temporary: + directory = Path(temporary) + config = directory / "config" + config.write_text("") + env = {key: value for key, value in os.environ.items() if not key.startswith("SCCACHE_")} + env.update( + SCCACHE_CONF=str(config), + SCCACHE_CACHED_CONF=str(directory / "cached-config"), + SCCACHE_SERVER_UDS=str(directory / "server.sock"), + SCCACHE_IDLE_TIMEOUT="60", + SCCACHE_DIR=str(directory / "cache"), + SCCACHE_BUCKET=bucket, + SCCACHE_REGION=region, + SCCACHE_S3_USE_SSL="true", + SCCACHE_LOG="debug", + SCCACHE_ERROR_LOG=str(directory / "errors"), + SCCACHE_DIRECT="false", + ) + # A fresh source forces a miss without deleting existing cache objects. + (directory / "probe.c").write_text( + "int sccache_probe_" + uuid.uuid4().hex + "(void) { return 0; }\n" + ) + results = [] + server_may_be_running = False + try: + for phase in ("write", "read"): + server_may_be_running = True + code, _ = run_sccache( + executable, ["--start-server"], env, directory, phase + "_start" + ) + if code: + return False + code, _ = run_sccache( + executable, + [compiler, "-c", "probe.c", "-o", "probe.o"], + env, + directory, + phase + "_compile", + ) + stats_code, output = run_sccache( + executable, + ["--show-stats", "--stats-format=json"], + env, + directory, + phase + "_stats_command", + ) + if code or stats_code: + return False + summary = stats_summary(output) + emit(phase + "_stats", **summary) + results.append(summary) + stop_code, _ = run_sccache( + executable, ["--stop-server"], env, directory, phase + "_stop" + ) + if stop_code: + return False + server_may_be_running = False + (directory / "probe.o").unlink(missing_ok=True) + return ( + all(item["s3_backend"] for item in results) + and results[0].get("cache_writes", 0) >= 1 + and results[1].get("cache_hits", 0) >= 1 + and all( + item.get(key, 0) == 0 + for item in results + for key in ( + "cache_errors", + "cache_read_errors", + "cache_write_errors", + "compile_fails", + ) + ) + ) + finally: + if server_may_be_running: + run_sccache(executable, ["--stop-server"], env, directory, "cleanup") + log = directory / "errors" + if log.exists(): + emit( + "sccache_log", + error_categories=error_categories( + "\n".join( + line + for line in log.read_text(errors="replace").splitlines() + if re.search(r"\b(?:ERROR|WARN)\b|Error executing cache write", line) + ) + ), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scope", choices=("host", "container"), required=True) + parser.add_argument("--bucket", help="enable the small S3 write/read probe") + parser.add_argument("--region", default="us-west-2") + parser.add_argument("--compiler", default="/usr/bin/gcc") + args = parser.parse_args() + emit( + "context", + scope=args.scope, + credential_environment={ + key: bool(os.environ.get(key)) + for key in ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_PROFILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_EC2_METADATA_DISABLED", + "HTTP_PROXY", + "HTTPS_PROXY", + ) + }, + ) + try: + metadata_ok = metadata_probe() + cache_ok = cache_probe(args.bucket, args.region, args.compiler) if args.bucket else None + emit("result", metadata_credentials_available=metadata_ok, s3_roundtrip=cache_ok) + # Metadata may be intentionally unavailable when other credentials work. + return 0 if (cache_ok if args.bucket else metadata_ok) else 1 + except Exception: + # Even unexpected exceptions must not print a credential-bearing traceback. + emit("result", result="probe_error") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())
