This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bce20ff30bb Catch lang-SDK Go example module drift before it reaches
main (#70568)
bce20ff30bb is described below
commit bce20ff30bbe9b96d0de5ee6456f2015611f4e53
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Jul 28 15:38:26 2026 +0200
Catch lang-SDK Go example module drift before it reaches main (#70568)
* Catch lang-SDK Go example module drift before it reaches main
kubernetes-tests/lang_sdk/go_example is a separate Go module that resolves
the SDK through a `replace` onto ../../../go-sdk, so it carries its own copy
of the SDK's indirect requirements. Nothing re-tidies it when a dependency
moves inside /go-sdk, and Dependabot bumps exactly one module per PR.
The blast radius is what makes this worth guarding. Go refuses to build an
inconsistent module, so once such a bump merges, "Kubernetes tests / K8S
Lang-SDK" fails at the Build Go bundle step on every pull request until
someone notices and tidies the module by hand — not just on the PR that
caused it.
Dependabot security updates do not consult .github/dependabot.yml, so no
per-directory configuration prevents this, and a second Dependabot PR for
the
example module would merge at a different time and leave main red in
between.
The drift has to fail the bump PR itself.
The check is `go mod tidy -diff` in the example module: it asks exactly the
question the failing CI step asks, never writes to the working tree, and
exits non-zero when the module is untidy.
* Let prek provide the Go toolchain for the tidy check
Static checks run on a runner whose preinstalled toolchains are deleted to
free disk space before prek starts, so the check could never find `go` there
and failed on every run. Asking prek for the toolchain is how the Go SDK's
own tidy hook already gets one, and it pins the same version everywhere.
---
.pre-commit-config.yaml | 17 +++
scripts/ci/prek/check_go_example_mod_tidy.py | 126 +++++++++++++++++++++
.../ci/prek/test_check_go_example_mod_tidy.py | 99 ++++++++++++++++
3 files changed, 242 insertions(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 9b448dca750..038d8ebc80e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -295,6 +295,23 @@ repos:
^\.pre-commit-config\.yaml$
pass_filenames: false
require_serial: true
+ - id: check-go-example-mod-tidy
+ name: Check lang-SDK Go example module is tidy against the Go SDK
+ entry: ./scripts/ci/prek/check_go_example_mod_tidy.py
+ # golang so prek provisions the toolchain the check needs: static
checks run on a
+ # runner whose preinstalled toolchains are wiped to free disk space,
and the SDK's
+ # own `go mod tidy` hook gets its Go the same way.
+ language: golang
+ # The example module keeps its own copy of the SDK's indirect
requirements
+ # (it resolves the SDK through a `replace`), so a dependency moving in
+ # go-sdk/go.mod leaves it stale. Watching both modules' manifests is
enough:
+ # any new requirement in the SDK necessarily lands in go-sdk/go.mod
first.
+ files: >
+ (?x)
+ ^go-sdk/go\.(mod|sum)$|
+ ^kubernetes-tests/lang_sdk/go_example/go\.(mod|sum)$
+ pass_filenames: false
+ require_serial: true
- id: check-partition-mapper-defaults-in-sync
name: Check partition-mapper core/SDK sync (FanOutMapper table +
SegmentWindow/FixedKeyMapper)
entry: ./scripts/ci/prek/check_partition_mapper_defaults_in_sync.py
diff --git a/scripts/ci/prek/check_go_example_mod_tidy.py
b/scripts/ci/prek/check_go_example_mod_tidy.py
new file mode 100755
index 00000000000..613e900106b
--- /dev/null
+++ b/scripts/ci/prek/check_go_example_mod_tidy.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Keep the lang-SDK Go example module tidy against the Go SDK.
+
+``kubernetes-tests/lang_sdk/go_example`` is a **separate** Go module that
+resolves the SDK from the in-repo sources::
+
+ replace github.com/apache/airflow/go-sdk => ../../../go-sdk
+
+Because of that ``replace`` it carries its own copy of the SDK's indirect
+requirements. Nothing re-tidies it when a dependency moves inside
+``/go-sdk`` — and Dependabot bumps exactly one module per PR. The example
+module is then left pinning the old versions, Go refuses to build an
+inconsistent module, and ``Kubernetes tests / K8S Lang-SDK`` fails at the
+"Build Go bundle" step::
+
+ go: updates to go.mod needed; to update it:
+ go mod tidy
+
+The damage is not limited to the bump PR: once it merges, that job is red on
+*every* pull request until someone notices and tidies the example module by
+hand. This happened with #70226 (``google.golang.org/grpc`` 1.79.3 -> 1.82.1
+in ``/go-sdk`` only) and was cleaned up after the fact by #70561.
+
+Note that Dependabot **security** updates do not consult
+``.github/dependabot.yml`` at all, so no amount of per-directory config
+prevents this — and a second Dependabot PR for the example module would merge
+at a different time, leaving ``main`` red in between. The drift has to fail
+the bump PR itself, which is what this check does.
+
+The check is ``go mod tidy -diff`` in the example module: it is the exact
+question the failing CI step asks, it never writes to the working tree, and it
+exits non-zero when the module is untidy.
+
+Run from the repo root:
+
+ uv run --project scripts python
scripts/ci/prek/check_go_example_mod_tidy.py
+
+Exits 0 if the example module is tidy, 1 otherwise.
+"""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+EXAMPLE_MODULE = pathlib.Path("kubernetes-tests/lang_sdk/go_example")
+GO_SDK_MODULE = pathlib.Path("go-sdk")
+
+
+def run_tidy_diff(module_dir: pathlib.Path, go_binary: str = "go") ->
tuple[int, str]:
+ """Ask Go whether ``module_dir`` is tidy. Returns ``(returncode,
combined_output)``."""
+ completed = subprocess.run(
+ [go_binary, "mod", "tidy", "-diff"],
+ cwd=module_dir,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ return completed.returncode, (completed.stdout + completed.stderr).strip()
+
+
+def format_report(returncode: int, output: str) -> tuple[int, str]:
+ """Turn a ``go mod tidy -diff`` result into ``(exit_code, report)``."""
+ if returncode == 0:
+ return 0, f"OK: {EXAMPLE_MODULE} is tidy against {GO_SDK_MODULE}."
+ lines = [
+ f"ERROR: {EXAMPLE_MODULE} is not tidy.",
+ "",
+ f"It is a separate Go module that resolves the SDK via a `replace`
onto {GO_SDK_MODULE},",
+ "so it keeps its own copy of the SDK's indirect requirements. A
dependency moved in",
+ f"{GO_SDK_MODULE} without this module being re-tidied, which breaks
the",
+ "'Kubernetes tests / K8S Lang-SDK' bundle build on every pull request
once merged.",
+ "",
+ "Fix it in this PR by running:",
+ "",
+ f" (cd {EXAMPLE_MODULE} && go mod tidy)",
+ "",
+ "and committing the resulting go.mod / go.sum changes.",
+ "",
+ "`go mod tidy -diff` reported:",
+ "",
+ output or "(no output)",
+ ]
+ return 1, "\n".join(lines)
+
+
+def main() -> int:
+ module_dir = REPO_ROOT / EXAMPLE_MODULE
+ if not (module_dir / "go.mod").is_file():
+ print(f"ERROR: {EXAMPLE_MODULE}/go.mod not found — has the example
module moved?")
+ return 1
+ if shutil.which("go") is None:
+ if os.environ.get("CI"):
+ print("ERROR: `go` is not on PATH but this is a CI run — the
toolchain is required here.")
+ return 1
+ print(f"SKIPPED: `go` is not on PATH, cannot verify that
{EXAMPLE_MODULE} is tidy.")
+ return 0
+ returncode, output = run_tidy_diff(module_dir)
+ exit_code, report = format_report(returncode, output)
+ print(report)
+ return exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/ci/prek/test_check_go_example_mod_tidy.py
b/scripts/tests/ci/prek/test_check_go_example_mod_tidy.py
new file mode 100644
index 00000000000..286d09efecc
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_go_example_mod_tidy.py
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import subprocess
+from unittest import mock
+
+import check_go_example_mod_tidy as checker
+import pytest
+
+# Trimmed to the shape that matters: the drift #70226 introduced and #70561
cleaned up.
+GRPC_DRIFT_DIFF = """\
+diff current/go.mod tidy/go.mod
+--- current/go.mod
++++ tidy/go.mod
+@@ -37,9 +37,9 @@
+- google.golang.org/grpc v1.79.3 // indirect
++ google.golang.org/grpc v1.82.1 // indirect
+"""
+
+
+def test_tidy_module_passes():
+ exit_code, report = checker.format_report(0, "")
+
+ assert exit_code == 0
+ assert "is tidy" in report
+
+
+def test_untidy_module_fails_with_the_fix_command_and_the_diff():
+ exit_code, report = checker.format_report(1, GRPC_DRIFT_DIFF)
+
+ assert exit_code == 1
+ assert "is not tidy" in report
+ assert "(cd kubernetes-tests/lang_sdk/go_example && go mod tidy)" in report
+ # The reason the contributor cares: this is what turns K8S Lang-SDK red
for everyone.
+ assert "K8S Lang-SDK" in report
+ assert "google.golang.org/grpc v1.82.1" in report
+
+
+def test_untidy_module_without_diff_output_still_reports():
+ exit_code, report = checker.format_report(1, "")
+
+ assert exit_code == 1
+ assert "(no output)" in report
+
+
[email protected]("check_go_example_mod_tidy.subprocess.run", autospec=True)
+def test_run_tidy_diff_never_writes_to_the_working_tree(mock_run, tmp_path):
+ mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0,
stdout="", stderr="")
+
+ checker.run_tidy_diff(tmp_path)
+
+ args = mock_run.call_args.args[0]
+ assert args == ["go", "mod", "tidy", "-diff"]
+ assert mock_run.call_args.kwargs["cwd"] == tmp_path
+
+
[email protected]("check_go_example_mod_tidy.subprocess.run", autospec=True)
+def test_run_tidy_diff_combines_stdout_and_stderr(mock_run, tmp_path):
+ mock_run.return_value = subprocess.CompletedProcess(
+ args=[], returncode=1, stdout="diff current/go.mod tidy/go.mod\n",
stderr="go: downloading\n"
+ )
+
+ returncode, output = checker.run_tidy_diff(tmp_path)
+
+ assert returncode == 1
+ assert "diff current/go.mod tidy/go.mod" in output
+ assert "go: downloading" in output
+
+
[email protected](
+ ("ci_env", "expected_exit", "expected_text"),
+ [
+ pytest.param({"CI": "true"}, 1, "this is a CI run",
id="ci-fails-loudly"),
+ pytest.param({}, 0, "SKIPPED", id="local-skips"),
+ ],
+)
[email protected]("check_go_example_mod_tidy.shutil.which", return_value=None)
+def test_missing_go_toolchain(mock_which, ci_env, expected_exit,
expected_text, monkeypatch, capsys):
+ monkeypatch.delenv("CI", raising=False)
+ for key, value in ci_env.items():
+ monkeypatch.setenv(key, value)
+
+ assert checker.main() == expected_exit
+ assert expected_text in capsys.readouterr().out
