This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch ci-image-keyed-cache in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0f0a97162db9d5745b7f6338c63d597daecbf3bc Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 15 16:04:11 2026 +0900 Push keyed cache for every Python version and sweep stale keyed tags The keyed cache a pull request pushes followed the test matrix, which a dependency change usually narrows to the default Python version, so the others still built dependencies from scratch after the merge. A keyed cache version that kept the shared tag past the keyed cleanup's scanned window escaped that cleanup for good; the slow cleanup already walks every old version, so it takes those as well. Generated-by: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NyQq9U7i5HhjDgrQ9qd3gq --- .github/workflows/ci-amd.yml | 4 +- .github/workflows/ci-arm.yml | 4 +- dev/breeze/doc/ci/04_selective_checks.md | 1 + .../src/airflow_breeze/utils/selective_checks.py | 6 +++ dev/breeze/tests/test_ci_image_commands.py | 6 +++ dev/breeze/tests/test_selective_checks.py | 13 ++++++ scripts/ci/cleanup_keyed_image_cache.py | 3 +- scripts/ci/cleanup_old_image_versions.py | 18 ++++++-- .../tests/ci/test_cleanup_old_image_versions.py | 52 ++++++++++++++++++++++ 9 files changed, 99 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 2584151dec4..4b11afb210d 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -277,7 +277,7 @@ jobs: arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} canary-run: ${{ steps.source-run-info.outputs.canary-run }} - all-python-versions: ${{ steps.selective-checks.outputs.all-python-versions }} + ci-image-cache-python-versions: ${{ steps.selective-checks.outputs.ci-image-cache-python-versions }} ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} ci-image-dependencies-changed: ${{ steps.selective-checks.outputs.ci-image-dependencies-changed }} core-test-types-list-as-strings-in-json: >- @@ -579,7 +579,7 @@ jobs: include-prod-images: "false" push-latest-images: "false" keyed-cache-only: "true" - python-versions: ${{ needs.build-info.outputs.all-python-versions }} + python-versions: ${{ needs.build-info.outputs.ci-image-cache-python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} use-uv: ${{ needs.build-info.outputs.use-uv }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index ccb40c5c6e4..ff16edd40f8 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -153,7 +153,7 @@ jobs: arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} canary-run: ${{ steps.source-run-info.outputs.canary-run }} - all-python-versions: ${{ steps.selective-checks.outputs.all-python-versions }} + ci-image-cache-python-versions: ${{ steps.selective-checks.outputs.ci-image-cache-python-versions }} ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} ci-image-dependencies-changed: ${{ steps.selective-checks.outputs.ci-image-dependencies-changed }} core-test-types-list-as-strings-in-json: >- @@ -455,7 +455,7 @@ jobs: include-prod-images: "false" push-latest-images: "false" keyed-cache-only: "true" - python-versions: ${{ needs.build-info.outputs.all-python-versions }} + python-versions: ${{ needs.build-info.outputs.ci-image-cache-python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} use-uv: ${{ needs.build-info.outputs.use-uv }} diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 40932879727..b1e21f16438 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -754,6 +754,7 @@ GitHub Actions to pass the list of parameters to a command to execute | all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | | basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | | ci-image-build | Whether CI image build is needed | true | | +| ci-image-cache-python-versions | Every supported Python version, for the keyed image cache an apache/airflow PR pushes | \['3.11', '3.12'\] | | | ci-image-dependencies-changed | Whether dependency inputs of the CI image changed (an apache/airflow PR then pushes its keyed cache) | false | | | core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core (2 slices; 1 on RunsOn) | API Always Providers | * | | debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 33faee53d50..d6291e2acd3 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -2281,6 +2281,12 @@ class SelectiveChecks: return True return False + @cached_property + def ci_image_cache_python_versions(self) -> list[str]: + # Every supported version, whatever the test matrix: a keyed cache missing for one version + # means a full dependency build for everyone using it. + return CURRENT_PYTHON_MAJOR_MINOR_VERSIONS + @cached_property def ci_image_dependencies_changed(self) -> bool: return len(self._matching_files(FileGroupForCi.CI_IMAGE_DEPENDENCY_FILES, CI_FILE_GROUP_MATCHES)) > 0 diff --git a/dev/breeze/tests/test_ci_image_commands.py b/dev/breeze/tests/test_ci_image_commands.py index af1ec3500e5..431ffb243db 100644 --- a/dev/breeze/tests/test_ci_image_commands.py +++ b/dev/breeze/tests/test_ci_image_commands.py @@ -181,6 +181,7 @@ def test_build_ci_image_if_needed_reuses_image_built_in_another_checkout( mock_run_build_ci_image.assert_not_called() [email protected]("airflow_breeze.commands.ci_image_commands.check_buildx_version") @mock.patch("airflow_breeze.commands.ci_image_commands.check_if_image_building_is_needed") @mock.patch("airflow_breeze.commands.ci_image_commands.run_build_ci_image") @mock.patch("airflow_breeze.commands.ci_image_commands.is_ci_image_built_from_current_sources") @@ -188,6 +189,7 @@ def test_build_ci_image_if_needed_forces_build_when_image_does_not_match_sources mock_is_ci_image_built_from_current_sources, mock_run_build_ci_image, mock_check_if_image_building_is_needed, + mock_check_buildx_version, tmp_path, monkeypatch, ): @@ -198,8 +200,10 @@ def test_build_ci_image_if_needed_forces_build_when_image_does_not_match_sources build_ci_image_if_needed(command_params=BuildCiParams()) assert mock_check_if_image_building_is_needed.call_args.kwargs["ci_image_params"].force_build is True mock_run_build_ci_image.assert_called_once() + mock_check_buildx_version.assert_called_once_with(quiet=True) [email protected]("airflow_breeze.commands.ci_image_commands.check_buildx_version") @mock.patch("airflow_breeze.commands.ci_image_commands.check_if_image_building_is_needed") @mock.patch("airflow_breeze.commands.ci_image_commands.run_build_ci_image") @mock.patch("airflow_breeze.commands.ci_image_commands.is_ci_image_built_from_current_sources") @@ -207,6 +211,7 @@ def test_build_ci_image_if_needed_does_not_reuse_image_when_force_build_requeste mock_is_ci_image_built_from_current_sources, mock_run_build_ci_image, mock_check_if_image_building_is_needed, + mock_check_buildx_version, tmp_path, monkeypatch, ): @@ -216,6 +221,7 @@ def test_build_ci_image_if_needed_does_not_reuse_image_when_force_build_requeste build_ci_image_if_needed(command_params=BuildCiParams(force_build=True)) mock_is_ci_image_built_from_current_sources.assert_not_called() mock_run_build_ci_image.assert_called_once() + mock_check_buildx_version.assert_called_once_with(quiet=True) @mock.patch("airflow_breeze.commands.ci_image_commands.check_if_image_building_is_needed") diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 75a61091e42..f7c684a2d25 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -3802,6 +3802,19 @@ def test_ci_image_dependencies_changed(files: tuple[str, ...], expected: str): assert_outputs_are_printed({"ci-image-dependencies-changed": expected}, str(stderr)) +def test_ci_image_cache_python_versions_are_all_current_versions(): + stderr = SelectiveChecks( + files=("providers/amazon/pyproject.toml",), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.PULL_REQUEST, + default_branch="main", + ) + assert_outputs_are_printed( + {"ci-image-cache-python-versions": CURRENT_PYTHON_VERSIONS_AS_LIST}, + str(stderr), + ) + + @pytest.mark.parametrize( ("files", "pr_labels", "github_event", "expected_constraints", "expected_pypi"), [ diff --git a/scripts/ci/cleanup_keyed_image_cache.py b/scripts/ci/cleanup_keyed_image_cache.py index 8520661309a..78b540296e6 100644 --- a/scripts/ci/cleanup_keyed_image_cache.py +++ b/scripts/ci/cleanup_keyed_image_cache.py @@ -27,7 +27,8 @@ and anything else are never touched. The age is counted from creation, not from the last update: a push of identical cache content reuses the version and only moves its ``updated_at``, and versions are listed by creation, so a version updated after creation could otherwise fall out of the scanned window and stay forever. -A key still in use is pushed again within hours of being deleted. +A key still in use is pushed again within hours of being deleted. A version that kept another +tag past the scanned window is left to cleanup_old_image_versions.py, which walks every version. Usage: cleanup_keyed_image_cache.py [--days N] [--dry-run] Environment: GITHUB_REPOSITORY (``owner/repo``), and a token for ``gh`` with ``packages: write``. diff --git a/scripts/ci/cleanup_old_image_versions.py b/scripts/ci/cleanup_old_image_versions.py index 3d321ea10f4..48201cfbfae 100644 --- a/scripts/ci/cleanup_old_image_versions.py +++ b/scripts/ci/cleanup_old_image_versions.py @@ -61,6 +61,8 @@ MANIFEST_MEDIA_TYPES = ", ".join( ) LINK_LAST_PAGE = re.compile(r'[?&]page=(\d+)>; rel="last"') PUSH_WINDOW = timedelta(minutes=5) +# The keyed CI image cache tags, see cleanup_keyed_image_cache.py. +KEYED_CACHE_TAG = re.compile(r"^cache-linux-(amd64|arm64)-[0-9a-f]{16}$") def get_image_package_pattern(repo: str) -> re.Pattern[str]: @@ -79,9 +81,17 @@ def _get_tags(version: dict) -> list[str]: return version.get("metadata", {}).get("container", {}).get("tags") or [] +def _is_disposable(version: dict) -> bool: + """Untagged, or tagged only as keyed cache, which nothing pulls by reference.""" + return all(KEYED_CACHE_TAG.match(tag) for tag in _get_tags(version)) + + def find_deletable_versions(versions: list[dict], keep: set[str], cutoff: datetime) -> list[dict]: """ - Return the untagged versions of one page that are safe to delete. + Return the versions of one page that are safe to delete. + + Untagged versions, and the keyed cache tags that the daily keyed-cache cleanup misses when a + version kept another tag past its scan window, are deleted once nothing references them. ``keep`` holds every manifest a tag references. As a second guard, an untagged version pushed within PUSH_WINDOW of a tagged one, or of the page edges where a tagged one may sit on the @@ -89,12 +99,12 @@ def find_deletable_versions(versions: list[dict], keep: set[str], cutoff: dateti """ if not versions: return [] - anchors = [_parse_time(v["created_at"]) for v in versions if _get_tags(v)] + anchors = [_parse_time(v["created_at"]) for v in versions if not _is_disposable(v)] anchors += [_parse_time(versions[0]["created_at"]), _parse_time(versions[-1]["created_at"])] return [ version for version in versions - if not _get_tags(version) + if _is_disposable(version) and version["name"] not in keep and (created := _parse_time(version["created_at"])) < cutoff and all(abs(created - anchor) > PUSH_WINDOW for anchor in anchors) @@ -163,6 +173,8 @@ def collect_keep_set(image: str) -> set[str]: next_page = re.search(r"<([^>]+)>;\s*rel=\"next\"", link) path = next_page.group(1) if next_page else None + tags = [tag for tag in tags if not KEYED_CACHE_TAG.match(tag)] + def read_tag(tag: str) -> set[str]: manifest, digest, _ = registry.get(f"/v2/{image}/manifests/{tag}", MANIFEST_MEDIA_TYPES) return get_referenced_digests(manifest) | ({digest} if digest else set()) diff --git a/scripts/tests/ci/test_cleanup_old_image_versions.py b/scripts/tests/ci/test_cleanup_old_image_versions.py index c1542963867..73e4c0ee89d 100644 --- a/scripts/tests/ci/test_cleanup_old_image_versions.py +++ b/scripts/tests/ci/test_cleanup_old_image_versions.py @@ -145,6 +145,58 @@ def test_select_versions_to_delete_does_not_load_tags_without_candidates(cleanup assert cleanup.select_versions_to_delete(pages.get, 1, load_keep, CUTOFF, budget=10) == [] +def test_find_deletable_versions_takes_keyed_cache_only_versions(cleanup): + keyed = _version("sha256:keyed", OLD, ["cache-linux-amd64-0123456789abcdef"]) + shared_too = _version( + "sha256:shared", "2026-08-20T00:00:00Z", ["cache-linux-amd64-0123456789abcdef", "cache-linux-amd64"] + ) + + deletable = cleanup.find_deletable_versions(_page(keyed, shared_too), set(), CUTOFF) + + assert [v["name"] for v in deletable] == ["sha256:keyed"] + + +class _FakeRegistry: + """Two pages of tags; the commit-tagged index, on the second page, lists an old manifest.""" + + def __init__(self, image: str): + self.image = image + + def get(self, path: str, accept: str = "application/json"): + pages: dict[str, tuple[dict, str]] = { + f"/v2/{self.image}/tags/list?n=1000": ( + {"tags": ["latest", "cache-linux-amd64-0123456789abcdef"]}, + f'</v2/{self.image}/tags/list?n=1000&last=latest>; rel="next"', + ), + f"/v2/{self.image}/tags/list?n=1000&last=latest": ({"tags": ["0123abcd4567ef89"]}, ""), + } + if path in pages: + listing, link = pages[path] + return listing, None, link + manifests: dict[str, tuple[dict, str]] = { + "latest": ({}, "sha256:latest"), + "0123abcd4567ef89": ( + {"manifests": [{"digest": "sha256:old-platform-manifest"}]}, + "sha256:commit-index", + ), + } + tag = path.rsplit("/", 1)[1] + if tag not in manifests: + raise AssertionError(f"unexpected manifest read: {tag}") + body, digest = manifests[tag] + return body, digest, "" + + +def test_collect_keep_set_protects_manifests_listed_by_commit_tags(cleanup, monkeypatch): + monkeypatch.setattr(cleanup, "_Registry", _FakeRegistry) + + keep = cleanup.collect_keep_set("apache/airflow/main/ci/python3.11") + + assert keep == {"sha256:latest", "sha256:commit-index", "sha256:old-platform-manifest"} + old_manifest = _version("sha256:old-platform-manifest", "2026-08-20T00:00:00Z") + assert cleanup.find_deletable_versions(_page(old_manifest), keep, CUTOFF) == [] + + def test_get_referenced_digests_lists_index_children(cleanup): index = { "mediaType": "application/vnd.oci.image.index.v1+json",
