This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch refresh-image-registry-cache-workflow in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b9f9d25db4e80b7ddfe9832959fa9fc19dee16af Author: Jarek Potiuk <[email protected]> AuthorDate: Wed Aug 5 17:58:22 2026 +0800 Refresh the image registry cache in a workflow that cannot be cancelled A branch built on push refreshed its registry cache from inside the CI run, which cancels in progress on the next push. With no cache the build runs cold, which widens the window the next push cancels in, which leaves the cache missing -- v3-3-test held only the default Python version's amd64 cache because that was the one matrix entry that finished in time. The refresh also has to cover every Python version: a version with no cache entry builds from scratch on every run, and selective checks narrows the list on a text-only push. --- .github/workflows/additional-ci-image-checks.yml | 73 +---------- .github/workflows/ci-amd.yml | 7 -- .github/workflows/ci-arm.yml | 7 -- .github/workflows/refresh-image-registry-cache.yml | 138 +++++++++++++++++++++ dev/breeze/tests/test_selective_checks.py | 21 ++++ 5 files changed, 161 insertions(+), 85 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index d0a4cb55fa6..42b71e1f32b 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -28,18 +28,10 @@ on: # yamllint disable-line rule:truthy description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string - python-versions: - description: "The list of python versions (stringified JSON array) to run the tests on." - required: true - type: string branch: description: "Branch used to run the CI jobs in (main/v*_*_test)." required: true type: string - constraints-branch: - description: "Branch used to get constraints from" - required: true - type: string default-python-version: description: "Which version of python should be used by default" required: true @@ -52,80 +44,19 @@ on: # yamllint disable-line rule:truthy description: "Whether to skip prek hooks (true/false)" required: true type: string - docker-cache: - description: "Docker cache specification to build the image (registry, local, disabled)." - required: true - type: string - disable-airflow-repo-cache: - description: "Disable airflow repo cache read from main." - required: true - type: string - canary-run: - description: "Whether this is a canary run (true/false)" - required: true - type: string latest-versions-only: description: "Whether to run only latest versions (true/false)" required: true type: string - include-success-outputs: - description: "Whether to include success outputs (true/false)" - required: true - type: string debug-resources: description: "Whether to debug resources (true/false)" required: true type: string - use-uv: - description: "Whether to use uv to build the image (true/false)" - required: true - type: string permissions: contents: read jobs: - # Push early BuildX cache to GitHub Registry in Apache repository. This cache does not wait for all the - # tests to complete - it is run very early in the build process in order to refresh cache using the - # current constraints. This will speed up cache refresh in cases when pyproject.toml changes or in case - # of Dockerfile changes. Failure in this step is not a problem (at most it will delay cache refresh). - # It does not attempt to upgrade to newer dependencies. - # We only push CI cache as PROD cache usually does not gain as much from fresh cache because - # it uses prepared airflow and provider distributions that invalidate the cache anyway most of the time - # - # Runs on `push` as well, not only on the scheduled canary. Release branches (`v*-*-test`) are built - # by `push` only, and their other refresh path -- the Regular cache push in `finalize-tests.yml` -- sits - # behind the whole test matrix. A branch that is red therefore never refreshes its cache at all, and the - # staler the cache gets the slower it is to turn green again. `main` is unaffected either way: it is not - # built on push, so it keeps refreshing through the scheduled canary. - # - # Pull requests stay excluded -- `canary-run` is also true for a PR carrying the `canary` label, and - # pushing cache from an unmerged branch is not something a label should authorise. This is the same - # condition the Regular cache push uses. - push-early-buildx-cache-to-github-registry: - name: Push Early Image Cache - uses: ./.github/workflows/push-image-cache.yml - permissions: - contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - packages: write - with: - runners: ${{ inputs.runners }} - cache-type: "Early" - include-prod-images: "false" - push-latest-images: "false" - platform: ${{ inputs.platform }} - python-versions: ${{ inputs.python-versions }} - branch: ${{ inputs.branch }} - constraints-branch: ${{ inputs.constraints-branch }} - use-uv: ${{ inputs.use-uv }} - include-success-outputs: ${{ inputs.include-success-outputs }} - docker-cache: ${{ inputs.docker-cache }} - disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} - if: inputs.canary-run == 'true' && github.event_name != 'pull_request' - - # Check that after earlier cache push, breeze command will build quickly - # This build is a bit slow from in-the scratch builds, so we should run it only in - # regular PRs + # Check that the image builds quickly from the registry cache. This build is slow from + # scratch, so we only run it on regular PRs. check-that-image-builds-quickly: timeout-minutes: 25 name: Check that image builds quickly diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 72ef23494d7..63b07e4ce6d 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -297,19 +297,12 @@ jobs: with: runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} - python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} - docker-cache: ${{ needs.build-info.outputs.docker-cache }} - disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} - canary-run: ${{ needs.build-info.outputs.canary-run }} latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.use-uv }} generate-constraints: name: "Generate constraints" diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 8c62297ea7e..0c179002e87 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -286,19 +286,12 @@ jobs: with: runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} - python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} - docker-cache: ${{ needs.build-info.outputs.docker-cache }} - disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} - canary-run: ${{ needs.build-info.outputs.canary-run }} latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} - include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} - use-uv: ${{ needs.build-info.outputs.use-uv }} generate-constraints: name: "Generate constraints" diff --git a/.github/workflows/refresh-image-registry-cache.yml b/.github/workflows/refresh-image-registry-cache.yml new file mode 100644 index 00000000000..c78c3fc8069 --- /dev/null +++ b/.github/workflows/refresh-image-registry-cache.yml @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Refresh image registry cache +# Refreshes the CI image cache in the GitHub registry, in a run of its own. +# +# It has to be its own workflow rather than a job inside `ci-amd.yml` / `ci-arm.yml`, because +# those cancel in progress on the next push and a job cannot opt out of its run being cancelled. +# A cache refresh is the one job that must not be cancelled: with no cache the branch builds cold +# (~20 minutes per Python version), which widens the window in which the next push cancels the +# run, which leaves the cache missing. `v3-3-test` sat in that loop -- its `linux/amd64` cache +# covered the default Python version alone, the single matrix entry that finished before the next +# push killed the other four seconds later. +# +# Every Python version is refreshed in both trigger paths. That is what makes the cache useful: +# a version with no cache entry builds from scratch on every run, whichever way it got skipped. +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + platform: + description: "Platform to refresh the cache for" + type: choice + options: + - "both" + - "linux/amd64" + - "linux/arm64" + default: "both" + push: + # Release-prep and providers branches only -- they are built on push, so the CI run that + # would have refreshed their cache is also the run that keeps being cancelled. `main` is not + # built on push; its cache is refreshed by the Push Image Cache job at the end of the + # scheduled canary, which runs to completion. + branches: + - v[0-9]+-[0-9]+-test + - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ +permissions: + contents: read +concurrency: + group: refresh-image-registry-cache-${{ github.ref }} + cancel-in-progress: false +jobs: + build-info: + name: "Build info" + # No outputs yet at this stage, so the runner has to be hard-coded to a public one. + runs-on: ["ubuntu-22.04"] + outputs: + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} + docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} + include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: Selective checks + id: selective-checks + env: + # Forces every Python version. Selective checks narrows `python-versions` down to the + # default one on a push where only text files changed + # (`_should_run_all_tests_and_versions`), which would leave the rest of the versions + # with no cache -- the failure mode this workflow exists to prevent. + PR_LABELS: '["all versions"]' + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + + refresh-ci-cache-amd: + name: "Refresh cache linux/amd64" + needs: [build-info] + uses: ./.github/workflows/push-image-cache.yml + permissions: + contents: read + # Only granted for `push` / `workflow_dispatch` on apache/airflow itself, never for forks. + packages: write + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + cache-type: "Refresh" + # The PROD cache is built from distributions prepared earlier in a CI run, and a standalone + # refresh has no such artifact. The CI cache is also where the build time goes. + include-prod-images: "false" + # `latest` is what a build pulls when it has no cache at all. Moving it belongs to a canary + # run that has passed its tests, not to a cache refresh. + push-latest-images: "false" + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + if: github.event_name == 'push' || inputs.platform == 'both' || inputs.platform == 'linux/amd64' + + refresh-ci-cache-arm: + name: "Refresh cache linux/arm64" + needs: [build-info] + uses: ./.github/workflows/push-image-cache.yml + permissions: + contents: read + packages: write + with: + runners: '["ubuntu-22.04-arm"]' + platform: "linux/arm64" + cache-type: "Refresh" + include-prod-images: "false" + push-latest-images: "false" + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + if: github.event_name == 'push' || inputs.platform == 'both' || inputs.platform == 'linux/arm64' diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index a99d5c7bf86..d516aaeee4f 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -29,6 +29,7 @@ from airflow_breeze.global_constants import ( ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, CI_AMD_PLATFORM, CI_ARM_PLATFORM, + CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, DEFAULT_KUBERNETES_VERSION, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, JAVA_SDK_VERSION, @@ -56,6 +57,9 @@ ALL_KUBERNETES_VERSIONS_AS_STRING = " ".join(ALLOWED_KUBERNETES_VERSIONS) ALL_KUBERNETES_VERSIONS_AS_LIST = "[" + ", ".join([f"'{v}'" for v in ALLOWED_KUBERNETES_VERSIONS]) + "]" ALL_PYTHON_VERSIONS_AS_STRING = " ".join(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS) ALL_PYTHON_VERSIONS_AS_LIST = "[" + ", ".join([f"'{v}'" for v in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS]) + "]" +CURRENT_PYTHON_VERSIONS_AS_LIST = ( + "[" + ", ".join([f"'{v}'" for v in CURRENT_PYTHON_MAJOR_MINOR_VERSIONS]) + "]" +) DEFAULT_HELM_K8S_VERSION = ALLOWED_KUBERNETES_VERSIONS[0].lstrip("v") LAST_HELM_K8S_VERSION = ALLOWED_KUBERNETES_VERSIONS[-1].lstrip("v") @@ -2884,6 +2888,23 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): ) +# The image cache is pushed for `python-versions`, so a narrowed list leaves the remaining +# versions with no cache and they build from scratch on every run. `refresh-image-registry-cache.yml` +# forces the label below for exactly that reason; this pins the behaviour it relies on. + + +def test_all_python_versions_with_all_versions_label(): + """Set on the event that would otherwise narrow: a text-only push.""" + stderr = SelectiveChecks( + files=("INTHEWILD.md",), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.PUSH, + pr_labels=("all versions",), + default_branch="main", + ) + assert_outputs_are_printed({"python-versions": CURRENT_PYTHON_VERSIONS_AS_LIST}, str(stderr)) + + @pytest.mark.parametrize( "github_event", [
