This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch pin-os-deps-image-layer in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 008485dc8bfe6e6aa8f3e89bbdc98126eb4342ce Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Nov 1 10:40:34 2026 +0100 Build the OS-dependencies images daily instead of on every mirror run Docker rebuilds the hardened base images several times a day, and rebuilding every OS-dependencies image for each new digest would run a full set of image builds on every mirror run. Building them daily is enough to keep the pinned images close to the apt repository, and the upgrade check no longer triggers builds of its own. As the base image pins now move more often than the OS-dependencies images, the upgrade check pins the latest published image of each Python patch release, whate [...] Generated-by: Claude Opus 5 --- .github/workflows/mirror-hardened-base-images.yml | 33 ++++++++++++++++------ .github/workflows/upgrade-check.yml | 8 +++--- dev/breeze/doc/09_release_management_tasks.rst | 12 +++++--- .../output_workflow-run_mirror-base-images.svg | 30 +++++++++++++++----- .../output_workflow-run_mirror-base-images.txt | 2 +- .../src/airflow_breeze/commands/ci_commands.py | 1 + .../commands/release_management_commands.py | 22 +++++++-------- .../airflow_breeze/commands/workflow_commands.py | 15 ++++++++-- .../commands/workflow_commands_config.py | 1 + dev/breeze/src/airflow_breeze/global_constants.py | 6 ++-- .../src/airflow_breeze/utils/gh_workflow_utils.py | 8 ++++-- dev/breeze/tests/test_ci_upgrade.py | 1 + dev/breeze/tests/test_gh_workflow_utils.py | 6 ++-- dev/breeze/tests/test_os_deps_images.py | 11 ++++---- dev/breeze/tests/test_workflow_commands.py | 19 +++++++++---- docker-stack-docs/build.rst | 4 +-- scripts/ci/prek/upgrade_important_versions.py | 24 ++++++++-------- .../ci/prek/test_upgrade_important_versions.py | 13 ++++++--- 18 files changed, 143 insertions(+), 73 deletions(-) diff --git a/.github/workflows/mirror-hardened-base-images.yml b/.github/workflows/mirror-hardened-base-images.yml index c66a397e417..e689c741cea 100644 --- a/.github/workflows/mirror-hardened-base-images.yml +++ b/.github/workflows/mirror-hardened-base-images.yml @@ -31,13 +31,16 @@ name: Mirror hardened base images # The builds do not use these moving tags: they are pinned to digests that only "breeze ci upgrade" # moves, in a PR that has to pass the full CI. A pinned image cannot pin the dhi.io apt repository # the OS dependencies are installed from, though: it moves on - and drops the versions it replaced - -# whenever Docker rebuilds its images. So this workflow also installs the OS dependencies once, in +# whenever Docker rebuilds its images. So the daily run also installs the OS dependencies once, in # images built on the mirrored base images, and the CI and release builds start from those, pinned -# the same way. An OS-dependencies image is built only when one of its inputs changed - the base -# image, install_os_dependencies.sh or its Dockerfile stage - as every input is part of its tag. +# the same way. The six-hourly runs only mirror. An OS-dependencies image is built only when one of +# its inputs changed - the base image, install_os_dependencies.sh or its Dockerfile stage - as every +# input is part of its tag. # # A run that moves a mirrored tag past its pin, or publishes a new OS-dependencies image, triggers -# the upgrade check, which moves both pins together, so they follow Docker's rebuilds within hours. +# the upgrade check. It pins the latest published OS-dependencies image of each Python patch release, +# whatever base image digest it was built on: the image carries its own Python, so it does not have +# to share the base image pin's digest. on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: @@ -47,8 +50,16 @@ on: # yamllint disable-line rule:truthy published. "breeze ci upgrade" for main turns it off, as it is the upgrade that pins the images. type: boolean default: true + build-os-deps-images: + description: >- + Also build and publish the OS-dependencies images. The upgrade check turns it off: it pins the + images the daily build published. + type: boolean + default: true schedule: - cron: '27 */6 * * *' + # The daily run also builds the OS-dependencies images - the jobs below match this exact string. + - cron: '47 2 * * *' permissions: contents: read concurrency: @@ -146,16 +157,18 @@ jobs: - name: "Mirror base images" id: mirror run: breeze release-management mirror-base-images - # Each platform builds natively, on a runner of its own, and each Python version in a job of its - # own. The OS-dependencies images are built on the tags the mirror serves now - the digests the - # upgrade check pins next - and only for the Python versions and the Debian version main builds on. + # Daily, or when dispatched with build-os-deps-images. Each platform builds natively, on a runner of + # its own, and each Python version in a job of its own. The OS-dependencies images are built on the + # tags the mirror serves now, and only for the Python versions and the Debian version main builds on. build-os-deps-images: name: "Build OS-dependencies images ${{ matrix.platform }}:${{ matrix.python }}" needs: [build-info, mirror] # Also after a failed mirror: the tags it did mirror are served, and the builds look them up. if: >- !cancelled() && needs.build-info.result == 'success' && - github.repository == 'apache/airflow' && github.ref == 'refs/heads/main' + github.repository == 'apache/airflow' && github.ref == 'refs/heads/main' && + ((github.event_name == 'schedule' && github.event.schedule == '47 2 * * *') || + (github.event_name == 'workflow_dispatch' && inputs.build-os-deps-images)) strategy: fail-fast: false matrix: @@ -197,7 +210,9 @@ jobs: # rest. if: >- !cancelled() && needs.build-info.result == 'success' && - github.repository == 'apache/airflow' && github.ref == 'refs/heads/main' + github.repository == 'apache/airflow' && github.ref == 'refs/heads/main' && + ((github.event_name == 'schedule' && github.event.schedule == '47 2 * * *') || + (github.event_name == 'workflow_dispatch' && inputs.build-os-deps-images)) runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '2+4', '8+16') }} timeout-minutes: 30 permissions: diff --git a/.github/workflows/upgrade-check.yml b/.github/workflows/upgrade-check.yml index bab6d24310d..04344a3c6e9 100644 --- a/.github/workflows/upgrade-check.yml +++ b/.github/workflows/upgrade-check.yml @@ -77,8 +77,7 @@ jobs: # A job of its own, so the token that may dispatch workflows never reaches the upgrade job, which # runs the third-party tools and hooks it upgrades. It only dispatches the mirror and waits for it. mirror-base-images: - # The mirror run also builds the OS-dependencies images on newly mirrored base images. - timeout-minutes: 180 + timeout-minutes: 30 name: >- [${{ inputs.target-branch }}] Mirror hardened base images runs-on: ["ubuntu-slim"] @@ -97,13 +96,14 @@ jobs: uses: ./.github/actions/breeze with: free-space: "false" - # The upgrade for main is the one the mirror would otherwise start a second time. + # The upgrade for main is the one the mirror would otherwise start a second time. The upgrade pins + # the OS-dependencies images the daily build published, so this run does not build new ones. - name: >- [${{ inputs.target-branch }}] Mirror the base images and wait for the run env: TRIGGER_UPGRADE_CHECK_FLAG: >- ${{ inputs.target-branch == 'main' && '--no-trigger-upgrade-check' || '--trigger-upgrade-check' }} - run: breeze workflow-run mirror-base-images "${TRIGGER_UPGRADE_CHECK_FLAG}" + run: breeze workflow-run mirror-base-images "${TRIGGER_UPGRADE_CHECK_FLAG}" --no-build-os-deps-images createupgrade-check: timeout-minutes: 45 name: >- diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index a3dd4cf31cf..0596878978e 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -1353,8 +1353,9 @@ already ran, one per installation type (``ci``, ``dev`` and ``runtime``). They a mirror serves, and pinned in ``dev/breeze/src/airflow_breeze/files/hardened_os_deps_image_digests.json`` by the same upgrade that moves the base image pins. -The "Mirror hardened base images" workflow runs this command natively on an AMD and an ARM runner after -it mirrored the base images. Every input of an image - the base image digest, the script, its Dockerfile +The "Mirror hardened base images" workflow runs this command daily, natively on an AMD and an ARM runner, +after it mirrored the base images - its six-hourly runs only mirror, and the run the upgrade check +dispatches (``breeze workflow-run mirror-base-images --no-build-os-deps-images``) builds nothing either. Every input of an image - the base image digest, the script, its Dockerfile stage and its build args - is part of its tag, so an image is built only when one of them changed. These are all available flags of ``release-management build-os-deps-images`` command: @@ -1381,8 +1382,11 @@ Publish the OS-dependencies images """""""""""""""""""""""""""""""""" Once both platforms are built, this command merges them into a multi-platform image and moves the tag -the upgrade check pins - the latest OS-dependencies image built on a given digest of the base image - -to it. When it moves a tag, the workflow triggers the upgrade check. +the upgrade check pins - the latest published OS-dependencies image of a Python patch release - to it. +The merged image keeps its own tag, which has every input in it and never moves, so a pinned digest +stays tagged. When it moves a tag, the workflow triggers the upgrade check, which pins the latest image +of each Python patch release whatever base image digest it was built on: the image carries its own +Python, so it need not share the digest of the base image pin, which moves several times a day. These are all available flags of ``release-management publish-os-deps-images`` command: diff --git a/dev/breeze/doc/images/output_workflow-run_mirror-base-images.svg b/dev/breeze/doc/images/output_workflow-run_mirror-base-images.svg index 9c61d5dd5f0..9562fa46d7b 100644 --- a/dev/breeze/doc/images/output_workflow-run_mirror-base-images.svg +++ b/dev/breeze/doc/images/output_workflow-run_mirror-base-images.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 367.2" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 464.79999999999995" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -42,7 +42,7 @@ <defs> <clipPath id="breeze-workflow-run-mirror-base-images-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="316.2" /> + <rect x="0" y="0" width="1463.0" height="413.79999999999995" /> </clipPath> <clipPath id="breeze-workflow-run-mirror-base-images-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -80,9 +80,21 @@ <clipPath id="breeze-workflow-run-mirror-base-images-line-11"> <rect x="0" y="269.9" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-workflow-run-mirror-base-images-line-12"> + <rect x="0" y="294.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-mirror-base-images-line-13"> + <rect x="0" y="318.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-mirror-base-images-line-14"> + <rect x="0" y="343.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-workflow-run-mirror-base-images-line-15"> + <rect x="0" y="367.5" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="365.2" rx="8"/><text class="breeze-workflow-run-mirror-base-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: workflow-run mirror-base-images</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="462.8" rx="8"/><text class="breeze-workflow-run-mirror-base-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: workflow-run mirror-base-images</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -101,10 +113,14 @@ </text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-6)">│</text><text class="breeze-workflow-run-mirror-base-images-r4" x="24.4" y="166.4" textLength="280.6" clip-path="url(#breeze-workflow-run-mirror-base-images-line-6)">--trigger-upgrade-check</text><text class="breeze-workflow-run-mirror-base-images-r1" x="305" y="166.4" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror- [...] </text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-7)">│</text><text class="breeze-workflow-run-mirror-base-images-r1" x="658.8" y="190.8" textLength="780.8" clip-path="url(#breeze-workflow-run-mirror-base-images-line-7)">its pin. Turn it off when the main upgrade check is what runs   </text><text class="breez [...] </text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-8)">│</text><text class="breeze-workflow-run-mirror-base-images-r1" x="658.8" y="215.2" textLength="146.4" clip-path="url(#breeze-workflow-run-mirror-base-images-line-8)">the mirror. </text><text class="breeze-workflow-run-mirror-base-images-r5" x="805.2" y="215.2" textLength="390.4" clip-path="url(#breeze-workflow-run-mirr [...] -</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="239.6" textLength="1464" clip-path="url(#breeze-workflow-run-mirror-base-images-line-9)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-mirror-base-images-r1" x="1464" y="239.6" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-9)"> -</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="264" textLength="24.4" clip-path="url(#breeze-workflow-run-mirror-base-images-line-10)">╭─</text><text class="breeze-workflow-run-mirror-base-images-r5" x="24.4" y="264" textLength="195.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-10)"> Common options </text><text class="breeze-workflow-run-mirror-base-images-r5" x="219.6" y="264" textLength="1220" clip-path="url(#breeze-workflow-run- [...] -</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-11)">│</text><text class="breeze-workflow-run-mirror-base-images-r4" x="24.4" y="288.4" textLength="73.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-11)">--help</text><text class="breeze-workflow-run-mirror-base-images-r6" x="122" y="288.4" textLength="24.4" clip-path="url(#breeze-workflow-run-mirror-base-images-line [...] -</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="312.8" textLength="1464" clip-path="url(#breeze-workflow-run-mirror-base-images-line-12)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-mirror-base-images-r1" x="1464" y="312.8" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-12)"> +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-9)">│</text><text class="breeze-workflow-run-mirror-base-images-r4" x="24.4" y="239.6" textLength="268.4" clip-path="url(#breeze-workflow-run-mirror-base-images-line-9)">--build-os-deps-images</text><text class="breeze-workflow-run-mirror-base-images-r1" x="292.8" y="239.6" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror [...] +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-10)">│</text><text class="breeze-workflow-run-mirror-base-images-r1" x="658.8" y="264" textLength="780.8" clip-path="url(#breeze-workflow-run-mirror-base-images-line-10)">images. Turn it off when the upgrade check is what runs the     </text><text class="br [...] +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-11)">│</text><text class="breeze-workflow-run-mirror-base-images-r1" x="658.8" y="288.4" textLength="658.8" clip-path="url(#breeze-workflow-run-mirror-base-images-line-11)">mirror: it pins the images the daily build published. </text><text class="breeze-workflow-run-mirror-base-images-r5" [...] +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-12)">│</text><text class="breeze-workflow-run-mirror-base-images-r5" x="658.8" y="312.8" textLength="256.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-12)">build-os-deps-images]</text><text class="breeze-workflow-run-mirror-base-images-r5" x="1451.8" y="312.8" textLength="12.2" clip-path="url(#breeze-workflow-run-mir [...] +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="337.2" textLength="1464" clip-path="url(#breeze-workflow-run-mirror-base-images-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-mirror-base-images-r1" x="1464" y="337.2" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-13)"> +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="361.6" textLength="24.4" clip-path="url(#breeze-workflow-run-mirror-base-images-line-14)">╭─</text><text class="breeze-workflow-run-mirror-base-images-r5" x="24.4" y="361.6" textLength="195.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-14)"> Common options </text><text class="breeze-workflow-run-mirror-base-images-r5" x="219.6" y="361.6" textLength="1220" clip-path="url(#breeze-workflo [...] +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-15)">│</text><text class="breeze-workflow-run-mirror-base-images-r4" x="24.4" y="386" textLength="73.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-15)">--help</text><text class="breeze-workflow-run-mirror-base-images-r6" x="122" y="386" textLength="24.4" clip-path="url(#breeze-workflow-run-mirror-base-images-line-15)"> [...] +</text><text class="breeze-workflow-run-mirror-base-images-r5" x="0" y="410.4" textLength="1464" clip-path="url(#breeze-workflow-run-mirror-base-images-line-16)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-workflow-run-mirror-base-images-r1" x="1464" y="410.4" textLength="12.2" clip-path="url(#breeze-workflow-run-mirror-base-images-line-16)"> </text> </g> </g> diff --git a/dev/breeze/doc/images/output_workflow-run_mirror-base-images.txt b/dev/breeze/doc/images/output_workflow-run_mirror-base-images.txt index 35b73c90d6e..3b8f096f4d9 100644 --- a/dev/breeze/doc/images/output_workflow-run_mirror-base-images.txt +++ b/dev/breeze/doc/images/output_workflow-run_mirror-base-images.txt @@ -1 +1 @@ -ff6777b06b1340e4e0d47c58e828aae6 +68b613117ca8dfe035627ccd1951e78f diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index ce0d2e19a2d..2823b1473f5 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -766,6 +766,7 @@ def mirror_base_images_before_upgrade(target_branch: str, github_token: str | No hint="Or skip it with --no-mirror-base-images.", # The mirror would otherwise start a second main upgrade next to this one. trigger_upgrade_check=target_branch != "main", + build_os_deps_images=False, ) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index cd2b9cf4f6d..595733d74b3 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -5441,10 +5441,11 @@ class OsDepsImageBuild(NamedTuple): image_type: OsDepsImageType python: str build_args: dict[str, str] - # The tag of the latest image built on this digest of the Python base image - the upgrade check - # pins the image it points at. - python_tag: str - # Every input of the image is in this tag, so an image is built only once. + # The tag of the latest published image of this Python patch release - the upgrade check pins the + # image it points at, whatever base image digest that was built on. + latest_tag: str + # Every input of the image is in this tag, so an image is built only once and the tag never moves - + # it is what keeps a pinned digest tagged after the latest tag has moved on. content_tag: str @@ -5469,7 +5470,7 @@ def get_os_deps_image_builds(python: str | None, image_types: tuple[str, ...]) - image_type=image_type, python=python_version, build_args=build_args, - python_tag=os_deps_python_tag, + latest_tag=get_os_deps_image_tag(image_type.name, python_version), content_tag=f"{os_deps_python_tag}-{inputs_hash[:12]}", ) ) @@ -5583,14 +5584,13 @@ def publish_os_deps_images(python: str | None, image_types: tuple[str, ...]): if digest is None: failed.append(image) continue - python_tag_image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{build.python_tag}" - if get_image_digest(python_tag_image) == digest: + latest_image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{build.latest_tag}" + if get_image_digest(latest_image) == digest: continue - moving_tag_image = f"{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{get_os_deps_image_tag(build.image_type.name, build.python)}" - if copy_image(f"{image}@{digest}", [python_tag_image, moving_tag_image]): - moved.append(build.python_tag) + if copy_image(f"{image}@{digest}", [latest_image]): + moved.append(build.latest_tag) else: - failed.append(python_tag_image) + failed.append(latest_image) if moved: console_print( f"[warning]New OS-dependencies images for: {', '.join(moved)}. " diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py index a1769f5a6cc..8c418a618bf 100644 --- a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py @@ -507,8 +507,19 @@ def workflow_run_release_constraints(version: str, ref: str, workflow_branch: st help="Let the mirror run the main upgrade check when a tag moves past its pin. Turn it off when the " "main upgrade check is what runs the mirror.", ) -def workflow_run_mirror_base_images(trigger_upgrade_check: bool): - mirror_hardened_base_images(APACHE_AIRFLOW_REPO, trigger_upgrade_check=trigger_upgrade_check) [email protected]( + "--build-os-deps-images/--no-build-os-deps-images", + default=True, + show_default=True, + help="Also build the OS-dependencies images on the mirrored base images. Turn it off when the " + "upgrade check is what runs the mirror: it pins the images the daily build published.", +) +def workflow_run_mirror_base_images(trigger_upgrade_check: bool, build_os_deps_images: bool): + mirror_hardened_base_images( + APACHE_AIRFLOW_REPO, + trigger_upgrade_check=trigger_upgrade_check, + build_os_deps_images=build_os_deps_images, + ) @workflow_run_group.command( diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py index 5b4eb705780..0dfe9fdaee0 100644 --- a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py @@ -93,6 +93,7 @@ WORKFLOW_RUN_PARAMETERS["breeze workflow-run mirror-base-images"] = [ "name": "Mirror flags", "options": [ "--trigger-upgrade-check", + "--build-os-deps-images", ], }, ] diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index c3e33fa5682..95d9e722939 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -991,7 +991,7 @@ def get_hardened_python_image_retention_tag(tag: str, digest: str) -> str: # The OS dependencies of the hardened images come from the dhi.io apt repository, which is live: it # moves on - and drops the versions it replaced - under a base image pinned to a digest. The # "Mirror hardened base images" workflow therefore also builds images with the OS dependencies already -# installed, one per installation type of install_os_dependencies.sh, and the CI and release builds +# installed, daily, one per installation type of install_os_dependencies.sh. The CI and release builds # start from those, pinned to a digest the same way as the Python base images. HARDENED_OS_DEPS_IMAGE_REPOSITORY = "ghcr.io/apache/airflow/base/os-deps" @@ -1033,12 +1033,12 @@ HARDENED_OS_DEPS_IMAGE_DIGESTS: dict[str, str] = json.loads(HARDENED_OS_DEPS_IMA def get_os_deps_image_tag(image_type: str, python: str) -> str: - """Return the moving tag of an OS-dependencies image - also the key of its pin.""" + """Return the tag of the latest published OS-dependencies image - also the key of its pin.""" return f"{image_type}-{get_hardened_python_image_tag(python, OS_DEPS_IMAGE_DEBIAN_VERSION)}" def get_os_deps_image_python_tag(image_type: str, python: str, python_digest: str) -> str: - """Return the tag of the latest OS-dependencies image built on one digest of the Python base image.""" + """Return the prefix of the tags of the OS-dependencies images built on one Python base image digest.""" return f"{get_os_deps_image_tag(image_type, python)}-{python_digest.removeprefix('sha256:')[:12]}" diff --git a/dev/breeze/src/airflow_breeze/utils/gh_workflow_utils.py b/dev/breeze/src/airflow_breeze/utils/gh_workflow_utils.py index 5362ed8a70e..47badcf49ef 100644 --- a/dev/breeze/src/airflow_breeze/utils/gh_workflow_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/gh_workflow_utils.py @@ -315,12 +315,16 @@ def check_can_dispatch_workflows(repo: str, hint: str) -> None: sys.exit(1) -def mirror_hardened_base_images(repo: str, hint: str = "", trigger_upgrade_check: bool = True) -> None: +def mirror_hardened_base_images( + repo: str, hint: str = "", trigger_upgrade_check: bool = True, build_os_deps_images: bool = True +) -> None: """ Mirror the latest hardened base images by running the mirror workflow on main, and wait for it. :param trigger_upgrade_check: whether the mirror runs the main upgrade check when a tag moves past its pin - off when the caller is that upgrade, which would otherwise run twice. + :param build_os_deps_images: whether the run also builds the OS-dependencies images - off for the + upgrade check, which pins the ones the daily build published instead of waiting for new ones. """ check_can_dispatch_workflows(repo, hint) console_print("[blue]Mirroring the latest hardened base images[/blue]") @@ -330,5 +334,5 @@ def mirror_hardened_base_images(repo: str, hint: str = "", trigger_upgrade_check workflow_name=HARDENED_BASE_IMAGES_MIRROR_WORKFLOW, repo=repo, branch="main", - **{"trigger-upgrade-check": trigger_upgrade_check}, + **{"trigger-upgrade-check": trigger_upgrade_check, "build-os-deps-images": build_os_deps_images}, ) diff --git a/dev/breeze/tests/test_ci_upgrade.py b/dev/breeze/tests/test_ci_upgrade.py index 022a407ae9a..08bbc3fcb07 100644 --- a/dev/breeze/tests/test_ci_upgrade.py +++ b/dev/breeze/tests/test_ci_upgrade.py @@ -161,6 +161,7 @@ def test_upgrade_mirrors_the_base_images_with_its_token( "apache/airflow", hint="Or skip it with --no-mirror-base-images.", trigger_upgrade_check=trigger_upgrade_check, + build_os_deps_images=False, ) diff --git a/dev/breeze/tests/test_gh_workflow_utils.py b/dev/breeze/tests/test_gh_workflow_utils.py index b0dca2d3c7e..dde9348b962 100644 --- a/dev/breeze/tests/test_gh_workflow_utils.py +++ b/dev/breeze/tests/test_gh_workflow_utils.py @@ -206,7 +206,9 @@ def test_mirror_hardened_base_images_runs_the_mirror_on_main_after_the_check(moc calls.attach_mock(mock_check, "check") calls.attach_mock(mock_trigger, "trigger") - mirror_hardened_base_images("apache/airflow", hint="hint", trigger_upgrade_check=False) + mirror_hardened_base_images( + "apache/airflow", hint="hint", trigger_upgrade_check=False, build_os_deps_images=False + ) assert calls.mock_calls == [ mock.call.check("apache/airflow", "hint"), @@ -214,6 +216,6 @@ def test_mirror_hardened_base_images_runs_the_mirror_on_main_after_the_check(moc workflow_name="mirror-hardened-base-images.yml", repo="apache/airflow", branch="main", - **{"trigger-upgrade-check": False}, + **{"trigger-upgrade-check": False, "build-os-deps-images": False}, ), ] diff --git a/dev/breeze/tests/test_os_deps_images.py b/dev/breeze/tests/test_os_deps_images.py index 7dd2b0ae1f2..49a6a9baaf4 100644 --- a/dev/breeze/tests/test_os_deps_images.py +++ b/dev/breeze/tests/test_os_deps_images.py @@ -240,9 +240,10 @@ def test_os_deps_image_builds_are_tagged_with_their_inputs(registry): builds = get_os_deps_image_builds("3.12", ("ci",)) assert len(builds) == 1 - assert builds[0].python_tag == f"ci-{PYTHON_TAG}-{'12' * 6}" - assert builds[0].content_tag.startswith(f"{builds[0].python_tag}-") - assert len(builds[0].content_tag) == len(builds[0].python_tag) + 13 + assert builds[0].latest_tag == f"ci-{PYTHON_TAG}" + digest_prefix = f"ci-{PYTHON_TAG}-{'12' * 6}-" + assert builds[0].content_tag.startswith(digest_prefix) + assert len(builds[0].content_tag) == len(digest_prefix) + 12 def test_os_deps_image_builds_fail_without_a_mirrored_base_image(): @@ -306,7 +307,7 @@ def test_publish_os_deps_images_merges_the_platforms_and_moves_the_tags(registry (platform_images, [f"{REPOSITORY}:{build.content_tag}"]), ( [f"{REPOSITORY}:{build.content_tag}@{OS_DEPS_DIGEST}"], - [f"{REPOSITORY}:{build.python_tag}", f"{REPOSITORY}:ci-{PYTHON_TAG}"], + [f"{REPOSITORY}:ci-{PYTHON_TAG}"], ), ] assert github_output.read_text() == "moved-os-deps-images=true\n" @@ -315,7 +316,7 @@ def test_publish_os_deps_images_merges_the_platforms_and_moves_the_tags(registry def test_publish_os_deps_images_does_not_move_tags_already_there(registry, tmp_path): (build,) = get_os_deps_image_builds("3.12", ("ci",)) registry.digests[f"{REPOSITORY}:{build.content_tag}"] = OS_DEPS_DIGEST - registry.digests[f"{REPOSITORY}:{build.python_tag}"] = OS_DEPS_DIGEST + registry.digests[f"{REPOSITORY}:{build.latest_tag}"] = OS_DEPS_DIGEST github_output = tmp_path / "github_output" result = CliRunner().invoke( diff --git a/dev/breeze/tests/test_workflow_commands.py b/dev/breeze/tests/test_workflow_commands.py index 36c81673eb8..f97809810f8 100644 --- a/dev/breeze/tests/test_workflow_commands.py +++ b/dev/breeze/tests/test_workflow_commands.py @@ -395,18 +395,27 @@ class TestSyncStagingToMain: @pytest.mark.parametrize( - ("args", "trigger_upgrade_check"), + ("args", "trigger_upgrade_check", "build_os_deps_images"), [ - pytest.param([], True, id="default"), - pytest.param(["--no-trigger-upgrade-check"], False, id="called-by-the-main-upgrade"), + pytest.param([], True, True, id="default"), + pytest.param( + ["--no-trigger-upgrade-check", "--no-build-os-deps-images"], + False, + False, + id="called-by-the-main-upgrade", + ), ], ) @patch("airflow_breeze.commands.workflow_commands.mirror_hardened_base_images", autospec=True) -def test_mirror_base_images_runs_the_mirror(mock_mirror, args, trigger_upgrade_check): +def test_mirror_base_images_runs_the_mirror(mock_mirror, args, trigger_upgrade_check, build_os_deps_images): result = CliRunner().invoke(workflow_run_mirror_base_images, args, catch_exceptions=False) assert result.exit_code == 0 - mock_mirror.assert_called_once_with("apache/airflow", trigger_upgrade_check=trigger_upgrade_check) + mock_mirror.assert_called_once_with( + "apache/airflow", + trigger_upgrade_check=trigger_upgrade_check, + build_os_deps_images=build_os_deps_images, + ) @patch("airflow_breeze.commands.workflow_commands.run_command", autospec=True) diff --git a/docker-stack-docs/build.rst b/docker-stack-docs/build.rst index 98738561563..7275c73b737 100644 --- a/docker-stack-docs/build.rst +++ b/docker-stack-docs/build.rst @@ -974,8 +974,8 @@ previous images that can matter when you extend or run the image are: repository: it moves on - and drops the package versions it replaced - whenever Docker rebuilds its images, so installing the OS dependencies from it can fail for a commit that built fine before. Airflow's CI and release builds therefore start their OS dependencies from images in which - ``install_os_dependencies.sh`` already ran, built on the mirrored base images and pinned to a digest by the - same upgrade, passed as ``DEV_OS_DEPS_IMAGE`` and ``RUNTIME_OS_DEPS_IMAGE``. The script records what it + ``install_os_dependencies.sh`` already ran, built daily on the mirrored base images and pinned to a digest + by the same upgrade, passed as ``DEV_OS_DEPS_IMAGE`` and ``RUNTIME_OS_DEPS_IMAGE``. The script records what it installed with and does nothing when nothing differs, so a build with other apt dependencies still installs them. A plain ``docker build`` leaves both at ``BASE_IMAGE`` and installs everything as before. * **Packages come from Docker's apt repository first.** The image keeps stock Debian's apt sources beside diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 7fbd739ed0b..00e84f787fd 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -526,33 +526,33 @@ def fetch_os_deps_image_digests( python_image_digests: dict[str, str], current_pins: dict[str, str] ) -> dict[str, str]: """ - Fetch the digest to pin for every OS-dependencies image, given the base image digests being pinned. + Fetch the digest to pin for every OS-dependencies image of the Python patch releases being pinned. - The mirror workflow tags the latest OS-dependencies image built on each digest of a base image, - so the pins move together and the upgrade PR tests the images built on the base images it pins. - When there is none for a digest yet, the current pin of that image stays: it was built on an - older base image, but on one the full CI has passed with. + The mirror workflow builds the OS-dependencies images daily, while the base image pins move several + times a day, so the latest published image of each Python patch release is pinned - whatever base + image digest it was built on. It carries its own Python, so it need not share the base image pin's + digest. When none is published for a patch release yet - a new one, typically - the current pin + stays if it is for the same patch release, and the image is left unpinned otherwise. """ pins: dict[str, str] = {} - for python_tag, python_digest in python_image_digests.items(): + for python_tag in python_image_digests: if not python_tag.endswith(f"-debian{OS_DEPS_IMAGE_DEBIAN_RELEASE}-dev"): continue for image_type in OS_DEPS_IMAGE_TYPES: pin = f"{image_type}-{python_tag}" - tag = f"{pin}-{python_digest.removeprefix('sha256:')[:12]}" - digest = get_ghcr_image_digest(HARDENED_OS_DEPS_IMAGE_REPOSITORY, tag) + digest = get_ghcr_image_digest(HARDENED_OS_DEPS_IMAGE_REPOSITORY, pin) if digest: pins[pin] = digest elif pin in current_pins: console.print( - f"[yellow]ghcr.io/{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{tag} is not built yet - " - f"keeping the current {pin} pin." + f"[yellow]ghcr.io/{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{pin} is not published - " + "keeping the current pin." ) pins[pin] = current_pins[pin] else: console.print( - f"[yellow]ghcr.io/{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{tag} is not built yet - " - f"{pin} stays unpinned and installs its OS dependencies from the apt repositories." + f"[yellow]ghcr.io/{HARDENED_OS_DEPS_IMAGE_REPOSITORY}:{pin} is not published yet - " + "it stays unpinned and installs its OS dependencies from the apt repositories." ) return pins diff --git a/scripts/tests/ci/prek/test_upgrade_important_versions.py b/scripts/tests/ci/prek/test_upgrade_important_versions.py index 1214c4893b4..f41762feba4 100644 --- a/scripts/tests/ci/prek/test_upgrade_important_versions.py +++ b/scripts/tests/ci/prek/test_upgrade_important_versions.py @@ -519,11 +519,17 @@ CURRENT_OS_DEPS_DIGEST = "sha256:" + "cd" * 32 "ci-3.13.17-debian13-dev": OS_DEPS_DIGEST, "dev-3.13.17-debian13-dev": CURRENT_OS_DEPS_DIGEST, }, - id="not-built-keeps-the-current-pin-and-drops-old-patchlevels", + id="not-published-keeps-the-current-pin-and-drops-old-patchlevels", + ), + pytest.param( + set(), + {"ci-3.13.16-debian13-dev": OLD_DIGEST}, + {}, + id="new-patchlevel-not-published-stays-unpinned", ), ], ) -def test_os_deps_image_digests_follow_the_pinned_base_images(monkeypatch, built, current_pins, expected): +def test_os_deps_image_digests_pin_the_latest_published_images(monkeypatch, built, current_pins, expected): from ci.prek import upgrade_important_versions as uiv looked_up = [] @@ -539,9 +545,8 @@ def test_os_deps_image_digests_follow_the_pinned_base_images(monkeypatch, built, ) assert pins == expected - digest_tag = NEW_DIGEST.removeprefix("sha256:")[:12] assert looked_up == [ - ("apache/airflow/base/os-deps", f"{image_type}-3.13.17-debian13-dev-{digest_tag}") + ("apache/airflow/base/os-deps", f"{image_type}-3.13.17-debian13-dev") for image_type in ("ci", "dev", "runtime") ]
