This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch fix-docs-build-separate-images
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/fix-docs-build-separate-images
by this push:
new 080eb280964 Build each CI image in the job that uses it instead of a
shared job
080eb280964 is described below
commit 080eb2809640e8a6782382dd03958db14675be67
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Aug 3 04:25:03 2026 +0200
Build each CI image in the job that uses it instead of a shared job
Docs and the registry now want different Pythons, and a job that builds an
image
for another job has to hand it over as a multi-gigabyte tarball - exported,
stashed, restored and imported - for an image whose only consumer is a
single
downstream job. Building in place skips that round trip entirely, and the
two
builds still run in parallel because both jobs now wait only on build-info.
What is worth keeping between runs is the BuildKit mount cache rather than
the
image, so the docs job restores and stashes it under this ref and its own
Python. The registry side already knew how to build its own image when the
caller had not stashed one; it just could not be told which ref to build,
so it
would have documented main rather than the tag being published.
---
.github/workflows/publish-docs-to-s3.yml | 126 ++++++++++++++++---------------
.github/workflows/registry-build.yml | 17 +++++
2 files changed, 82 insertions(+), 61 deletions(-)
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index eab0ba59e98..6e2c1f1044f 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -237,49 +237,8 @@ jobs:
echo "publish-execution-api-schema=${PUBLISH_EXEC}" >>
${GITHUB_OUTPUT}
echo "publish-supervisor-schema=${PUBLISH_SUP}" >> ${GITHUB_OUTPUT}
- build-ci-image:
- name: "Build CI image"
- needs: [build-info]
- uses: ./.github/workflows/ci-image-build.yml
- permissions:
- contents: read
- packages: write
- with:
- runners: '["ubuntu-22.04"]'
- platform: "linux/amd64"
- # Built from the docs ref so the image matches the sources being
documented, and
- # cached against main's registry cache, which the regular Test workflow
keeps warm.
- checkout-ref: ${{ inputs.ref }}
- push-image: "false"
- upload-image-artifact: "true"
- # Leaves the BuildKit mount cache behind for the next publish of this
same ref;
- # ci-image-build.yml always restores it, so consecutive publishes stop
re-downloading
- # the whole dependency set.
- upload-mount-cache-artifact: "true"
- # Docs are published from refs cut days or weeks before main, and main's
registry cache
- # stops matching at the first Dockerfile.ci change made since the cut -
#70285 alone
- # costs a from-scratch Python build. The image the previous publish
stashed was built
- # from these very sources, so it is the cache main's cannot be.
Publishing the same ref
- # twice - an RC and then the final docs - skips the build outright.
- seed-cache-from-stashed-image: "true"
- image-stash-ref: ${{ needs.build-info.outputs.image-stash-ref }}
- # One matrix job, two images built in parallel - docs and registry each
get their own
- # Python, and ci-image-build.yml keys every cache and stash it writes by
Python version.
- python-versions: ${{ format('["{0}", "{1}"]',
needs.build-info.outputs.docs-python-version,
- needs.build-info.outputs.registry-python-version) }}
- branch: "main"
- constraints-branch: "constraints-main"
- use-uv: "true"
- upgrade-to-newer-dependencies: "false"
- docker-cache: "registry"
- disable-airflow-repo-cache: "false"
- # Building against main's cache can fail outright when the ref's
pyproject.toml has
- # diverged from main — the case the docs job's inline `breeze ci-image
build || docker
- # buildx build` fallback used to cover.
- retry-without-cache: "true"
-
build-docs:
- needs: [build-info, build-ci-image]
+ needs: [build-info]
timeout-minutes: 150
name: "Build documentation"
runs-on: ubuntu-latest
@@ -345,27 +304,69 @@ jobs:
uses: ./.github/actions/breeze
with:
python-version: "${{ needs.build-info.outputs.docs-python-version }}"
- # The image comes from the `build-ci-image` matrix, which builds it from
this same ref and
- # stashes it under the ref's own key - the shared per-branch stash holds
main's image, not
- # this ref's. The stash key carries the Python version, so this restores
the docs image and
- # the registry restores its own; a run that had nothing to build
restores what the previous
- # publish left behind.
- - name: >
- Restore CI docker image built for ref ${{ inputs.ref }}
- linux/amd64:${{ needs.build-info.outputs.docs-python-version }}
+ # The image is built here rather than in a job of its own. Docs are the
only consumer of
+ # this Python's image, so a separate job would only export a
multi-gigabyte tarball and
+ # import it straight back. What is worth carrying between runs is the
BuildKit mount cache,
+ # which is restored and saved below under this ref's own key.
+ - name: "Restore mount cache linux/amd64:${{
needs.build-info.outputs.docs-python-version }}"
uses:
apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355
with:
- key: "ci-image-save-v3-linux/amd64-\
+ key: "ci-cache-mount-save-v3-linux/amd64-\
${{ needs.build-info.outputs.docs-python-version }}\
-${{ needs.build-info.outputs.image-stash-ref }}"
- path: "/mnt/"
- only-current-branch: 'true'
- fail-on-download: 'true'
- - name: "Load CI image linux/amd64:${{
needs.build-info.outputs.docs-python-version }}"
+ path: "/tmp/"
+ id: restore-mount-cache
+ - name: "Import mount cache linux/amd64:${{
needs.build-info.outputs.docs-python-version }}"
env:
- PYTHON: ${{ needs.build-info.outputs.docs-python-version }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{
needs.build-info.outputs.docs-python-version }}
run: >
- breeze ci-image load --platform "linux/amd64" --python "${PYTHON}"
--image-file-dir "/mnt"
+ breeze ci-image import-mount-cache
+ --cache-file
/tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
+ if: steps.restore-mount-cache.outputs.stash-hit == 'true'
+ - name: "Login to ghcr.io"
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ ACTOR: ${{ github.actor }}
+ run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR}
--password-stdin
+ - name: "Build CI image linux/amd64:${{
needs.build-info.outputs.docs-python-version }}"
+ env:
+ DOCKER_CACHE: "registry"
+ DISABLE_AIRFLOW_REPO_CACHE: "false"
+ UPGRADE_TO_NEWER_DEPENDENCIES: "false"
+ CONSTRAINTS_GITHUB_REPOSITORY: >-
+ ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
+ secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_USERNAME: ${{ github.actor }}
+ PUSH: "false"
+ run: |
+ if breeze ci-image build --platform "linux/amd64"; then
+ exit 0
+ fi
+ # Reached when this ref's pyproject.toml has drifted far enough from
the branch the
+ # registry cache was built for that the cached layers cannot satisfy
it. Slow, but
+ # correct - and the case that matters here, since docs are published
from refs cut
+ # days or weeks before main.
+ echo "Build with registry cache failed - retrying with the cache
disabled."
+ DOCKER_CACHE="disabled" breeze ci-image build --platform
"linux/amd64"
+ - name: "Export mount cache linux/amd64:${{
needs.build-info.outputs.docs-python-version }}"
+ env:
+ PYTHON_MAJOR_MINOR_VERSION: ${{
needs.build-info.outputs.docs-python-version }}
+ run: >
+ breeze ci-image export-mount-cache
+ --cache-file
/tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
+ - name: "Stash mount cache linux/amd64:${{
needs.build-info.outputs.docs-python-version }}"
+ uses:
apache/infrastructure-actions/stash/save@0ff9972b5872e19c9f4555c9159c2fea4f794355
+ with:
+ key: "ci-cache-mount-save-v3-linux/amd64-\
+ ${{ needs.build-info.outputs.docs-python-version }}\
+ -${{ needs.build-info.outputs.image-stash-ref }}"
+ path: "/tmp/ci-cache-mount-save-v3-\
+ ${{ needs.build-info.outputs.docs-python-version }}.tar.gz"
+ if-no-files-found: 'error'
+ # Read by the next publish of this same ref - an RC and then the
final docs, days apart.
+ retention-days: '6'
- name: "Restore docs inventory cache"
uses:
apache/infrastructure-actions/stash/restore@0ff9972b5872e19c9f4555c9159c2fea4f794355
with:
@@ -658,7 +659,7 @@ jobs:
update-registry:
# Runs alongside the docs build rather than after it: the registry reads
nothing the docs
# publish produces, so waiting only added its whole duration to the
release manager's wait.
- needs: [build-info, build-ci-image]
+ needs: [build-info]
if: needs.build-info.outputs.registry-providers != '' ||
needs.build-info.outputs.registry-full-build == 'true'
name: "Update Provider Registry"
permissions:
@@ -668,10 +669,13 @@ jobs:
with:
destination: ${{ needs.build-info.outputs.destination }}
provider: ${{ needs.build-info.outputs.registry-providers }}
+ # The registry extracts on 3.12 and the docs build on the default
Python, so each builds
+ # the image it needs instead of sharing one. registry-build.yml builds
its own whenever the
+ # caller has not stashed one, keyed by this ref so consecutive publishes
reuse it.
python-version: ${{ needs.build-info.outputs.registry-python-version }}
- # `build-ci-image` already built and stashed both images this run -
under the ref's key, one
- # per Python. This picks the 3.12 one, which is what the registry has
always extracted on.
- ci-image-already-built: true
+ ci-image-already-built: false
+ # Without this the registry would document main rather than the ref
being published.
+ checkout-ref: ${{ inputs.ref }}
image-stash-ref: ${{ needs.build-info.outputs.image-stash-ref }}
secrets:
DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
diff --git a/.github/workflows/registry-build.yml
b/.github/workflows/registry-build.yml
index c5bae276172..fa8866c22a4 100644
--- a/.github/workflows/registry-build.yml
+++ b/.github/workflows/registry-build.yml
@@ -60,6 +60,13 @@ on: # yamllint disable-line rule:truthy
required: false
type: boolean
default: false
+ checkout-ref:
+ description: >
+ Ref the CI image is built from. Defaults to the branch tip; a
release publish passes the
+ tag it is publishing so the registry extracts from those sources
rather than from main.
+ required: false
+ type: string
+ default: ""
# `github.event_name` inside a called workflow reports the *caller's*
event, never
# `workflow_call`, so it cannot distinguish the two entry points. An
input declared
# only here can: it defaults to true when called and is undefined
(falsy) on a
@@ -117,10 +124,17 @@ jobs:
with:
runners: '["ubuntu-22.04"]'
platform: "linux/amd64"
+ # Empty means the branch tip; a release publish passes the tag it is
publishing.
+ checkout-ref: ${{ inputs.checkout-ref }}
push-image: "false"
upload-image-artifact: "true"
# Kept so a standalone registry dispatch reuses the mount cache its last
run left behind.
upload-mount-cache-artifact: "true"
+ # A release ref is cut days or weeks before main, so main's registry
cache stops matching at
+ # the first Dockerfile.ci change since the cut. What the previous
publish of this same ref
+ # stashed was built from these very sources, so it is the cache main's
cannot be.
+ seed-cache-from-stashed-image: ${{ inputs.image-stash-ref != '' }}
+ image-stash-ref: ${{ inputs.image-stash-ref }}
python-versions: ${{ format('["{0}"]', inputs.python-version) }}
branch: "main"
constraints-branch: "constraints-main"
@@ -128,6 +142,9 @@ jobs:
upgrade-to-newer-dependencies: "false"
docker-cache: "registry"
disable-airflow-repo-cache: "false"
+ # Building a release ref against main's cache can fail outright once its
pyproject.toml has
+ # diverged; falling back to an uncached build is slow but correct.
+ retry-without-cache: "true"
build-and-publish-registry:
timeout-minutes: 45
