This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch share-ci-image-docs-registry in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a944a17e84be1323dc05050ad43b3a1be5a7d43a Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Jul 28 14:08:02 2026 +0200 Build the docs CI image once and let the registry share it A docs publish paid for two CI image builds of the same sources, one for the docs and one for the provider registry, and only started the second once the docs were already live. Building it once up front and handing the same image to both lets them run side by side, which is most of the wait a release manager sits through after the docs have landed. Keeping the mount cache means consecutive publishes stop re-downloading the whole dependency set, and pinning both to one Python removes the second image that only differed by interpreter. --- .github/workflows/publish-docs-to-s3.yml | 73 +++++++++++++++++++++----------- .github/workflows/registry-build.yml | 54 ++++++++++++++++++++--- 2 files changed, 97 insertions(+), 30 deletions(-) diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 9b27edc1669..d15a0969080 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -98,7 +98,9 @@ jobs: publish-supervisor-schema: ${{ steps.parameters.outputs.publish-supervisor-schema }} # yamllint disable rule:line-length skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder && '--skip-write-to-stable-folder' || '' }} - default-python-version: "3.10" + # The docs build and the registry extraction share one CI image, so they share its + # Python too. 3.12 is what the registry has always extracted on. + default-python-version: "3.12" registry-providers: ${{ steps.derive_registry_inputs.outputs.registry-providers }} registry-full-build: ${{ steps.derive_registry_inputs.outputs.registry-full-build }} if: contains(fromJSON('[ @@ -226,8 +228,35 @@ jobs: echo "publish-execution-api-schema=${PUBLISH_EXEC}" >> ${GITHUB_OUTPUT} echo "publish-supervisor-schema=${PUBLISH_SUP}" >> ${GITHUB_OUTPUT} - build-docs: + build-ci-image: + name: "Build CI image" needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + packages: write + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + # Built from the docs ref so the image matches the sources being documented, and + # cached against main's registry cache, which the regular Test workflow keeps warm. + checkout-ref: ${{ inputs.ref }} + push-image: "false" + upload-image-artifact: "true" + # Leaves the BuildKit mount cache behind for the next docs/registry run on this branch; + # ci-image-build.yml always restores it, so consecutive publishes stop re-downloading + # the whole dependency set. + upload-mount-cache-artifact: "true" + python-versions: ${{ format('["{0}"]', needs.build-info.outputs.default-python-version) }} + branch: "main" + constraints-branch: "constraints-main" + use-uv: "true" + upgrade-to-newer-dependencies: "false" + docker-cache: "registry" + disable-airflow-repo-cache: "false" + + build-docs: + needs: [build-info, build-ci-image] timeout-minutes: 150 name: "Build documentation" runs-on: ubuntu-latest @@ -293,29 +322,20 @@ jobs: uses: ./.github/actions/breeze with: python-version: "${{ needs.build-info.outputs.default-python-version }}" - - name: "Login to ghcr.io" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${ACTOR}" --password-stdin - - name: "Building image from the ${{ inputs.ref }} reference" + # The image comes from the `build-ci-image` job, which builds it from this same ref and + # stashes it. The registry build restores the very same stash, so a run builds it once. + - name: "Restore CI docker image linux/amd64:${{ needs.build-info.outputs.default-python-version }}" + uses: apache/infrastructure-actions/stash/restore@49df447b39b18354895520e0a63731b7cad7cbec + with: + key: ci-image-save-v3-linux/amd64-${{ needs.build-info.outputs.default-python-version }} + path: "/mnt/" + only-current-branch: 'true' + fail-on-download: 'true' + - name: "Load CI image linux/amd64:${{ needs.build-info.outputs.default-python-version }}" env: - INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }} - INCLUDE_COMMITS: ${{ startsWith(inputs.ref, 'providers') && 'true' || 'false' }} - # The regular `breeze ci-image build` path uses the registry cache pushed by the last - # successful "main" Test workflow (DOCKER_CACHE=registry, --cache-from=<main>:cache-<platform>), - # which is the fast path for builds from main or main-like refs. When building from an old - # ref whose pyproject.toml has diverged from main, reusing the main cache can fail the build, - # so we fall back to a plain buildx build with no cache-from. + PYTHON: ${{ needs.build-info.outputs.default-python-version }} run: > - breeze ci-image build || - docker buildx build --load --builder default --progress=auto --pull - --build-arg AIRFLOW_EXTRAS=devel-ci --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES=false - --build-arg AIRFLOW_USE_UV=true - --build-arg BUILD_PROGRESS=auto --build-arg INSTALL_MYSQL_CLIENT_TYPE=mariadb - --build-arg VERSION_SUFFIX_FOR_PYPI=dev0 - -t "ghcr.io/apache/airflow/main/ci/python${PYTHON_MAJOR_MINOR_VERSION}:latest" --target main . - -f Dockerfile.ci --platform linux/amd64 + breeze ci-image load --platform "linux/amd64" --python "${PYTHON}" --image-file-dir "/mnt" - name: "Restore docs inventory cache" uses: apache/infrastructure-actions/stash/restore@49df447b39b18354895520e0a63731b7cad7cbec with: @@ -606,7 +626,9 @@ jobs: --destination-location "${SCHEMAS_DESTINATION}" "${args[@]}" update-registry: - needs: [publish-docs-to-s3, build-info] + # Runs alongside the docs build rather than after it: the registry reads nothing the docs + # publish produces, so waiting only added its whole duration to the release manager's wait. + needs: [build-info, build-ci-image] if: needs.build-info.outputs.registry-providers != '' || needs.build-info.outputs.registry-full-build == 'true' name: "Update Provider Registry" permissions: @@ -616,6 +638,9 @@ jobs: with: destination: ${{ needs.build-info.outputs.destination }} provider: ${{ needs.build-info.outputs.registry-providers }} + python-version: ${{ needs.build-info.outputs.default-python-version }} + # `build-ci-image` already built and stashed the image this run. + ci-image-already-built: true secrets: DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/registry-build.yml b/.github/workflows/registry-build.yml index b64dcbfcb33..3e134129331 100644 --- a/.github/workflows/registry-build.yml +++ b/.github/workflows/registry-build.yml @@ -33,6 +33,11 @@ on: # yamllint disable-line rule:truthy required: false type: string default: "" + python-version: + description: "Python version of the CI image the extraction runs in" + required: false + type: string + default: "3.12" workflow_call: inputs: destination: @@ -45,6 +50,16 @@ on: # yamllint disable-line rule:truthy required: false type: string default: "" + python-version: + description: "Python version of the CI image the extraction runs in" + required: false + type: string + default: "3.12" + ci-image-already-built: + description: "Caller has already built and stashed the CI image for this run" + required: false + type: boolean + default: false secrets: DOCS_AWS_ACCESS_KEY_ID: required: true @@ -62,7 +77,10 @@ jobs: permissions: contents: read packages: write + # Skipped when the caller stashed an image for this run; a standalone dispatch has no + # such image and still builds its own. if: > + (inputs.ci-image-already-built != true) && ( github.event_name == 'workflow_call' || contains(fromJSON('[ "ashb", @@ -78,14 +96,15 @@ jobs: "uranusjr", "utkarsharma2", "vincbeck" - ]'), github.event.sender.login) + ]'), github.event.sender.login)) with: runners: '["ubuntu-22.04"]' platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" - upload-mount-cache-artifact: "false" - python-versions: '["3.12"]' + # Kept so a standalone registry dispatch reuses the mount cache its last run left behind. + upload-mount-cache-artifact: "true" + python-versions: ${{ format('["{0}"]', inputs.python-version) }} branch: "main" constraints-branch: "constraints-main" use-uv: "true" @@ -97,6 +116,27 @@ jobs: timeout-minutes: 45 name: "Build & Publish Registry" needs: [build-ci-image] + # `build-ci-image` is skipped when the caller stashed the image, so this cannot simply + # inherit its result — but it must keep enforcing the same committer allowlist that + # skipping that job used to enforce for us on a dispatch. + if: > + !cancelled() && needs.build-ci-image.result != 'failure' && ( + github.event_name == 'workflow_call' || + contains(fromJSON('[ + "ashb", + "bugraoz93", + "eladkal", + "ephraimbuddy", + "jedcunningham", + "jscheffl", + "kaxil", + "pierrejeambrun", + "shahar1", + "potiuk", + "uranusjr", + "utkarsharma2", + "vincbeck" + ]'), github.event.sender.login)) runs-on: ubuntu-latest env: SCARF_ANALYTICS: "false" @@ -125,7 +165,7 @@ jobs: - name: "Prepare breeze & CI image" uses: ./.github/actions/prepare_breeze_and_image with: - python: "3.12" + python: "${{ inputs.python-version }}" platform: "linux/amd64" use-uv: "true" make-mnt-writeable-and-cleanup: "true" @@ -183,6 +223,7 @@ jobs: env: PROVIDER: ${{ inputs.provider }} DESTINATION: ${{ inputs.destination }} + PYTHON_VERSION: ${{ inputs.python-version }} run: | # Staging dispatches preview unreleased providers (maintainers want to # verify newly-bumped versions look right before tagging). Live builds @@ -193,9 +234,10 @@ jobs: ALLOW_UNRELEASED="--allow-unreleased" fi if [[ -n "${PROVIDER}" ]]; then - breeze registry extract-data --python 3.12 --provider "${PROVIDER}" ${ALLOW_UNRELEASED} + breeze registry extract-data --python "${PYTHON_VERSION}" \ + --provider "${PROVIDER}" ${ALLOW_UNRELEASED} else - breeze registry extract-data --python 3.12 ${ALLOW_UNRELEASED} + breeze registry extract-data --python "${PYTHON_VERSION}" ${ALLOW_UNRELEASED} fi # --- Incremental: merge new data with existing ---
