This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch separate-release-image-workflows-per-python
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f0577a5a172f9e2999cc515ae56228dfd8cc46e7
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed May 7 13:24:17 2025 +0200
Split release image into per-python independent matrix of workflows
The release workflow now will run separately for each image - which
means that if both AMD / ARM images of the same python version have
finished, the merge step for that Python version will run immediately
rather than waiting for all Python versions to complete. This means
that some images might be available a bit faster and that even if
a single image releaase will fail for some reason, the other images
will appear before we re-run that failed image job.
It also adds the possibility of overriding the python version
images - we can now additionally filter which image versions should
be run.
---
.github/workflows/release_dockerhub_image.yml | 218 +++------------------
...mage.yml => release_single_dockerhub_image.yml} | 123 +++---------
2 files changed, 64 insertions(+), 277 deletions(-)
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index 9d8655ecaea..9f8d5a4cd57 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -27,6 +27,10 @@ on: # yamllint disable-line rule:truthy
type: boolean
description: 'Limit to amd64 only (faster testing)'
default: false
+ limitPythonVersions:
+ type: string
+ description: 'Limit python versions - space separated in quotes(e.g.
"3.9 3.10")'
+ default: ''
permissions:
contents: read
packages: read
@@ -55,6 +59,7 @@ jobs:
VERBOSE: true
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
AMD_ONLY: ${{ github.event.inputs.amdOnly }}
+ LIMIT_PYTHON_VERSIONS: ${{ inputs.limitPythonVersions }}
if: contains(fromJSON('[
"ashb",
"eladkal",
@@ -106,195 +111,36 @@ jobs:
else
echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >>
"${GITHUB_OUTPUT}"
fi
-
- build-images:
- timeout-minutes: 50
- # yamllint disable rule:line-length
- name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}, ${{ matrix.platform }}"
- runs-on: ${{ (matrix.platform == 'linux/amd64') &&
fromJSON(needs.build-info.outputs.amd-runners) ||
fromJSON(needs.build-info.outputs.arm-runners) }}
- needs: [build-info]
- strategy:
- fail-fast: false
- matrix:
- python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions)
}}
- platform: ${{ fromJSON(needs.build-info.outputs.platformMatrix) }}
- env:
- AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- PLATFORM: ${{ matrix.platform }}
- SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' &&
'--skip-latest' || '' }}
- COMMIT_SHA: ${{ github.sha }}
- REPOSITORY: ${{ github.repository }}
- steps:
- - name: "Cleanup repo"
+ - name: "Override python versions if specified"
shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- use-uv: "false"
- - name: Free space
- run: breeze ci free-space --answer yes
- - name: "Cleanup dist and context file"
- run: rm -fv ./dist/* ./docker-context-files/*
- - name: "Login to hub.docker.com"
- run: >
- echo ${{ secrets.DOCKERHUB_TOKEN }} |
- docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER
}}
- - name: "Get env vars for metadata"
- shell: bash
- run: |
- echo
"ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >>
"${GITHUB_ENV}"
- echo
"MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json"
>> "${GITHUB_ENV}"
- echo
"MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json"
>> "${GITHUB_ENV}"
- - name: Login to ghcr.io
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- ACTOR: ${{ github.actor }}
- run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR}
--password-stdin
- - name: "Install buildx plugin"
- # yamllint disable rule:line-length
+ id: determine-python-versions
run: |
- sudo apt-get update
- sudo apt-get install ca-certificates curl
- sudo install -m 0755 -d /etc/apt/keyrings
- sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o
/etc/apt/keyrings/docker.asc
- sudo chmod a+r /etc/apt/keyrings/docker.asc
-
- # Add the repository to Apt sources:
- echo \
- "deb [arch=$(dpkg --print-architecture)
signed-by=/etc/apt/keyrings/docker.asc]
https://download.docker.com/linux/ubuntu \
- $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
- sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
- sudo apt-get update
- sudo apt install docker-buildx-plugin
- - name: "Create airflow_cache builder"
- run: docker buildx create --name airflow_cache --driver
docker-container
- - name: >
- Build regular images: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}, ${{ matrix.platform }}
- run: >
- breeze release-management release-prod-images --dockerhub-repo
"${REPOSITORY}"
- --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
- --python ${PYTHON_MAJOR_MINOR_VERSION}
- --metadata-folder dist
- - name: >
- Verify regular image: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}, ${{ matrix.platform }}
- run: >
- breeze prod-image verify --pull --manifest-file
dist/${MANIFEST_FILE_NAME}
- - name: >
- Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}, ${{ matrix.platform }}
- run: >
- breeze release-management release-prod-images --dockerhub-repo
"${REPOSITORY}"
- --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
- --python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images
- --metadata-folder dist
- - name: >
- Verify slim image: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}, ${{ matrix.platform }}
- run: >
- breeze prod-image verify --pull --slim-image --manifest-file
dist/${MANIFEST_SLIM_FILE_NAME}
- - name: "List upload-able artifacts"
- shell: bash
- run: find ./dist -name '*.json'
- - name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}"
- uses: actions/upload-artifact@v4
- with:
- name: ${{ env.ARTIFACT_NAME }}
- path: ./dist/metadata-*
- retention-days: 7
- if-no-files-found: error
- - name: "Docker logout"
- run: docker logout
- if: always()
+ # override python versions if specified
+ if [[ "${LIMIT_PYTHON_VERSIONS}" != "" ]]; then
+ # join python versions with commas
+ IFS=' ' read -r -a pythonVersions <<< "${LIMIT_PYTHON_VERSIONS}"
+ pythonVersions=("${pythonVersions[@]// /\",\"}")
+ echo 'python-versions=["'"${pythonVersions}"'"]' >>
"${GITHUB_OUTPUT}"
+ fi
- merge-images:
- timeout-minutes: 5
- name: "Merge: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}"
- runs-on: ["ubuntu-22.04"]
- needs: [build-info, build-images]
+ release-images:
+ name: "Release images"
+ needs: [build-info]
strategy:
fail-fast: false
matrix:
- python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions)
}}
- env:
- AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' &&
'--skip-latest' || '' }}
- COMMIT_SHA: ${{ github.sha }}
- REPOSITORY: ${{ github.repository }}
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- use-uv: "false"
- - name: Free space
- run: breeze ci free-space --answer yes
- - name: "Cleanup dist and context file"
- run: rm -fv ./dist/* ./docker-context-files/*
- - name: "Login to hub.docker.com"
- run: >
- echo ${{ secrets.DOCKERHUB_TOKEN }} |
- docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER
}}
- - name: Login to ghcr.io
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- ACTOR: ${{ github.actor }}
- run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR}
--password-stdin
- - name: "Download metadata artifacts"
- uses: actions/download-artifact@v4
- with:
- path: ./dist
- pattern: metadata-${{ matrix.python-version }}-*
- - name: "List downloaded artifacts"
- shell: bash
- run: find ./dist -name '*.json'
- - name: "Install buildx plugin"
- # yamllint disable rule:line-length
- run: |
- sudo apt-get update
- sudo apt-get install ca-certificates curl
- sudo install -m 0755 -d /etc/apt/keyrings
- sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o
/etc/apt/keyrings/docker.asc
- sudo chmod a+r /etc/apt/keyrings/docker.asc
-
- # Add the repository to Apt sources:
- echo \
- "deb [arch=$(dpkg --print-architecture)
signed-by=/etc/apt/keyrings/docker.asc]
https://download.docker.com/linux/ubuntu \
- $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
- sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
- sudo apt-get update
- sudo apt install docker-buildx-plugin
- - name: "Install regctl"
- # yamllint disable rule:line-length
- run: |
- mkdir -p ~/bin
- curl -L
https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64
>${HOME}/bin/regctl
- chmod 755 ${HOME}/bin/regctl
- echo "${HOME}/bin" >>${GITHUB_PATH}
- - name: "Merge regular images ${{ github.event.inputs.airflowVersion }},
${{ matrix.python-version }}"
- run: >
- breeze release-management merge-prod-images --dockerhub-repo
"${REPOSITORY}"
- --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
- --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist
- - name: "Merge slim images ${{ github.event.inputs.airflowVersion }},
${{ matrix.python-version }}"
- run: >
- breeze release-management merge-prod-images --dockerhub-repo
"${REPOSITORY}"
- --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST}
- --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist
--slim-images
- - name: "Docker logout"
- run: docker logout
- if: always()
+ python: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }}
+ uses: ./.github/workflows/release_single_dockerhub_image.yml
+ secrets:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ permissions:
+ contents: read
+ with:
+ airflowVersion: ${{ needs.build-info.outputs.airflowVersion }}
+ pythonVersion: ${{ matrix.python }}
+ platformMatrix: ${{ needs.build-info.outputs.platformMatrix }}
+ skipLatest: ${{ needs.build-info.outputs.skipLatest }}
+ armRunners: ${{ needs.build-info.outputs.arm-runners }}
+ amdRunners: ${{ needs.build-info.outputs.amd-runners }}
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_single_dockerhub_image.yml
similarity index 71%
copy from .github/workflows/release_dockerhub_image.yml
copy to .github/workflows/release_single_dockerhub_image.yml
index 9d8655ecaea..fa6f3b76ac6 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_single_dockerhub_image.yml
@@ -16,17 +16,34 @@
# under the License.
#
---
-name: "Release PROD images"
+name: "Release single PROD image"
on: # yamllint disable-line rule:truthy
- workflow_dispatch:
+ workflow_call:
inputs:
airflowVersion:
description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)'
+ type: string
+ required: true
+ platformMatrix:
+ description: 'Platform matrix formatted as json (e.g. ["linux/amd64",
"linux/arm64"])'
+ type: string
+ required: true
+ pythonVersion:
+ description: 'Python version (e.g. 3.8, 3.9, 3.10, 3.11)'
+ type: string
+ required: true
+ skipLatest:
+ description: "Skip tagging latest release (true/false)"
+ type: string
+ required: true
+ amdRunners:
+ description: "Amd64 runners (e.g. [\"ubuntu-22.04\",
\"ubuntu-24.04\"])"
+ type: string
+ required: true
+ armRunners:
+ description: "Arm64 runners (e.g. [\"ubuntu-22.04\",
\"ubuntu-24.04\"])"
+ type: string
required: true
- amdOnly:
- type: boolean
- description: 'Limit to amd64 only (faster testing)'
- default: false
permissions:
contents: read
packages: read
@@ -37,92 +54,20 @@ env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERBOSE: true
jobs:
- build-info:
- timeout-minutes: 10
- name: "Build Info"
- runs-on: ["ubuntu-24.04"]
- outputs:
- pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
- allPythonVersions: ${{
steps.selective-checks.outputs.all-python-versions }}
- defaultPythonVersion: ${{
steps.selective-checks.outputs.default-python-version }}
- platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }}
- airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }}
- skipLatest: ${{ steps.selective-checks.outputs.skipLatest }}
- amd-runners: ${{ steps.selective-checks.outputs.amd-runners }}
- arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
- env:
- GITHUB_CONTEXT: ${{ toJson(github) }}
- VERBOSE: true
- AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
- AMD_ONLY: ${{ github.event.inputs.amdOnly }}
- if: contains(fromJSON('[
- "ashb",
- "eladkal",
- "ephraimbuddy",
- "jedcunningham",
- "kaxil",
- "pierrejeambrun",
- "potiuk",
- "utkarsharma2"
- ]'), github.event.sender.login)
- steps:
- - name: "Input parameters summary"
- shell: bash
- run: |
- echo "Input parameters summary"
- echo "========================="
- echo "Airflow version: '${AIRFLOW_VERSION}'"
- echo "AMD only: '${AMD_ONLY}'"
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
- - name: "Install uv"
- run: curl -LsSf https://astral.sh/uv/install.sh | sh
- - name: "Check airflow version"
- id: check-airflow-version
- shell: bash
- run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}"
>> "${GITHUB_OUTPUT}"
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- use-uv: "true"
- - name: Selective checks
- id: selective-checks
- env:
- VERBOSE: "false"
- run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- - name: "Determine build matrix"
- shell: bash
- id: determine-matrix
- run: |
- if [[ "${AMD_ONLY}" = "true" ]]; then
- echo 'platformMatrix=["linux/amd64"]' >> "${GITHUB_OUTPUT}"
- else
- echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >>
"${GITHUB_OUTPUT}"
- fi
-
build-images:
timeout-minutes: 50
# yamllint disable rule:line-length
name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}, ${{ matrix.platform }}"
- runs-on: ${{ (matrix.platform == 'linux/amd64') &&
fromJSON(needs.build-info.outputs.amd-runners) ||
fromJSON(needs.build-info.outputs.arm-runners) }}
- needs: [build-info]
+ runs-on: ${{ (matrix.platform == 'linux/amd64') &&
fromJSON(inputs.amdRunners) || fromJSON(inputs.armRunners) }}
strategy:
fail-fast: false
matrix:
- python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions)
}}
- platform: ${{ fromJSON(needs.build-info.outputs.platformMatrix) }}
+ platform: ${{ fromJSON(inputs.platformMatrix) }}
env:
- AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ AIRFLOW_VERSION: ${{ inputs.airflowVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }}
PLATFORM: ${{ matrix.platform }}
- SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' &&
'--skip-latest' || '' }}
+ SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }}
COMMIT_SHA: ${{ github.sha }}
REPOSITORY: ${{ github.repository }}
steps:
@@ -216,15 +161,11 @@ jobs:
timeout-minutes: 5
name: "Merge: ${{ github.event.inputs.airflowVersion }}, ${{
matrix.python-version }}"
runs-on: ["ubuntu-22.04"]
- needs: [build-info, build-images]
- strategy:
- fail-fast: false
- matrix:
- python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions)
}}
+ needs: [build-images]
env:
- AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' &&
'--skip-latest' || '' }}
+ AIRFLOW_VERSION: ${{ inputs.airflowVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.python-version }}
+ SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }}
COMMIT_SHA: ${{ github.sha }}
REPOSITORY: ${{ github.repository }}
steps:
