This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 3880c3ca5bb [v3-0-test] Remove special case for building PROD image in
non-main branch (#50115) (#50118)
3880c3ca5bb is described below
commit 3880c3ca5bba9ae75920313ca8ade47a966308af
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri May 2 13:32:20 2025 +0200
[v3-0-test] Remove special case for building PROD image in non-main branch
(#50115) (#50118)
When we removed "chicken-egg-providers" from the build in #49799 we
should be able now to build PROD image in v3*test branch without special
case of only using PyPI providers. This should work because in
v3-*test branch we automatically detect (by presence of relesed
tag) whether we should build provider from sources or use it from
PyPI. In normal circumstances, we would need use providers fro
PyPI, but sometimes (For example for 3.0.1) we need to use some
providers from sources before they get released (for example
common.messaging 1.0.1 that is the lowest possible common.messaging
for Airflow 3.0.1.
This PR removes that special case and switches to the same method
of building PROD image in main and in release branch.
(cherry picked from commit 9deb2a7)
---
.github/workflows/additional-ci-image-checks.yml | 2 +-
.github/workflows/additional-prod-image-tests.yml | 2 --
.github/workflows/ci-image-build.yml | 2 +-
.github/workflows/ci-image-checks.yml | 2 +-
.github/workflows/ci.yml | 1 -
.github/workflows/prod-image-build.yml | 37 +++++++----------------
.github/workflows/prod-image-extra-checks.yml | 10 +-----
.github/workflows/push-image-cache.yml | 2 +-
8 files changed, 16 insertions(+), 42 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index c4faa7dac05..86740494b64 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -37,7 +37,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch:
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index 6fe821ad390..56badcd10ad 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -68,7 +68,6 @@ jobs:
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
use-uv: "false"
- build-provider-distributions: ${{ inputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
@@ -84,7 +83,6 @@ jobs:
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
use-uv: "false"
- build-provider-distributions: ${{ inputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index b4392930567..1df2e26c090 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -77,7 +77,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch:
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index b99cc403367..76c850132ea 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -41,7 +41,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
canary-run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 160820264e3..879e16df373 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -688,7 +688,6 @@ jobs:
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
branch: ${{ needs.build-info.outputs.default-branch }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
- build-provider-distributions: ${{
needs.build-info.outputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 1c8f7504c25..844b0d46e7d 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -85,17 +85,13 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch:
description: "Branch used to construct constraints URL from."
required: true
type: string
- build-provider-distributions:
- description: "Whether to build provider distributions (true/false). If
false providers are from PyPI"
- required: true
- type: string
upgrade-to-newer-dependencies:
description: "Whether to attempt to upgrade image to newer
dependencies (true/false)"
required: true
@@ -143,14 +139,22 @@ jobs:
with:
use-uv: ${{ inputs.use-uv }}
if: inputs.upload-package-artifact == 'true'
- - name: "Prepare providers packages"
+ - name: "Prepare providers packages - all providers built from sources"
shell: bash
run: >
breeze release-management prepare-provider-distributions
--distributions-list-file ./prod_image_installed_providers.txt
--distribution-format wheel --include-not-ready-providers
--skip-tag-check
if: >
- inputs.upload-package-artifact == 'true' &&
inputs.build-provider-distributions == 'true'
+ inputs.upload-package-artifact == 'true' && inputs.branch == 'main'
+ - name: "Prepare providers packages with only new versions of providers"
+ shell: bash
+ run: >
+ breeze release-management prepare-provider-distributions
+ --distributions-list-file ./prod_image_installed_providers.txt
+ --distribution-format wheel --include-not-ready-providers
+ if: >
+ inputs.upload-package-artifact == 'true' && inputs.branch != 'main'
- name: "Prepare airflow package"
shell: bash
run: >
@@ -262,25 +266,6 @@ jobs:
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
INCLUDE_NOT_READY_PROVIDERS: "true"
- if: inputs.build-provider-distributions == 'true'
- - name: "Build PROD images with PyPi providers ${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
- shell: bash
- run: >
- breeze prod-image build
- --builder airflow_cache
- --commit-sha "${{ github.sha }}"
- --install-distributions-from-context
- --airflow-constraints-mode constraints
- --use-constraints-for-context-distributions
- env:
- PUSH: ${{ inputs.push-image }}
- DOCKER_CACHE: ${{ inputs.docker-cache }}
- DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
- DEBIAN_VERSION: ${{ inputs.debian-version }}
- INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
- INCLUDE_NOT_READY_PROVIDERS: "true"
- if: inputs.build-provider-distributions != 'true'
- name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
run: breeze prod-image verify
- name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
diff --git a/.github/workflows/prod-image-extra-checks.yml
b/.github/workflows/prod-image-extra-checks.yml
index 287348d7754..356900c9222 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -33,17 +33,13 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
use-uv:
description: "Whether to use uv to build the image (true/false)"
required: true
type: string
- build-provider-distributions:
- description: "Whether to build provider distributions (true/false). If
false providers are from PyPI"
- required: true
- type: string
upgrade-to-newer-dependencies:
description: "Whether to attempt to upgrade image to newer
dependencies (false/RANDOM_VALUE)"
required: true
@@ -78,7 +74,6 @@ jobs:
# Always build images during the extra checks and never push them
push-image: "false"
use-uv: ${{ inputs.use-uv }}
- build-provider-distributions: ${{ inputs.build-provider-distributions }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
@@ -87,8 +82,6 @@ jobs:
pip-image:
uses: ./.github/workflows/prod-image-build.yml
- # Skip testing PIP image on release branches as all images there are built
with pip
- if: ${{ inputs.use-uv == 'true' }}
with:
runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
build-type: "pip"
@@ -102,7 +95,6 @@ jobs:
# Always build images during the extra checks and never push them
push-image: "false"
use-uv: "false"
- build-provider-distributions: ${{ inputs.build-provider-distributions }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index ba16ac4fcb3..7db441c3cc7 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -57,7 +57,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch:
