This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 51b755e7a2a Add option to include pre-release packages in PROD images
(#49348)
51b755e7a2a is described below
commit 51b755e7a2a9d46a64f6c6f971ebde9c10b2d21b
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Apr 16 14:53:29 2025 +0200
Add option to include pre-release packages in PROD images (#49348)
We are releasing RC3 image with not yet released provider packages
so we need to allow installing pre-release distributions in the
images.
---
.github/workflows/release_dockerhub_image.yml | 6 +++++
Dockerfile | 8 +++++++
Dockerfile.ci | 6 +++++
...tput_release-management_release-prod-images.svg | 28 ++++++++++++++--------
...tput_release-management_release-prod-images.txt | 2 +-
.../commands/release_management_commands.py | 9 +++++++
.../commands/release_management_commands_config.py | 1 +
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
scripts/docker/common.sh | 6 +++++
9 files changed, 56 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index 90fde26f50b..7f83b7274c6 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -27,6 +27,11 @@ on: # yamllint disable-line rule:truthy
description: 'Skip Latest: Set to true if not latest.'
default: ''
required: false
+ includePreRelease:
+ description: 'Include pre-release: Set to true to include
pre-releases.'
+ default: ''
+ type: boolean
+ required: false
permissions:
contents: read
packages: read
@@ -47,6 +52,7 @@ jobs:
defaultPythonVersion: ${{
steps.selective-checks.outputs.default-python-version }}
chicken-egg-providers: ${{
steps.selective-checks.outputs.chicken-egg-providers }}
skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' ||
'--skip-latest' }}
+ includePreRelease: ${{ github.event.inputs.includePreRelease == '' && '
' || '--include-pre-release' }}
limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' ||
'--limit-platform linux/amd64' }}
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
diff --git a/Dockerfile b/Dockerfile
index f74ff8edc43..37ac245da5b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -461,6 +461,9 @@ function common::get_packaging_tool() {
export UPGRADE_IF_NEEDED="--upgrade"
UV_CONCURRENT_DOWNLOADS=$(nproc --all)
export UV_CONCURRENT_DOWNLOADS
+ if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease allow"
+ fi
else
echo
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
@@ -471,6 +474,9 @@ function common::get_packaging_tool() {
export EXTRA_UNINSTALL_FLAGS="--yes"
export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --upgrade-strategy
eager"
export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed"
+ if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --pre"
+ fi
fi
}
@@ -1529,6 +1535,7 @@ ARG AIRFLOW_SETUPTOOLS_VERSION
ARG AIRFLOW_UV_VERSION
ARG AIRFLOW_USE_UV
ARG UV_HTTP_TIMEOUT
+ARG INCLUDE_PRE_RELEASE="false"
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
@@ -1554,6 +1561,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_HOME=${AIRFLOW_HOME} \
AIRFLOW_IMAGE_TYPE=${AIRFLOW_IMAGE_TYPE} \
AIRFLOW_UID=${AIRFLOW_UID} \
+ INCLUDE_PRE_RELEASE=${INCLUDE_PRE_RELEASE} \
UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING}
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 22a92199ea6..b722ff01329 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -400,6 +400,9 @@ function common::get_packaging_tool() {
export UPGRADE_IF_NEEDED="--upgrade"
UV_CONCURRENT_DOWNLOADS=$(nproc --all)
export UV_CONCURRENT_DOWNLOADS
+ if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease allow"
+ fi
else
echo
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
@@ -410,6 +413,9 @@ function common::get_packaging_tool() {
export EXTRA_UNINSTALL_FLAGS="--yes"
export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --upgrade-strategy
eager"
export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed"
+ if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --pre"
+ fi
fi
}
diff --git
a/dev/breeze/doc/images/output_release-management_release-prod-images.svg
b/dev/breeze/doc/images/output_release-management_release-prod-images.svg
index f189f6e9c5e..532d7fe7821 100644
--- a/dev/breeze/doc/images/output_release-management_release-prod-images.svg
+++ b/dev/breeze/doc/images/output_release-management_release-prod-images.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 782.0"
xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 830.8"
xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -45,7 +45,7 @@
<defs>
<clipPath id="breeze-release-management-release-prod-images-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="731.0" />
+ <rect x="0" y="0" width="1463.0" height="779.8" />
</clipPath>
<clipPath id="breeze-release-management-release-prod-images-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -134,9 +134,15 @@
<clipPath id="breeze-release-management-release-prod-images-line-28">
<rect x="0" y="684.7" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-release-management-release-prod-images-line-29">
+ <rect x="0" y="709.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-release-prod-images-line-30">
+ <rect x="0" y="733.5" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="780" rx="8"/><text
class="breeze-release-management-release-prod-images-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: release-management release-prod-images</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="828.8" rx="8"/><text
class="breeze-release-management-release-prod-images-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: release-management release-prod-images</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -169,13 +175,15 @@
</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="508" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-20)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="61" y="508"
textLength="158.6"
clip-path="url(#breeze-release-management-release-prod-images-line-20)">--skip-latest</text><text
class="breeze-release-management-release-prod-images-r1" x="390.4" y="508"
textLength="1049.2" clip-path="url(#bree [...]
</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="532.4" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-21)">│</text><text
class="breeze-release-management-release-prod-images-r1" x="390.4" y="532.4"
textLength="1049.2"
clip-path="url(#breeze-release-management-release-prod-images-line-21)">updated). This should only be used if you release image for previous bran
[...]
</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="556.8" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-22)">│</text><text
class="breeze-release-management-release-prod-images-r1" x="390.4" y="556.8"
textLength="1049.2"
clip-path="url(#breeze-release-management-release-prod-images-line-22)">Automatically set when rc/alpha/beta images are built.        &
[...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="581.2" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-23)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="61" y="581.2"
textLength="158.6"
clip-path="url(#breeze-release-management-release-prod-images-line-23)">--slim-images</text><text
class="breeze-release-management-release-prod-images-r1" x="390.4" y="581.2"
textLength="719.8" clip-path="url( [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="605.6" textLength="1464"
clip-path="url(#breeze-release-management-release-prod-images-line-24)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-release-prod-images-r1" x="1464" y="605.6"
textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-24)">
-</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="630" textLength="24.4"
clip-path="url(#breeze-release-management-release-prod-images-line-25)">╭─</text><text
class="breeze-release-management-release-prod-images-r5" x="24.4" y="630"
textLength="195.2"
clip-path="url(#breeze-release-management-release-prod-images-line-25)"> Common options </text><text
class="breeze-release-management-release-prod-images-r5" x="219.6" y="630"
textLength="1220" c [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="654.4" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-26)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="24.4" y="654.4"
textLength="109.8"
clip-path="url(#breeze-release-management-release-prod-images-line-26)">--dry-run</text><text
class="breeze-release-management-release-prod-images-r9" x="158.6" y="654.4"
textLength="24.4" clip-path="url(#br [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="678.8" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-27)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="24.4" y="678.8"
textLength="109.8"
clip-path="url(#breeze-release-management-release-prod-images-line-27)">--verbose</text><text
class="breeze-release-management-release-prod-images-r9" x="158.6" y="678.8"
textLength="24.4" clip-path="url(#br [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="703.2" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-28)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="24.4" y="703.2"
textLength="73.2"
clip-path="url(#breeze-release-management-release-prod-images-line-28)">--help</text><text
class="breeze-release-management-release-prod-images-r9" x="158.6" y="703.2"
textLength="24.4" clip-path="url(#breeze [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="727.6" textLength="1464"
clip-path="url(#breeze-release-management-release-prod-images-line-29)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-release-prod-images-r1" x="1464" y="727.6"
textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-29)">
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="581.2" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-23)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="61" y="581.2"
textLength="256.2"
clip-path="url(#breeze-release-management-release-prod-images-line-23)">--include-pre-release</text><text
class="breeze-release-management-release-prod-images-r1" x="390.4" y="581.2"
textLength="1049.2" clip-p [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="605.6" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-24)">│</text><text
class="breeze-release-management-release-prod-images-r1" x="390.4" y="605.6"
textLength="1049.2"
clip-path="url(#breeze-release-management-release-prod-images-line-24)">when we want to build an RC image with RC provider versions.   
[...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="630" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-25)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="61" y="630"
textLength="158.6"
clip-path="url(#breeze-release-management-release-prod-images-line-25)">--slim-images</text><text
class="breeze-release-management-release-prod-images-r1" x="390.4" y="630"
textLength="719.8" clip-path="url(#breez [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="654.4" textLength="1464"
clip-path="url(#breeze-release-management-release-prod-images-line-26)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-release-prod-images-r1" x="1464" y="654.4"
textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-26)">
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="678.8" textLength="24.4"
clip-path="url(#breeze-release-management-release-prod-images-line-27)">╭─</text><text
class="breeze-release-management-release-prod-images-r5" x="24.4" y="678.8"
textLength="195.2"
clip-path="url(#breeze-release-management-release-prod-images-line-27)"> Common options </text><text
class="breeze-release-management-release-prod-images-r5" x="219.6" y="678.8"
textLength="1 [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="703.2" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-28)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="24.4" y="703.2"
textLength="109.8"
clip-path="url(#breeze-release-management-release-prod-images-line-28)">--dry-run</text><text
class="breeze-release-management-release-prod-images-r9" x="158.6" y="703.2"
textLength="24.4" clip-path="url(#br [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="727.6" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-29)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="24.4" y="727.6"
textLength="109.8"
clip-path="url(#breeze-release-management-release-prod-images-line-29)">--verbose</text><text
class="breeze-release-management-release-prod-images-r9" x="158.6" y="727.6"
textLength="24.4" clip-path="url(#br [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="752" textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-30)">│</text><text
class="breeze-release-management-release-prod-images-r4" x="24.4" y="752"
textLength="73.2"
clip-path="url(#breeze-release-management-release-prod-images-line-30)">--help</text><text
class="breeze-release-management-release-prod-images-r9" x="158.6" y="752"
textLength="24.4" clip-path="url(#breeze-relea [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0"
y="776.4" textLength="1464"
clip-path="url(#breeze-release-management-release-prod-images-line-31)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-release-prod-images-r1" x="1464" y="776.4"
textLength="12.2"
clip-path="url(#breeze-release-management-release-prod-images-line-31)">
</text>
</g>
</g>
diff --git
a/dev/breeze/doc/images/output_release-management_release-prod-images.txt
b/dev/breeze/doc/images/output_release-management_release-prod-images.txt
index fb3e183250a..08d32db2e99 100644
--- a/dev/breeze/doc/images/output_release-management_release-prod-images.txt
+++ b/dev/breeze/doc/images/output_release-management_release-prod-images.txt
@@ -1 +1 @@
-3458927ab24515697706bc892a79795d
+18bfc23895e3344f5e43d89294d1aa10
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 024c207f1b0..2c638f03a0e 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -2150,6 +2150,12 @@ def clean_old_provider_artifacts(
"This should only be used if you release image for previous branches.
Automatically set when "
"rc/alpha/beta images are built.",
)
[email protected](
+ "--include-pre-release",
+ is_flag=True,
+ help="Whether to Include pre-release distributions from PyPI when building
images. Useful when we "
+ "want to build an RC image with RC provider versions.",
+)
@click.option(
"--slim-images",
is_flag=True,
@@ -2164,6 +2170,7 @@ def release_prod_images(
limit_python: str | None,
commit_sha: str | None,
skip_latest: bool,
+ include_pre_release: bool,
chicken_egg_providers: str,
):
perform_environment_checks()
@@ -2214,6 +2221,7 @@ def release_prod_images(
"AIRFLOW_CONSTRAINTS": "constraints-no-providers",
"PYTHON_BASE_IMAGE": f"python:{python}-slim-bookworm",
"AIRFLOW_VERSION": airflow_version,
+ "INCLUDE_PRE_RELEASE": "true" if include_pre_release else
"false",
}
if commit_sha:
slim_build_args["COMMIT_SHA"] = commit_sha
@@ -2249,6 +2257,7 @@ def release_prod_images(
regular_build_args = {
"PYTHON_BASE_IMAGE": f"python:{python}-slim-bookworm",
"AIRFLOW_VERSION": airflow_version,
+ "INCLUDE_PRE_RELEASE": "true" if include_pre_release else
"false",
}
if commit_sha:
regular_build_args["COMMIT_SHA"] = commit_sha
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index 827624e957d..2ab70a0a072 100644
---
a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++
b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -308,6 +308,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str
| list[str]]]] = {
"--limit-python",
"--limit-platform",
"--skip-latest",
+ "--include-pre-release",
"--slim-images",
],
}
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 0c4aecc53a4..c17d3f10da7 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -635,7 +635,7 @@ DEFAULT_EXTRAS = [
# END OF EXTRAS LIST UPDATED BY PRE COMMIT
]
-CHICKEN_EGG_PROVIDERS = " ".join(["git", "common.messaging", "fab"])
+CHICKEN_EGG_PROVIDERS = " ".join(["git", "common.messaging", "fab",
"standard"])
PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str | list[str]]] = [
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index 89f42f35284..e6bcaf5195f 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -48,6 +48,9 @@ function common::get_packaging_tool() {
export UPGRADE_IF_NEEDED="--upgrade"
UV_CONCURRENT_DOWNLOADS=$(nproc --all)
export UV_CONCURRENT_DOWNLOADS
+ if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease allow"
+ fi
else
echo
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
@@ -58,6 +61,9 @@ function common::get_packaging_tool() {
export EXTRA_UNINSTALL_FLAGS="--yes"
export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --upgrade-strategy
eager"
export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed"
+ if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --pre"
+ fi
fi
}