This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 55834a7f1f6 Migrated prod builds to use python built from source
(#53770)
55834a7f1f6 is described below
commit 55834a7f1f693a71d22e33cf6df18933739eb754
Author: Aritra Basu <[email protected]>
AuthorDate: Sat Aug 30 20:18:30 2025 +0530
Migrated prod builds to use python built from source (#53770)
Moved the prod build to also use the src built python
as done by CI. This let's us iterate faster on our
python versions without needing to wait for the community
image of python.
This involves a number of changes:
* 3.0.5 -> 3.0.6 airflow version change
* use released official packages rather than git repo to install Python
* added wget (it was added in the original image to pull python packages so
added for compatibility) added those flags to python build (same as in the
* original build)
* --with-ensurepip --build="$gnuArch"
* --enable-loadable-sqlite-extension
* --enable-option-checking=fatal
* --enable-shared
* --with-lto
* added cleanup of apt after installing packages
* added removal of .pyc/.test etc. files (saves 350 MB)
* added relinking of symbolic links from /usr/python/bin to /usr/local/bin
in
the "main" image as well as in the build image.
* we do not need AIRFLOW_SETUPTOOLS_VERSION any more - this was only added
to
upgrade setuptools, because the Python official image had a very old
setuptools version.
* checked all "customize" scripts and make them "work"
* Updated the 'version upgrade" script to upgrade AIRFLOW_PYTHON_VERSION
everywhere
* Updated Changelog and documentation to update new ways of building images
* removed installation with GitHub URL (it won't work easily after splitting
to multiple packages - not easily at least) and it's not needed
* all python, pip and similar links are created in /usr/python/bin
* /usr/python/bin is always first in the PATH - before /usr/local/bin
* added changelog entry explaining that Python's installation home has been
moved to /usr/python/ from /usr/local
* removal of installed editable distributions in breeze happens now first
and
THEN we install when --use-airflow-version is used.
* LD_LIBRARY_PATH was not set so the shared python libraries could not be
loaded when venv was created
Co-authored-by: Jarek Potiuk <[email protected]>
---
Dockerfile | 169 ++++++++++++++++-----
Dockerfile.ci | 142 ++++++++++++-----
RELEASE_NOTES.rst | 2 +
.../modules_management.rst | 2 +-
airflow-core/docs/start.rst | 2 +-
contributing-docs/03_contributors_quick_start.rst | 2 +-
.../commands/release_management_commands.py | 4 +-
dev/breeze/src/airflow_breeze/global_constants.py | 16 +-
.../src/airflow_breeze/params/build_ci_params.py | 6 +-
.../src/airflow_breeze/params/build_prod_params.py | 6 +-
.../airflow_breeze/params/common_build_params.py | 2 +-
docker-stack-docs/build-arg-ref.rst | 20 ++-
docker-stack-docs/build.rst | 43 ------
docker-stack-docs/changelog.rst | 23 +++
.../customizing/add-build-essential-custom.sh | 9 +-
.../docker-examples/customizing/custom-pip.sh | 2 +-
.../docker-examples/customizing/custom-sources.sh | 9 +-
.../customizing/github-different-repository.sh | 40 -----
.../docker-examples/customizing/github-main.sh | 40 -----
.../customizing/github-v2-2-test.sh | 41 -----
.../docker-examples/customizing/mysql-client.sh | 2 +-
.../customizing/own-requirements.sh | 6 +-
.../customizing/pypi-dev-runtime-deps.sh | 9 +-
.../customizing/pypi-extras-and-deps.sh | 7 +-
.../customizing/pypi-selected-version.sh | 7 +-
.../docker-examples/customizing/stable-airflow.sh | 2 +-
.../docker-examples/customizing/use-uv.sh | 2 +-
.../restricted/restricted_environments.sh | 3 +-
docker-stack-docs/pyproject.toml | 2 +-
docs/spelling_wordlist.txt | 1 +
reproducible_build.yaml | 4 +-
scripts/ci/prek/upgrade_important_versions.py | 57 ++++---
scripts/docker/common.sh | 6 -
.../docker/install_airflow_when_building_images.sh | 4 +-
scripts/docker/install_os_dependencies.sh | 118 ++++++++++++--
scripts/docker/install_os_dependencies_ci.sh | 167 --------------------
.../in_container/install_airflow_and_providers.py | 28 ++--
.../in_container/install_airflow_python_client.py | 2 +-
.../in_container/run_check_imports_in_providers.py | 2 +-
39 files changed, 489 insertions(+), 520 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 927b45479a3..62be753cd80 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,7 +48,7 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow
# latest released version here
ARG AIRFLOW_VERSION="3.0.6"
-ARG PYTHON_BASE_IMAGE="python:3.10-slim-bookworm"
+ARG BASE_IMAGE="debian:bookworm-slim"
# You can swap comments between those two args to test pip from the main
version
@@ -56,7 +56,6 @@ ARG PYTHON_BASE_IMAGE="python:3.10-slim-bookworm"
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
ARG AIRFLOW_PIP_VERSION=25.2
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main";
-ARG AIRFLOW_SETUPTOOLS_VERSION=80.9.0
ARG AIRFLOW_UV_VERSION=0.8.14
ARG AIRFLOW_USE_UV="false"
ARG UV_HTTP_TIMEOUT="300"
@@ -98,26 +97,34 @@ COPY <<"EOF" /install_os_dependencies.sh
set -euo pipefail
if [[ "$#" != 1 ]]; then
- echo "ERROR! There should be 'runtime' or 'dev' parameter passed as
argument.".
+ echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
exit 1
fi
+AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
+GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
+
if [[ "${1}" == "runtime" ]]; then
INSTALLATION_TYPE="RUNTIME"
elif [[ "${1}" == "dev" ]]; then
- INSTALLATION_TYPE="dev"
+ INSTALLATION_TYPE="DEV"
+elif [[ "${1}" == "ci" ]]; then
+ INSTALLATION_TYPE="CI"
else
- echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime'
or 'dev'.".
+ echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
exit 1
fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
- DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
+ DEV_APT_DEPS="apt-transport-https apt-utils build-essential dirmngr \
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev"
+software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev wget \
+gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
+libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
+libzstd-dev"
export DEV_APT_DEPS
fi
}
@@ -136,10 +143,10 @@ function get_runtime_apt_deps() {
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
- RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
+ RUNTIME_APT_DEPS="apt-transport-https apt-utils \
curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
-lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
+lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc wget"
export RUNTIME_APT_DEPS
fi
}
@@ -162,7 +169,7 @@ function install_docker_cli() {
function install_debian_dev_dependencies() {
apt-get update
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release
ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_DEV_APT_ENV?}
if [[ ${DEV_APT_COMMAND} != "" ]]; then
@@ -184,10 +191,30 @@ function install_debian_dev_dependencies() {
apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
}
+
+function link_python() {
+ # link python binaries to /usr/local/bin and /usr/python/bin with and
without 3 suffix
+ # Links in /usr/local/bin are needed for tools that expect python to be
there
+ # Links in /usr/python/bin are needed for tools that are detecting home of
python installation including
+ # lib/site-packages. The /usr/python/bin should be first in PATH in order
to help with the last part.
+ ldconfig
+ for dst in pip3 python3 python3-config; do
+ src="$(echo "${dst}" | tr -d 3)"
+ echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
+ ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}"
+ for dir in /usr/local/bin /usr/python/bin; do
+ if [[ ! -e "${dir}/${src}" ]]; then
+ echo "Creating ${src} - > ${dst} link in ${dir}"
+ ln -sv "${dir}/${dst}" "${dir}/${src}"
+ fi
+ done
+ done
+}
+
function install_debian_runtime_dependencies() {
apt-get update
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release
ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_RUNTIME_APT_ENV?}
if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then
@@ -201,6 +228,71 @@ function install_debian_runtime_dependencies() {
apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS}
${ADDITIONAL_RUNTIME_APT_DEPS}
apt-get autoremove -yqq --purge
apt-get clean
+ link_python
+ rm -rf /var/lib/apt/lists/* /var/log/*
+}
+
+function install_python() {
+ wget -O python.tar.xz
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz";
+ wget -O python.tar.xz.asc
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc";;
+ declare -A keys=(
+ # gpg: key B26995E310250568: public key "\xc5\x81ukasz Langa (GPG
langa.pl) <[email protected]>" imported
+ # https://peps.python.org/pep-0596/#release-manager-and-crew
+ [3.9]="E3FF2839C048B25C084DEBE9B26995E310250568"
+ # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado
<[email protected]>" imported
+ # https://peps.python.org/pep-0619/#release-manager-and-crew
+ [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D"
+ # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado
<[email protected]>" imported
+ # https://peps.python.org/pep-0664/#release-manager-and-crew
+ [3.11]="A035C8C19219BA821ECEA86B64E628F8D684696D"
+ # gpg: key A821E680E5FA6305: public key "Thomas Wouters
<[email protected]>" imported
+ # https://peps.python.org/pep-0693/#release-manager-and-crew
+ [3.12]="7169605F62C751356D054A26A821E680E5FA6305"
+ # gpg: key A821E680E5FA6305: public key "Thomas Wouters
<[email protected]>" imported
+ # https://peps.python.org/pep-0719/#release-manager-and-crew
+ [3.13]="7169605F62C751356D054A26A821E680E5FA6305"
+ )
+ major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}"
+ echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})"
+ GNUPGHOME="$(mktemp -d)"; export GNUPGHOME;
+ gpg_key="${keys[${major_minor_version}]}"
+ echo "Using GPG key ${gpg_key}"
+ gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "${gpg_key}"
+ gpg --batch --verify python.tar.xz.asc python.tar.xz;
+ gpgconf --kill all
+ rm -rf "$GNUPGHOME" python.tar.xz.asc
+ mkdir -p /usr/src/python
+ tar --extract --directory /usr/src/python --strip-components=1 --file
python.tar.xz
+ rm python.tar.xz
+ cd /usr/src/python
+ arch="$(dpkg --print-architecture)"; arch="${arch##*-}"
+ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"
+ EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"
+ EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer
-mno-omit-leaf-frame-pointer";
+ LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
+ LDFLAGS="${LDFLAGS:--Wl},--strip-all"
+ ./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip
--build="$gnuArch" \
+ --enable-loadable-sqlite-extensions --enable-option-checking=fatal
--enable-shared --with-lto
+ make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+ "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
+ make -s -j "$(nproc)" install
+ cd /
+ rm -rf /usr/src/python
+ find /usr/python -depth \
+ \( \
+ \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+ -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name
'libpython*.a' \) \) \
+ \) -exec rm -rf '{}' +
+ link_python
+}
+
+function install_golang() {
+ curl
"https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg
--print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
+ rm -rf /usr/local/go && tar -C /usr/local -xzf
go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
+}
+
+function apt_clean() {
+ apt-get purge -y --auto-remove -o
APT::AutoRemove::RecommendsImportant=false
rm -rf /var/lib/apt/lists/* /var/log/*
}
@@ -208,11 +300,16 @@ if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
get_runtime_apt_deps
install_debian_runtime_dependencies
install_docker_cli
-
+ apt_clean
else
get_dev_apt_deps
install_debian_dev_dependencies
+ install_python
+ if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
+ install_golang
+ fi
install_docker_cli
+ apt_clean
fi
EOF
@@ -590,12 +687,6 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
fi
fi
- if [[ ${AIRFLOW_SETUPTOOLS_VERSION=} != "" ]]; then
- echo
- echo "${COLOR_BLUE}Installing setuptools version
${AIRFLOW_SETUPTOOLS_VERSION} {COLOR_RESET}"
- echo
- pip install --root-user-action ignore
setuptools==${AIRFLOW_SETUPTOOLS_VERSION}
- fi
if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
echo
echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
@@ -958,13 +1049,11 @@ function install_from_external_spec() {
local installation_command_flags
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then
installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
- elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then
- installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}] @
${AIRFLOW_VERSION_SPECIFICATION/apache-airflow @//}"
else
echo
echo "${COLOR_RED}The '${INSTALLATION_METHOD}' installation method is
not supported${COLOR_RESET}"
echo
- echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow',
'apache-airflow @ URL')${COLOR_RESET}"
+ echo "${COLOR_YELLOW}Supported methods are ('.',
'apache-airflow')${COLOR_RESET}"
echo
exit 1
fi
@@ -1460,16 +1549,20 @@ EOF
##############################################################################################
# This is the build image where we build all dependencies
##############################################################################################
-FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
+FROM ${BASE_IMAGE} as airflow-build-image
# Nolog bash flag is currently ignored - but you can replace it with
# xtrace - to show commands executed)
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o",
"nolog", "-c"]
-ARG PYTHON_BASE_IMAGE
-ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
+ARG BASE_IMAGE
+
+# Make sure noninteractive debian install is used and language variables set
+# as well as LD_LIBRARY_PATH to /usr/local/lib and /usr/python/lib is set, so
that
+# shared libraries installed there are found
+ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
- LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
+ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
LD_LIBRARY_PATH=/usr/python/lib:/usr/local/lib \
PIP_CACHE_DIR=/tmp/.cache/pip \
UV_CACHE_DIR=/tmp/.cache/uv
@@ -1485,6 +1578,8 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \
ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} \
ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV}
+ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
+
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
RUN bash /scripts/docker/install_os_dependencies.sh dev
@@ -1568,7 +1663,6 @@ RUN if [[ -f /docker-context-files/pip.conf ]]; then \
ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
ARG AIRFLOW_PIP_VERSION
-ARG AIRFLOW_SETUPTOOLS_VERSION
ARG AIRFLOW_UV_VERSION
ARG AIRFLOW_USE_UV
ARG UV_HTTP_TIMEOUT
@@ -1576,7 +1670,6 @@ ARG INCLUDE_PRE_RELEASE="false"
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
- AIRFLOW_SETUPTOOLS_VERSION=${AIRFLOW_SETUPTOOLS_VERSION} \
UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT} \
AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \
AIRFLOW_VERSION=${AIRFLOW_VERSION} \
@@ -1627,7 +1720,7 @@ ARG USE_CONSTRAINTS_FOR_CONTEXT_DISTRIBUTIONS="false"
# By default PIP installs everything to ~/.local and it's also treated as
VIRTUALENV
ENV VIRTUAL_ENV="${AIRFLOW_USER_HOME_DIR}/.local"
-
+ENV PATH="/usr/python/bin:$PATH"
RUN bash /scripts/docker/install_packaging_tools.sh; bash
/scripts/docker/create_prod_venv.sh
COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO}
@@ -1684,7 +1777,7 @@ RUN
--mount=type=cache,id=prod-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/tmp/.
# This is the actual Airflow image - much smaller than the build one. We copy
# installed Airflow and all its dependencies from the build image to make it
smaller.
##############################################################################################
-FROM ${PYTHON_BASE_IMAGE} as main
+FROM ${BASE_IMAGE} as main
# Nolog bash flag is currently ignored - but you can replace it with other
flags (for example
# xtrace - to show commands executed)
@@ -1698,12 +1791,14 @@ LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.image="airflow" \
org.apache.airflow.uid="${AIRFLOW_UID}"
-ARG PYTHON_BASE_IMAGE
+ARG BASE_IMAGE
-ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
- # Make sure noninteractive debian install is used and language variables
set
+# Make sure noninteractive debian install is used and language variables set
+# as well as LD_LIBRARY_PATH to /usr/local/lib and /usr/python/lib is set, so
that
+# shared libraries installed there are found
+ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
- LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 LD_LIBRARY_PATH=/usr/local/lib \
+ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
LD_LIBRARY_PATH=/usr/python/lib:/usr/local/lib \
PIP_CACHE_DIR=/tmp/.cache/pip \
UV_CACHE_DIR=/tmp/.cache/uv
@@ -1729,6 +1824,7 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \
AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}
+COPY --from=airflow-build-image "/usr/python/" "/usr/python/"
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
RUN bash /scripts/docker/install_os_dependencies.sh runtime
@@ -1741,7 +1837,7 @@ ARG AIRFLOW_HOME
ARG AIRFLOW_IMAGE_TYPE
# By default PIP installs everything to ~/.local
-ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \
+ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:/usr/python/bin:${PATH}" \
VIRTUAL_ENV="${AIRFLOW_USER_HOME_DIR}/.local" \
AIRFLOW_UID=${AIRFLOW_UID} \
AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
@@ -1803,9 +1899,9 @@ RUN sed --in-place=.bak
"s/secure_path=\"/secure_path=\"$(echo -n ${AIRFLOW_USER
ARG AIRFLOW_VERSION
ARG AIRFLOW_PIP_VERSION
-ARG AIRFLOW_SETUPTOOLS_VERSION
ARG AIRFLOW_UV_VERSION
ARG AIRFLOW_USE_UV
+ARG AIRFLOW_PYTHON_VERSION
# See
https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
# to learn more about the way how signals are handled by the image
@@ -1813,11 +1909,11 @@ ARG AIRFLOW_USE_UV
ENV DUMB_INIT_SETSID="1" \
PS1="(airflow)" \
AIRFLOW_VERSION=${AIRFLOW_VERSION} \
+ AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION} \
AIRFLOW__CORE__LOAD_EXAMPLES="false" \
PATH="/root/bin:${PATH}" \
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
- AIRFLOW_SETUPTOOLS_VERSION=${AIRFLOW_SETUPTOOLS_VERSION} \
AIRFLOW_USE_UV=${AIRFLOW_USE_UV}
# Add protection against running pip as root user
@@ -1845,6 +1941,7 @@ LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.component="airflow" \
org.apache.airflow.image="airflow" \
org.apache.airflow.version="${AIRFLOW_VERSION}" \
+ org.apache.airflow.python.version="${AIRFLOW_PYTHON_VERSION}" \
org.apache.airflow.uid="${AIRFLOW_UID}" \
org.apache.airflow.main-image.build-id="${BUILD_ID}" \
org.apache.airflow.main-image.commit-sha="${COMMIT_SHA}" \
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 2dd7ec03ec0..560750cadb3 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -31,8 +31,8 @@ FROM ${BASE_IMAGE} as scripts
# make the PROD Dockerfile standalone
##############################################################################################
-# The content below is automatically copied from
scripts/docker/install_os_dependencies_ci.sh
-COPY <<"EOF" /install_os_dependencies_ci.sh
+# The content below is automatically copied from
scripts/docker/install_os_dependencies.sh
+COPY <<"EOF" /install_os_dependencies.sh
#!/usr/bin/env bash
set -euo pipefail
@@ -41,7 +41,7 @@ if [[ "$#" != 1 ]]; then
exit 1
fi
-AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10}
+AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
if [[ "${1}" == "runtime" ]]; then
@@ -57,11 +57,11 @@ fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
- DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
+ DEV_APT_DEPS="apt-transport-https apt-utils build-essential dirmngr \
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \
+software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev wget \
gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
libzstd-dev"
@@ -83,10 +83,10 @@ function get_runtime_apt_deps() {
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
- RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
+ RUNTIME_APT_DEPS="apt-transport-https apt-utils \
curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
-lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
+lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc wget"
export RUNTIME_APT_DEPS
fi
}
@@ -109,7 +109,7 @@ function install_docker_cli() {
function install_debian_dev_dependencies() {
apt-get update
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release
ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_DEV_APT_ENV?}
if [[ ${DEV_APT_COMMAND} != "" ]]; then
@@ -131,10 +131,30 @@ function install_debian_dev_dependencies() {
apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
}
+
+function link_python() {
+ # link python binaries to /usr/local/bin and /usr/python/bin with and
without 3 suffix
+ # Links in /usr/local/bin are needed for tools that expect python to be
there
+ # Links in /usr/python/bin are needed for tools that are detecting home of
python installation including
+ # lib/site-packages. The /usr/python/bin should be first in PATH in order
to help with the last part.
+ ldconfig
+ for dst in pip3 python3 python3-config; do
+ src="$(echo "${dst}" | tr -d 3)"
+ echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
+ ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}"
+ for dir in /usr/local/bin /usr/python/bin; do
+ if [[ ! -e "${dir}/${src}" ]]; then
+ echo "Creating ${src} - > ${dst} link in ${dir}"
+ ln -sv "${dir}/${dst}" "${dir}/${src}"
+ fi
+ done
+ done
+}
+
function install_debian_runtime_dependencies() {
apt-get update
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release
ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_RUNTIME_APT_ENV?}
if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then
@@ -148,19 +168,62 @@ function install_debian_runtime_dependencies() {
apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS}
${ADDITIONAL_RUNTIME_APT_DEPS}
apt-get autoremove -yqq --purge
apt-get clean
+ link_python
rm -rf /var/lib/apt/lists/* /var/log/*
}
function install_python() {
- git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1
https://github.com/python/cpython.git
- cd cpython
- ./configure --enable-optimizations
- make -s -j "$(nproc)" all
+ wget -O python.tar.xz
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz";
+ wget -O python.tar.xz.asc
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc";;
+ declare -A keys=(
+ # gpg: key B26995E310250568: public key "\xc5\x81ukasz Langa (GPG
langa.pl) <[email protected]>" imported
+ # https://peps.python.org/pep-0596/#release-manager-and-crew
+ [3.9]="E3FF2839C048B25C084DEBE9B26995E310250568"
+ # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado
<[email protected]>" imported
+ # https://peps.python.org/pep-0619/#release-manager-and-crew
+ [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D"
+ # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado
<[email protected]>" imported
+ # https://peps.python.org/pep-0664/#release-manager-and-crew
+ [3.11]="A035C8C19219BA821ECEA86B64E628F8D684696D"
+ # gpg: key A821E680E5FA6305: public key "Thomas Wouters
<[email protected]>" imported
+ # https://peps.python.org/pep-0693/#release-manager-and-crew
+ [3.12]="7169605F62C751356D054A26A821E680E5FA6305"
+ # gpg: key A821E680E5FA6305: public key "Thomas Wouters
<[email protected]>" imported
+ # https://peps.python.org/pep-0719/#release-manager-and-crew
+ [3.13]="7169605F62C751356D054A26A821E680E5FA6305"
+ )
+ major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}"
+ echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})"
+ GNUPGHOME="$(mktemp -d)"; export GNUPGHOME;
+ gpg_key="${keys[${major_minor_version}]}"
+ echo "Using GPG key ${gpg_key}"
+ gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "${gpg_key}"
+ gpg --batch --verify python.tar.xz.asc python.tar.xz;
+ gpgconf --kill all
+ rm -rf "$GNUPGHOME" python.tar.xz.asc
+ mkdir -p /usr/src/python
+ tar --extract --directory /usr/src/python --strip-components=1 --file
python.tar.xz
+ rm python.tar.xz
+ cd /usr/src/python
+ arch="$(dpkg --print-architecture)"; arch="${arch##*-}"
+ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"
+ EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"
+ EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer
-mno-omit-leaf-frame-pointer";
+ LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
+ LDFLAGS="${LDFLAGS:--Wl},--strip-all"
+ ./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip
--build="$gnuArch" \
+ --enable-loadable-sqlite-extensions --enable-option-checking=fatal
--enable-shared --with-lto
+ make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+ "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
make -s -j "$(nproc)" install
- ln -s /usr/local/bin/python3 /usr/local/bin/python
- ln -s /usr/local/bin/pip3 /usr/local/bin/pip
- cd ..
- rm -rf cpython
+ cd /
+ rm -rf /usr/src/python
+ find /usr/python -depth \
+ \( \
+ \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+ -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name
'libpython*.a' \) \) \
+ \) -exec rm -rf '{}' +
+ link_python
}
function install_golang() {
@@ -168,13 +231,17 @@ function install_golang() {
rm -rf /usr/local/go && tar -C /usr/local -xzf
go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
}
+function apt_clean() {
+ apt-get purge -y --auto-remove -o
APT::AutoRemove::RecommendsImportant=false
+ rm -rf /var/lib/apt/lists/* /var/log/*
+}
+
if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
get_runtime_apt_deps
install_debian_runtime_dependencies
install_docker_cli
-
+ apt_clean
else
-
get_dev_apt_deps
install_debian_dev_dependencies
install_python
@@ -182,6 +249,7 @@ else
install_golang
fi
install_docker_cli
+ apt_clean
fi
EOF
@@ -559,12 +627,6 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
fi
fi
- if [[ ${AIRFLOW_SETUPTOOLS_VERSION=} != "" ]]; then
- echo
- echo "${COLOR_BLUE}Installing setuptools version
${AIRFLOW_SETUPTOOLS_VERSION} {COLOR_RESET}"
- echo
- pip install --root-user-action ignore
setuptools==${AIRFLOW_SETUPTOOLS_VERSION}
- fi
if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
echo
echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
@@ -741,13 +803,11 @@ function install_from_external_spec() {
local installation_command_flags
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then
installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
- elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then
- installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}] @
${AIRFLOW_VERSION_SPECIFICATION/apache-airflow @//}"
else
echo
echo "${COLOR_RED}The '${INSTALLATION_METHOD}' installation method is
not supported${COLOR_RESET}"
echo
- echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow',
'apache-airflow @ URL')${COLOR_RESET}"
+ echo "${COLOR_YELLOW}Supported methods are ('.',
'apache-airflow')${COLOR_RESET}"
echo
exit 1
fi
@@ -1305,9 +1365,11 @@ ARG
AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow";
ARG DEPENDENCIES_EPOCH_NUMBER="15"
# Make sure noninteractive debian install is used and language variables set
+# As well as LD_LIBRARY_PATH to /usr/local/lib and /usr/python/lib is set, so
that
+# shared libraries installed there are found
ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
- LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
+ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
LD_LIBRARY_PATH=/usr/python/lib:/usr/local/lib \
DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
INSTALL_MYSQL_CLIENT="true" \
INSTALL_MSSQL_CLIENT="true" \
@@ -1331,13 +1393,13 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}
-ARG AIRFLOW_PYTHON_VERSION=v3.10.18
-ENV AIRFLOW_PYTHON_VERSION=$AIRFLOW_PYTHON_VERSION
-ENV GOLANG_MAJOR_MINOR_VERSION=1.25.0
+ARG AIRFLOW_PYTHON_VERSION="3.12.11"
+ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
+ENV GOLANG_MAJOR_MINOR_VERSION="1.25.0"
-COPY --from=scripts install_os_dependencies_ci.sh /scripts/docker/
+COPY --from=scripts install_os_dependencies.sh /scripts/docker/
-RUN bash /scripts/docker/install_os_dependencies_ci.sh ci
+RUN bash /scripts/docker/install_os_dependencies.sh ci
COPY --from=scripts common.sh /scripts/docker/
@@ -1463,21 +1525,17 @@ COPY --from=scripts common.sh
install_packaging_tools.sh install_additional_depe
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
ARG AIRFLOW_PIP_VERSION=25.2
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main";
-ARG AIRFLOW_SETUPTOOLS_VERSION=80.9.0
ARG AIRFLOW_UV_VERSION=0.8.14
-# TODO(potiuk): automate with upgrade check (possibly)
ARG AIRFLOW_PREK_VERSION="0.1.4"
+# UV_LINK_MODE=copy is needed since we are using cache mounted from the host
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
- AIRFLOW_SETUPTOOLS_VERSION=${AIRFLOW_SETUPTOOLS_VERSION} \
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
- # This is needed since we are using cache mounted from the host
UV_LINK_MODE=copy \
AIRFLOW_PREK_VERSION=${AIRFLOW_PREK_VERSION}
-# The PATH is needed for PIPX to find the tools installed and cargo to build
the wheels
-ENV PATH="/root/.local/bin:/root/.cargo/bin:${PATH}"
-
+# The PATH is needed for python to find installed and cargo to build the wheels
+ENV PATH="/usr/python/bin:/root/.local/bin:/root/.cargo/bin:${PATH}"
# Useful for creating a cache id based on the underlying architecture,
preventing the use of cached python packages from
# an incorrect architecture.
ARG TARGETARCH
@@ -1487,6 +1545,7 @@ ARG DEPENDENCY_CACHE_EPOCH="0"
# Install useful command line tools in their own virtualenv so that they do
not clash with
# dependencies installed in Airflow also reinstall PIP and UV to make sure
they are installed
# in the version specified above
+
RUN bash /scripts/docker/install_packaging_tools.sh
COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/
@@ -1562,6 +1621,7 @@ LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.component="airflow" \
org.apache.airflow.image="airflow-ci" \
org.apache.airflow.version="${AIRFLOW_VERSION}" \
+ org.apache.airflow.python.version="${AIRFLOW_PYTHON_VERSION}" \
org.apache.airflow.uid="0" \
org.apache.airflow.gid="0" \
org.apache.airflow.build-id="${BUILD_ID}" \
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
index c982b01abdc..b10c955bb97 100644
--- a/RELEASE_NOTES.rst
+++ b/RELEASE_NOTES.rst
@@ -70,6 +70,8 @@ Doc Only Changes
Airflow 3.0.5 (2025-08-20)
--------------------------
+This release has been yanked.
+
Significant Changes
^^^^^^^^^^^^^^^^^^^
diff --git
a/airflow-core/docs/administration-and-deployment/modules_management.rst
b/airflow-core/docs/administration-and-deployment/modules_management.rst
index e6b8b7d14fc..c593486ac3e 100644
--- a/airflow-core/docs/administration-and-deployment/modules_management.rst
+++ b/airflow-core/docs/administration-and-deployment/modules_management.rst
@@ -246,7 +246,7 @@ Below is the sample output of the ``airflow info`` command:
uname | uname_result(system='Linux', node='85cd7ab7018e',
release='4.19.76-linuxkit', version='#1 SMP Tue May 26 11:42:35 UTC 2020',
machine='x86_64', processor='')
locale | ('en_US', 'UTF-8')
python_version | 3.9.6 (default, Nov 25 2020, 02:47:44) [GCC 8.3.0]
- python_location | /usr/local/bin/python
+ python_location | /usr/python/bin/python
Tools info
git | git version 2.20.1
diff --git a/airflow-core/docs/start.rst b/airflow-core/docs/start.rst
index d4016500161..eadc39469ef 100644
--- a/airflow-core/docs/start.rst
+++ b/airflow-core/docs/start.rst
@@ -68,7 +68,7 @@ This quick start guide will help you bootstrap an Airflow
standalone instance on
:substitutions:
- AIRFLOW_VERSION=3.0.0
+ AIRFLOW_VERSION=3.0.3
# Extract the version of Python you have installed. If you're currently
using a Python version that is not supported by Airflow, you may want to set
this manually.
# See above for supported versions.
diff --git a/contributing-docs/03_contributors_quick_start.rst
b/contributing-docs/03_contributors_quick_start.rst
index d36e777b87a..b019ff5f951 100644
--- a/contributing-docs/03_contributors_quick_start.rst
+++ b/contributing-docs/03_contributors_quick_start.rst
@@ -693,7 +693,7 @@ All Tests are inside ./tests directory.
root@63528318c8b1:/opt/airflow# pytest tests/utils/test_dates.py
============================================================= test session
starts ==============================================================
- platform linux -- Python 3.10.20, pytest-8.3.3, pluggy-1.5.0 --
/usr/local/bin/python
+ platform linux -- Python 3.10.20, pytest-8.3.3, pluggy-1.5.0 --
/usr/python/bin/python
cachedir: .pytest_cache
rootdir: /opt/airflow
configfile: pyproject.toml
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index d84b71fc075..74a5ac583b3 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -90,6 +90,7 @@ from
airflow_breeze.commands.common_package_installation_options import (
)
from airflow_breeze.commands.release_management_group import release_management
from airflow_breeze.global_constants import (
+ ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION,
ALLOWED_DEBIAN_VERSIONS,
ALLOWED_DISTRIBUTION_FORMATS,
ALLOWED_PLATFORMS,
@@ -2152,7 +2153,8 @@ def release_prod_images(
for python in python_versions:
build_args = {
"AIRFLOW_CONSTRAINTS": "constraints-no-providers",
- "PYTHON_BASE_IMAGE": f"python:{python}-slim-bookworm",
+ "BASE_IMAGE": "debian:bookworm-slim",
+ "AIRFLOW_PYTHON_VERSION":
ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(python, python),
"AIRFLOW_VERSION": airflow_version,
"INCLUDE_PRE_RELEASE": "true" if include_pre_release else "false",
"INSTALL_DISTRIBUTIONS_FROM_CONTEXT": "false",
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index f57ca358c91..2ee0c045ecd 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -53,7 +53,7 @@ DEFAULT_PYTHON_MAJOR_MINOR_VERSION_FOR_IMAGES = "3.12"
# Maps each supported Python version to the minimum Airflow version that
supports it.
# Used to filter Airflow versions incompatible with a given Python runtime.
-PYTHON_TO_MIN_AIRFLOW_MAPPING = {"3.10": "2.4.0"}
+PYTHON_TO_MIN_AIRFLOW_MAPPING = {"3.10": "v3.10.18"}
ALLOWED_ARCHITECTURES = [Architecture.X86_64, Architecture.ARM]
# Database Backends used when starting Breeze. The "none" value means that the
configuration is invalid.
@@ -783,14 +783,12 @@ PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str
| list[str]]] = [
},
]
-ALL_PYTHON_VERSION_TO_PATCH_VERSION: dict[str, str] = {
- "3.6": "v3.6.15",
- "3.7": "v3.7.17",
- "3.8": "v3.8.20",
- "3.9": "v3.9.23",
- "3.10": "v3.10.18",
- "3.11": "v3.11.13",
- "3.12": "v3.12.11",
+ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION: dict[str, str] = {
+ "3.9": "3.9.23",
+ "3.10": "3.10.18",
+ "3.11": "3.11.13",
+ "3.12": "3.12.11",
+ "3.13": "3.13.7",
}
# Number of slices for low dep tests
diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py
b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
index b4c42c341de..c1c79d79bec 100644
--- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
@@ -21,7 +21,7 @@ from dataclasses import dataclass
from pathlib import Path
from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
-from airflow_breeze.global_constants import ALL_PYTHON_VERSION_TO_PATCH_VERSION
+from airflow_breeze.global_constants import
ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION
from airflow_breeze.params.common_build_params import CommonBuildParams
from airflow_breeze.utils.path_utils import BUILD_CACHE_PATH
@@ -68,9 +68,9 @@ class BuildCiParams(CommonBuildParams):
self._opt_arg("UV_HTTP_TIMEOUT", get_uv_timeout(self))
self._req_arg("AIRFLOW_VERSION", self.airflow_version)
- self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image)
+ self._req_arg("BASE_IMAGE", self.python_base_image)
self._req_arg(
- "AIRFLOW_PYTHON_VERSION",
ALL_PYTHON_VERSION_TO_PATCH_VERSION.get(self.python, self.python)
+ "AIRFLOW_PYTHON_VERSION",
ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(self.python, self.python)
)
if self.upgrade_to_newer_dependencies:
self._opt_arg("UPGRADE_RANDOM_INDICATOR_STRING",
f"{random.randrange(2**32):x}")
diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py
b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
index 94d70ae508e..ea9822d80a6 100644
--- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
@@ -25,6 +25,7 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH,
DEFAULT_AIRFLOW_CONST
from airflow_breeze.global_constants import (
AIRFLOW_SOURCES_FROM,
AIRFLOW_SOURCES_TO,
+ ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION,
get_airflow_extras,
)
from airflow_breeze.params.common_build_params import CommonBuildParams
@@ -234,7 +235,10 @@ class BuildProdParams(CommonBuildParams):
self._req_arg("DOCKER_CONTEXT_FILES", self.docker_context_files)
self._req_arg("INSTALL_DISTRIBUTIONS_FROM_CONTEXT",
self.install_distributions_from_context)
self._req_arg("INSTALL_POSTGRES_CLIENT", self.install_postgres_client)
- self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image)
+ self._req_arg("BASE_IMAGE", self.python_base_image)
+ self._req_arg(
+ "AIRFLOW_PYTHON_VERSION",
ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION.get(self.python, self.python)
+ )
# optional build args
self._set_common_opt_args()
self._opt_arg("ADDITIONAL_RUNTIME_APT_COMMAND",
self.additional_runtime_apt_command)
diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py
b/dev/breeze/src/airflow_breeze/params/common_build_params.py
index fd20d27b364..265464ea6b5 100644
--- a/dev/breeze/src/airflow_breeze/params/common_build_params.py
+++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py
@@ -116,7 +116,7 @@ class CommonBuildParams:
"""Construct Python Base Image"""
if self.python_image is not None:
return self.python_image
- return f"python:{self.python}-slim-{self.debian_version}"
+ return f"debian:{self.debian_version}-slim"
@property
def airflow_image_repository(self):
diff --git a/docker-stack-docs/build-arg-ref.rst
b/docker-stack-docs/build-arg-ref.rst
index 0e7125d6504..ec65153d42a 100644
--- a/docker-stack-docs/build-arg-ref.rst
+++ b/docker-stack-docs/build-arg-ref.rst
@@ -30,9 +30,9 @@ Those are the most common arguments that you use when you
want to build a custom
+------------------------------------------+-------------------------------------------+---------------------------------------------+
| Build argument | Default value
| Description |
+==========================================+===========================================+=============================================+
-| ``PYTHON_BASE_IMAGE`` | ``python:3.10-slim-bookworm``
| Base python image. |
+| ``AIRFLOW_VERSION`` | :subst-code:`|airflow-version|`
| Version of Airflow. |
+------------------------------------------+-------------------------------------------+---------------------------------------------+
-| ``AIRFLOW_VERSION`` | :subst-code:`|airflow-version|`
| version of Airflow. |
+| ``AIRFLOW_PYTHON_VERSION`` | ``3.12.11``
| Version of Python. |
+------------------------------------------+-------------------------------------------+---------------------------------------------+
| ``AIRFLOW_EXTRAS`` | (see below the table)
| Default extras with which Airflow is |
| |
| installed. |
@@ -70,8 +70,8 @@ Those are the most common arguments that you use when you
want to build a custom
| ``AIRFLOW_CONSTRAINTS_REFERENCE`` |
| Reference (branch or tag) from GitHub |
| |
| where constraints file is taken from |
| |
| It can be ``constraints-main`` or |
-| |
| ``constraints-2-0`` for |
-| |
| 2.0.* installation. In case of building |
+| |
| ``constraints-3-X`` for |
+| |
| 3.X.* installation. In case of building |
| |
| specific version you want to point it |
| |
| to specific tag, for example |
| |
| :subst-code:`constraints-|airflow-version|`.|
@@ -200,13 +200,13 @@ Those parameters are useful only if you want to install
Airflow using different
(installing from PyPI packages).
This is usually only useful if you have your own fork of Airflow and want to
build the images locally from
-those sources - either locally or directly from GitHub sources. This way you
do not need to release your
-Airflow and Providers via PyPI - they can be installed directly from sources
or from GitHub repository.
+those sources - either locally or from packages. This way you do not need to
release your
+Airflow and Providers via PyPI - they can be installed directly from sources.
+
Another option of installation is to build Airflow from previously prepared
binary Python packages which might
be useful if you need to build Airflow in environments that require high
levels of security.
You can see some examples of those in:
- * :ref:`Building from GitHub<image-build-github>`,
* :ref:`Using custom installation sources<image-build-custom>`,
* :ref:`Build images in security restricted
environments<image-build-secure-environments>`
@@ -215,9 +215,7 @@ You can see some examples of those in:
+==========================================+==========================================+==========================================+
| ``AIRFLOW_INSTALLATION_METHOD`` | ``apache-airflow``
| Installation method of Apache Airflow. |
| |
| ``apache-airflow`` for installation from |
-| |
| PyPI. It can be GitHub repository URL |
-| |
| including branch or tag to install from |
-| |
| that repository or "." to install from |
+| |
| PyPI or "." to install from |
| |
| local sources. Installing from sources |
| |
| requires appropriate values of the |
| |
| ``AIRFLOW_SOURCES_FROM`` and |
@@ -263,7 +261,7 @@ You can see some examples of those in:
Caching dependencies
....................
-We are using ``--mount-type=cache`` volumes to speed up installation of
dependencies for Airflow images. Combined with uv
+We are using ``--mount-type=cache`` volumes to speed up installation of
dependencies for Airflow images. Combined with ``uv``
speed and extensive use of caching, as well as quick restoring of the cache in
CI environment, this allows us to build images
quickly - for both CI and local development purposes. The cache can be easily
invalidated by providing a new value of
``DEPENDENCY_CACHE_EPOCH`` build argument or changing it inside the Dockerfile.
diff --git a/docker-stack-docs/build.rst b/docker-stack-docs/build.rst
index 2133aa63bcb..21915e7b53f 100644
--- a/docker-stack-docs/build.rst
+++ b/docker-stack-docs/build.rst
@@ -802,49 +802,6 @@ Building images with MySQL client
:end-before: [END build]
-.. _image-build-github:
-
-
-Building from GitHub
-....................
-
-This method is usually used for development purpose. But in case you have your
own fork you can point
-it to your forked version of source code without having to release it to PyPI.
It is enough to have
-a branch or tag in your repository and use the tag or branch in the URL that
you point the installation to.
-
-In case of GitHub builds you need to pass the constraints reference manually
in case you want to use
-specific constraints, otherwise the default ``constraints-main`` is used.
-
-The following example builds the production image in version ``3.10`` with
default extras from the latest main version and
-constraints are taken from latest version of the constraints-main branch in
GitHub.
-
-.. exampleinclude:: docker-examples/customizing/github-main.sh
- :language: bash
- :start-after: [START build]
- :end-before: [END build]
-
-The following example builds the production image with default extras from the
-latest ``v2-*-test`` version and constraints are taken from the latest version
of
-the ``constraints-2-*`` branch in GitHub (for example ``v2-2-test`` branch
matches ``constraints-2-2``).
-Note that this command might fail occasionally as only the "released version"
constraints when building a
-version and "main" constraints when building main are guaranteed to work.
-
-.. exampleinclude:: docker-examples/customizing/github-v2-2-test.sh
- :language: bash
- :start-after: [START build]
- :end-before: [END build]
-
-You can also specify another repository to build from. If you also want to use
different constraints
-repository source, you must specify it as additional
``CONSTRAINTS_GITHUB_REPOSITORY`` build arg.
-
-The following example builds the production image using ``potiuk/airflow``
fork of Airflow and constraints
-are also downloaded from that repository.
-
-.. exampleinclude:: docker-examples/customizing/github-different-repository.sh
- :language: bash
- :start-after: [START build]
- :end-before: [END build]
-
.. _image-build-custom:
Using custom installation sources
diff --git a/docker-stack-docs/changelog.rst b/docker-stack-docs/changelog.rst
index 9968c77b31c..d4b054b5a18 100644
--- a/docker-stack-docs/changelog.rst
+++ b/docker-stack-docs/changelog.rst
@@ -34,6 +34,29 @@ the Airflow team.
any Airflow version from the ``Airflow 2`` line. There is no guarantee
that it will work, but if it does,
then you can use latest features from that image to build images for
previous Airflow versions.
+Airflow 3.1.0
+~~~~~~~~~~~~~
+
+In Airflow 3.1.0 we changed the base images used for building the Airflow
images. Previously the images
+were based on "official" Python images from DockerHub, however those images
sometimes lag behind the
+latest security patches and their maintainers made decisions about using older
setuptools and pip versions,
+however we want to be able to use the latest versions of those tools to build
the images. Therefore
+we decided to use ``bookworm-slim`` images from Debian as the base images for
Airflow images and we compile
+and install the latest Python version in the image based on the official
packages from the Python Software
+Foundation. This change should be transparent for those who extend the images
or use them "as is", however
+for those who want to build custom images, there are changes to the build-args
passed to the ``docker build``
+command:
+
+* The ``PYTHON_BASE_IMAGE`` arg is no longer supported - the base image is
always ``debian:bookworm-slim``
+* Instead ``AIRFLOW_PYTHON_VERSION`` arg is introduced to select the Python
version to be installed in the
+ image. It should be full Python version - for example ``3.12.7`` or
two-digit version like ``3.12``
+ (the latest released Python patch version will be installed in that case).
+* Python is now installed in ``/usr/python/`` directory not in ``/usr/local/``
as it was previously.
+ This should not affect regular uses because there are symlinks from
``/usr/local/bin/`` to
+ ``/usr/python/bin/``, and ``/usr/python/bin`` is first on the PATH, however
if the ``/usr/local/``
+ location has been hard-coded to - for example - detect ``site-packages``
dir, the folders moved to
+ the ``/usr/python`` directory.
+
Airflow 3.0.2
~~~~~~~~~~~~~
diff --git
a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
index c19be12d4e5..e2c0e55692a 100755
---
a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
+++
b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
@@ -26,14 +26,15 @@ pushd "${TEMP_DOCKER_DIR}"
cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
# [START build]
-export AIRFLOW_VERSION=2.8.2
+export AIRFLOW_VERSION=3.0.3
export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
+ --build-arg BASE_IMAGE="debian:bookworm-slim" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
- --build-arg ADDITIONAL_PYTHON_DEPS="mpi4py==3.1.6" \
+ --build-arg AIRFLOW_PYTHON_VERSION="3.12.11" \
+ --build-arg ADDITIONAL_PYTHON_DEPS="mpi4py==4.1.0" \
--build-arg ADDITIONAL_DEV_APT_DEPS="libopenmpi-dev" \
--build-arg ADDITIONAL_RUNTIME_APT_DEPS="openmpi-common" \
--tag "my-build-essential-image:0.0.1"
diff --git a/docker-stack-docs/docker-examples/customizing/custom-pip.sh
b/docker-stack-docs/docker-examples/customizing/custom-pip.sh
index 54802d7b400..be2b377c3b5 100755
--- a/docker-stack-docs/docker-examples/customizing/custom-pip.sh
+++ b/docker-stack-docs/docker-examples/customizing/custom-pip.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
diff --git a/docker-stack-docs/docker-examples/customizing/custom-sources.sh
b/docker-stack-docs/docker-examples/customizing/custom-sources.sh
index 33d7bb7cff3..4ea37142e13 100755
--- a/docker-stack-docs/docker-examples/customizing/custom-sources.sh
+++ b/docker-stack-docs/docker-examples/customizing/custom-sources.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
@@ -26,17 +26,18 @@ pushd "${TEMP_DOCKER_DIR}"
cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
# [START build]
-export AIRFLOW_VERSION=2.2.4
+export AIRFLOW_VERSION=3.0.3
export DOCKER_BUILDKIT=1
docker build . -f Dockerfile \
--pull \
--platform 'linux/amd64' \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
+ --build-arg BASE_IMAGE="debian:bookworm-slim" \
+ --build-arg AIRFLOW_PYTHON_VERSION="3.12.11" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack,odbc" \
--build-arg ADDITIONAL_PYTHON_DEPS=" \
- azure-storage-blob<12.9.0 \
+ azure-storage-blob \
oauth2client \
beautifulsoup4 \
dateparser \
diff --git
a/docker-stack-docs/docker-examples/customizing/github-different-repository.sh
b/docker-stack-docs/docker-examples/customizing/github-different-repository.sh
deleted file mode 100755
index 82a7e9870fd..00000000000
---
a/docker-stack-docs/docker-examples/customizing/github-different-repository.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This is an example docker build script. It is not intended for PRODUCTION use
-set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
-TEMP_DOCKER_DIR=$(mktemp -d)
-pushd "${TEMP_DOCKER_DIR}"
-
-cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
-
-# [START build]
-export DOCKER_BUILDKIT=1
-
-docker build . \
- --pull \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
- --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow @
https://github.com/potiuk/airflow/archive/main.tar.gz"; \
- --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-main" \
- --build-arg CONSTRAINTS_GITHUB_REPOSITORY="potiuk/airflow" \
- --tag "github-different-repository-image:0.0.1"
-# [END build]
-docker rmi --force "github-different-repository-image:0.0.1"
-popd
-rm -rf "${TEMP_DOCKER_DIR}"
diff --git a/docker-stack-docs/docker-examples/customizing/github-main.sh
b/docker-stack-docs/docker-examples/customizing/github-main.sh
deleted file mode 100755
index 8b4e55523fd..00000000000
--- a/docker-stack-docs/docker-examples/customizing/github-main.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This is an example docker build script. It is not intended for PRODUCTION use
-set -euo pipefail
-
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
-TEMP_DOCKER_DIR=$(mktemp -d)
-pushd "${TEMP_DOCKER_DIR}"
-
-cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
-
-# [START build]
-export DOCKER_BUILDKIT=1
-
-docker build . \
- --pull \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
- --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow @
https://github.com/apache/airflow/archive/main.tar.gz"; \
- --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-main" \
- --tag "my-github-main:0.0.1"
-# [END build]
-docker rmi --force "my-github-main:0.0.1"
-popd
-rm -rf "${TEMP_DOCKER_DIR}"
diff --git a/docker-stack-docs/docker-examples/customizing/github-v2-2-test.sh
b/docker-stack-docs/docker-examples/customizing/github-v2-2-test.sh
deleted file mode 100755
index 88e5bb7f55e..00000000000
--- a/docker-stack-docs/docker-examples/customizing/github-v2-2-test.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This is an example docker build script. It is not intended for PRODUCTION use
-set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
-
-TEMP_DOCKER_DIR=$(mktemp -d)
-pushd "${TEMP_DOCKER_DIR}"
-
-cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
-
-
-# [START build]
-export DOCKER_BUILDKIT=1
-
-docker build . \
- --pull \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
- --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow @
https://github.com/apache/airflow/archive/v2-2-test.tar.gz"; \
- --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-2" \
- --tag "my-github-v2-2:0.0.1"
-# [END build]
-docker rmi --force "my-github-v2-2:0.0.1"
-popd
-rm -rf "${TEMP_DOCKER_DIR}"
diff --git a/docker-stack-docs/docker-examples/customizing/mysql-client.sh
b/docker-stack-docs/docker-examples/customizing/mysql-client.sh
index 1835e7688de..3585598f363 100755
--- a/docker-stack-docs/docker-examples/customizing/mysql-client.sh
+++ b/docker-stack-docs/docker-examples/customizing/mysql-client.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
diff --git a/docker-stack-docs/docker-examples/customizing/own-requirements.sh
b/docker-stack-docs/docker-examples/customizing/own-requirements.sh
index 50293707c6e..89aa98be93c 100755
--- a/docker-stack-docs/docker-examples/customizing/own-requirements.sh
+++ b/docker-stack-docs/docker-examples/customizing/own-requirements.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
@@ -27,8 +27,8 @@ cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
mkdir -p docker-context-files
cat <<EOF >./docker-context-files/requirements.txt
-beautifulsoup4==4.10.0
-apache-airflow==2.9.0.dev0
+beautifulsoup4==4.13.4
+apache-airflow==3.0.2
EOF
export DOCKER_BUILDKIT=1
diff --git
a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
index aa9929d9721..f6faea085a9 100755
--- a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
+++ b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
@@ -27,15 +27,16 @@ cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
export DOCKER_BUILDKIT=1
# [START build]
-export AIRFLOW_VERSION=2.2.4
+export AIRFLOW_VERSION=3.0.3
export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
+ --build-arg BASE_IMAGE="debian:bookworm-slim" \
+ --build-arg AIRFLOW_PYTHON_VERSION="3.12.11" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" \
- --build-arg ADDITIONAL_PYTHON_DEPS="pandas" \
+ --build-arg ADDITIONAL_PYTHON_DEPS="pandas==2.1.2" \
--build-arg ADDITIONAL_DEV_APT_DEPS="gcc g++" \
--build-arg ADDITIONAL_RUNTIME_APT_DEPS="default-jre-headless" \
--tag "my-pypi-dev-runtime:0.0.1"
diff --git
a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
index c86b2385289..8b90cacc5cb 100755
--- a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
+++ b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
@@ -26,12 +26,13 @@ pushd "${TEMP_DOCKER_DIR}"
cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
# [START build]
-export AIRFLOW_VERSION=2.3.4
+export AIRFLOW_VERSION=3.0.3
export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
+ --build-arg BASE_IMAGE="debian:bookworm-slim" \
+ --build-arg AIRFLOW_PYTHON_VERSION="3.12.11" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="mssql,hdfs" \
--build-arg ADDITIONAL_PYTHON_DEPS="oauth2client" \
diff --git
a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
index 182b61f255e..d5660170c04 100755
--- a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
+++ b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
@@ -26,11 +26,12 @@ pushd "${TEMP_DOCKER_DIR}"
cp "${AIRFLOW_SOURCES}/Dockerfile" "${TEMP_DOCKER_DIR}"
# [START build]
-export AIRFLOW_VERSION=2.3.4
+export AIRFLOW_VERSION=3.0.3
export DOCKER_BUILDKIT=1
docker build . \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
+ --build-arg BASE_IMAGE="debian:bookworm-slim" \
+ --build-arg AIRFLOW_PYTHON_VERSION="3.12.11" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--tag "my-pypi-selected-version:0.0.1"
# [END build]
diff --git a/docker-stack-docs/docker-examples/customizing/stable-airflow.sh
b/docker-stack-docs/docker-examples/customizing/stable-airflow.sh
index d3ee6903267..e646e5a8bc9 100755
--- a/docker-stack-docs/docker-examples/customizing/stable-airflow.sh
+++ b/docker-stack-docs/docker-examples/customizing/stable-airflow.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
diff --git a/docker-stack-docs/docker-examples/customizing/use-uv.sh
b/docker-stack-docs/docker-examples/customizing/use-uv.sh
index 4c5ffdc1aad..d0e54a43edd 100755
--- a/docker-stack-docs/docker-examples/customizing/use-uv.sh
+++ b/docker-stack-docs/docker-examples/customizing/use-uv.sh
@@ -18,7 +18,7 @@
# This is an example docker build script. It is not intended for PRODUCTION use
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)"
+AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../" && pwd)"
TEMP_DOCKER_DIR=$(mktemp -d)
pushd "${TEMP_DOCKER_DIR}"
diff --git
a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
index d4e76e4ea41..d7de8aa86ed 100755
--- a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
+++ b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh
@@ -47,7 +47,8 @@ export DOCKER_BUILDKIT=1
docker build . \
--pull \
- --build-arg PYTHON_BASE_IMAGE="python:3.10-slim-bookworm" \
+ --build-arg BASE_IMAGE="debian:bookworm-slim" \
+ --build-arg AIRFLOW_PYTHON_VERSION="3.12.11" \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg INSTALL_MYSQL_CLIENT="false" \
diff --git a/docker-stack-docs/pyproject.toml b/docker-stack-docs/pyproject.toml
index b4d40bbd6b1..360c8891fb6 100644
--- a/docker-stack-docs/pyproject.toml
+++ b/docker-stack-docs/pyproject.toml
@@ -22,7 +22,7 @@ requires = [
build-backend = "hatchling.build"
[project]
-name = "apache-airflow-docker-stack"
+name = "docker-stack"
description = "Programmatically author, schedule and monitor data pipelines"
requires-python = ">=3.10"
authors = [
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index bfefa5d42f1..fd3bbe39f39 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -1967,6 +1967,7 @@ utilise
Utils
utils
uuid
+uv
V1JobList
validator
validators
diff --git a/reproducible_build.yaml b/reproducible_build.yaml
index a10719799c7..b5d7fdc65c7 100644
--- a/reproducible_build.yaml
+++ b/reproducible_build.yaml
@@ -1,2 +1,2 @@
-release-notes-hash: 23fc4b54def11ad193d424f7eebaae5a
-source-date-epoch: 1756513808
+release-notes-hash: 81e41c5508d536a674c6ebdced78aeaf
+source-date-epoch: 1756547695
diff --git a/scripts/ci/prek/upgrade_important_versions.py
b/scripts/ci/prek/upgrade_important_versions.py
index 51d9540b1bd..0f22065e078 100755
--- a/scripts/ci/prek/upgrade_important_versions.py
+++ b/scripts/ci/prek/upgrade_important_versions.py
@@ -39,6 +39,9 @@ from packaging.version import Version
sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure
common_prek_utils is imported
from common_prek_utils import AIRFLOW_CORE_ROOT_PATH, AIRFLOW_ROOT_PATH,
console
+DOCKER_IMAGES_EXAMPLE_DIR_PATH = AIRFLOW_ROOT_PATH / "docker-stack-docs" /
"docker-examples"
+
+
# List of files to update and whether to keep total length of the original
value when replacing.
FILES_TO_UPDATE: list[tuple[Path, bool]] = [
(AIRFLOW_ROOT_PATH / "Dockerfile", False),
@@ -62,12 +65,16 @@ FILES_TO_UPDATE: list[tuple[Path, bool]] = [
(AIRFLOW_ROOT_PATH / ".github" / "actions" / "install-prek" /
"action.yml", False),
(AIRFLOW_ROOT_PATH / ".github" / "workflows" / "basic-tests.yml", False),
(AIRFLOW_ROOT_PATH / "dev" / "breeze" / "doc" / "ci" / "02_images.md",
True),
+ (AIRFLOW_ROOT_PATH / "docker-stack-docs" / "build-arg-ref.rst", True),
(AIRFLOW_ROOT_PATH / "dev" / "breeze" / "pyproject.toml", False),
(AIRFLOW_ROOT_PATH / ".pre-commit-config.yaml", False),
(AIRFLOW_ROOT_PATH / ".github" / "workflows" / "ci-amd.yml", False),
(AIRFLOW_CORE_ROOT_PATH / "pyproject.toml", False),
]
+for file in DOCKER_IMAGES_EXAMPLE_DIR_PATH.rglob("*.sh"):
+ FILES_TO_UPDATE.append((file, False))
+
def get_latest_pypi_version(package_name: str) -> str:
response = requests.get(
@@ -97,7 +104,7 @@ def get_latest_python_version(python_major_minor: str,
github_token: str | None)
if versions:
latest_version = sorted(versions, key=Version, reverse=True)[0]
break
- return latest_version
+ return latest_version[1:] if latest_version and
latest_version.startswith("v") else latest_version
def get_latest_golang_version() -> str:
@@ -124,6 +131,7 @@ class Quoting(Enum):
SINGLE_QUOTED = 1
DOUBLE_QUOTED = 2
REVERSE_SINGLE_QUOTED = 3
+ REVERSE_DOUBLE_QUOTED = 4
PIP_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
@@ -136,14 +144,19 @@ PIP_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
]
PYTHON_PATTERNS: list[tuple[str, Quoting]] = [
- (r"(\"{python_major_minor}\": \")(v[0-9.]+)(\")", Quoting.UNQUOTED),
+ (r"(\"{python_major_minor}\": \")([0-9.]+)(\")", Quoting.UNQUOTED),
]
GOLANG_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
- (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
(re.compile(r"(\| *`GOLANG_MAJOR_MINOR_VERSION` *\| *)(`[0-9.]+`)( *\|)"),
Quoting.REVERSE_SINGLE_QUOTED),
]
+AIRFLOW_IMAGE_PYTHON_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
+ (re.compile(r"(AIRFLOW_PYTHON_VERSION=)(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
+ (re.compile(r"(\| ``AIRFLOW_PYTHON_VERSION`` *\| )(``[0-9.]+``)( *\|)"),
Quoting.REVERSE_DOUBLE_QUOTED),
+]
+
UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
(re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
(re.compile(r"(uv>=)([0-9.]+)"), Quoting.UNQUOTED),
@@ -160,10 +173,6 @@ UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
),
]
-SETUPTOOLS_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
- (re.compile(r"(AIRFLOW_SETUPTOOLS_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
-]
-
PREK_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
(re.compile(r"(AIRFLOW_PREK_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
(re.compile(r"(AIRFLOW_PREK_VERSION = )(\"[0-9.]+\")"),
Quoting.DOUBLE_QUOTED),
@@ -195,6 +204,8 @@ def get_replacement(value: str, quoting: Quoting) -> str:
return f"'{value}'"
if quoting == Quoting.REVERSE_SINGLE_QUOTED:
return f"`{value}`"
+ if quoting == Quoting.REVERSE_DOUBLE_QUOTED:
+ return f"``{value}``"
return value
@@ -202,7 +213,6 @@ UPGRADE_UV: bool = os.environ.get("UPGRADE_UV",
"true").lower() == "true"
UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true"
UPGRADE_PYTHON: bool = os.environ.get("UPGRADE_PYTHON", "true").lower() ==
"true"
UPGRADE_GOLANG: bool = os.environ.get("UPGRADE_GOLANG", "true").lower() ==
"true"
-UPGRADE_SETUPTOOLS: bool = os.environ.get("UPGRADE_SETUPTOOLS",
"true").lower() == "true"
UPGRADE_PREK: bool = os.environ.get("UPGRADE_PREK", "true").lower() == "true"
UPGRADE_NODE_LTS: bool = os.environ.get("UPGRADE_NODE_LTS", "true").lower() ==
"true"
UPGRADE_HATCH: bool = os.environ.get("UPGRADE_HATCH", "true").lower() == "true"
@@ -210,7 +220,8 @@ UPGRADE_PYYAML: bool = os.environ.get("UPGRADE_PYYAML",
"true").lower() == "true
UPGRADE_GITPYTHON: bool = os.environ.get("UPGRADE_GITPYTHON", "true").lower()
== "true"
UPGRADE_RICH: bool = os.environ.get("UPGRADE_RICH", "true").lower() == "true"
-ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.9", "3.10", "3.11", "3.12"]
+ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"]
+DEFAULT_PROD_IMAGE_PYTHON_VERSION = "3.12"
GITHUB_TOKEN: str | None = os.environ.get("GITHUB_TOKEN")
@@ -277,12 +288,26 @@ if __name__ == "__main__":
if UPGRADE_PYTHON:
for python_version in ALL_PYTHON_MAJOR_MINOR_VERSIONS:
latest_python_version =
get_latest_python_version(python_version, GITHUB_TOKEN)
- if latest_python_version:
+ if not latest_python_version:
+ console.print(
+ f"[red]Failed to get latest version for python
{python_version}. Skipping.[/]"
+ )
+ sys.exit(1)
+ console.print(f"[bright_blue]Latest python {python_version}
version: {latest_python_version}")
+ for line_format, quoting in PYTHON_PATTERNS:
+ line_pattern =
re.compile(line_format.format(python_major_minor=python_version))
+ console.print(line_pattern)
+ new_content = replace_version(
+ line_pattern,
+ get_replacement(latest_python_version, quoting),
+ new_content,
+ keep_length,
+ )
+ if python_version == DEFAULT_PROD_IMAGE_PYTHON_VERSION:
console.print(
- f"[bright_blue]Latest python {python_version} version:
{latest_python_version}"
+ f"[bright_blue]Latest image python {python_version}
version: {latest_python_version}"
)
- for line_format, quoting in PYTHON_PATTERNS:
- line_pattern =
re.compile(line_format.format(python_major_minor=python_version))
+ for line_pattern, quoting in AIRFLOW_IMAGE_PYTHON_PATTERNS:
console.print(line_pattern)
new_content = replace_version(
line_pattern,
@@ -296,12 +321,6 @@ if __name__ == "__main__":
new_content = replace_version(
line_pattern, get_replacement(golang_version, quoting),
new_content, keep_length
)
- if UPGRADE_SETUPTOOLS:
- console.print(f"[bright_blue]Latest setuptools version:
{setuptools_version}")
- for line_pattern, quoting in SETUPTOOLS_PATTERNS:
- new_content = replace_version(
- line_pattern, get_replacement(setuptools_version,
quoting), new_content, keep_length
- )
if UPGRADE_UV:
console.print(f"[bright_blue]Latest uv version: {uv_version}")
for line_pattern, quoting in UV_PATTERNS:
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index f061d375e61..89881cc8f7b 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -159,12 +159,6 @@ function common::install_packaging_tools() {
pip install --root-user-action ignore --disable-pip-version-check
"pip==${AIRFLOW_PIP_VERSION}"
fi
fi
- if [[ ${AIRFLOW_SETUPTOOLS_VERSION=} != "" ]]; then
- echo
- echo "${COLOR_BLUE}Installing setuptools version
${AIRFLOW_SETUPTOOLS_VERSION} {COLOR_RESET}"
- echo
- pip install --root-user-action ignore
setuptools==${AIRFLOW_SETUPTOOLS_VERSION}
- fi
if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
echo
echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
diff --git a/scripts/docker/install_airflow_when_building_images.sh
b/scripts/docker/install_airflow_when_building_images.sh
index f5db2771939..8c419f58972 100644
--- a/scripts/docker/install_airflow_when_building_images.sh
+++ b/scripts/docker/install_airflow_when_building_images.sh
@@ -134,13 +134,11 @@ function install_from_external_spec() {
local installation_command_flags
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then
installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
- elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then
- installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}] @
${AIRFLOW_VERSION_SPECIFICATION/apache-airflow @//}"
else
echo
echo "${COLOR_RED}The '${INSTALLATION_METHOD}' installation method is
not supported${COLOR_RESET}"
echo
- echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow',
'apache-airflow @ URL')${COLOR_RESET}"
+ echo "${COLOR_YELLOW}Supported methods are ('.',
'apache-airflow')${COLOR_RESET}"
echo
exit 1
fi
diff --git a/scripts/docker/install_os_dependencies.sh
b/scripts/docker/install_os_dependencies.sh
index dab59660082..3f17bdf5e89 100644
--- a/scripts/docker/install_os_dependencies.sh
+++ b/scripts/docker/install_os_dependencies.sh
@@ -19,26 +19,34 @@
set -euo pipefail
if [[ "$#" != 1 ]]; then
- echo "ERROR! There should be 'runtime' or 'dev' parameter passed as
argument.".
+ echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
exit 1
fi
+AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}
+GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
+
if [[ "${1}" == "runtime" ]]; then
INSTALLATION_TYPE="RUNTIME"
elif [[ "${1}" == "dev" ]]; then
- INSTALLATION_TYPE="dev"
+ INSTALLATION_TYPE="DEV"
+elif [[ "${1}" == "ci" ]]; then
+ INSTALLATION_TYPE="CI"
else
- echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime'
or 'dev'.".
+ echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
exit 1
fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
- DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
+ DEV_APT_DEPS="apt-transport-https apt-utils build-essential dirmngr \
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev"
+software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev wget \
+gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
+libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
+libzstd-dev"
export DEV_APT_DEPS
fi
}
@@ -57,10 +65,10 @@ function get_runtime_apt_deps() {
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
- RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
+ RUNTIME_APT_DEPS="apt-transport-https apt-utils \
curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
-lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
+lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc wget"
export RUNTIME_APT_DEPS
fi
}
@@ -83,7 +91,7 @@ function install_docker_cli() {
function install_debian_dev_dependencies() {
apt-get update
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release
ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_DEV_APT_ENV?}
if [[ ${DEV_APT_COMMAND} != "" ]]; then
@@ -105,10 +113,30 @@ function install_debian_dev_dependencies() {
apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
}
+
+function link_python() {
+ # link python binaries to /usr/local/bin and /usr/python/bin with and
without 3 suffix
+ # Links in /usr/local/bin are needed for tools that expect python to be
there
+ # Links in /usr/python/bin are needed for tools that are detecting home of
python installation including
+ # lib/site-packages. The /usr/python/bin should be first in PATH in order
to help with the last part.
+ ldconfig
+ for dst in pip3 python3 python3-config; do
+ src="$(echo "${dst}" | tr -d 3)"
+ echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
+ ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}"
+ for dir in /usr/local/bin /usr/python/bin; do
+ if [[ ! -e "${dir}/${src}" ]]; then
+ echo "Creating ${src} - > ${dst} link in ${dir}"
+ ln -sv "${dir}/${dst}" "${dir}/${src}"
+ fi
+ done
+ done
+}
+
function install_debian_runtime_dependencies() {
apt-get update
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release
ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_RUNTIME_APT_ENV?}
if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then
@@ -122,6 +150,71 @@ function install_debian_runtime_dependencies() {
apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS}
${ADDITIONAL_RUNTIME_APT_DEPS}
apt-get autoremove -yqq --purge
apt-get clean
+ link_python
+ rm -rf /var/lib/apt/lists/* /var/log/*
+}
+
+function install_python() {
+ wget -O python.tar.xz
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz";
+ wget -O python.tar.xz.asc
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc";;
+ declare -A keys=(
+ # gpg: key B26995E310250568: public key "\xc5\x81ukasz Langa (GPG
langa.pl) <[email protected]>" imported
+ # https://peps.python.org/pep-0596/#release-manager-and-crew
+ [3.9]="E3FF2839C048B25C084DEBE9B26995E310250568"
+ # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado
<[email protected]>" imported
+ # https://peps.python.org/pep-0619/#release-manager-and-crew
+ [3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D"
+ # gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado
<[email protected]>" imported
+ # https://peps.python.org/pep-0664/#release-manager-and-crew
+ [3.11]="A035C8C19219BA821ECEA86B64E628F8D684696D"
+ # gpg: key A821E680E5FA6305: public key "Thomas Wouters
<[email protected]>" imported
+ # https://peps.python.org/pep-0693/#release-manager-and-crew
+ [3.12]="7169605F62C751356D054A26A821E680E5FA6305"
+ # gpg: key A821E680E5FA6305: public key "Thomas Wouters
<[email protected]>" imported
+ # https://peps.python.org/pep-0719/#release-manager-and-crew
+ [3.13]="7169605F62C751356D054A26A821E680E5FA6305"
+ )
+ major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}"
+ echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})"
+ GNUPGHOME="$(mktemp -d)"; export GNUPGHOME;
+ gpg_key="${keys[${major_minor_version}]}"
+ echo "Using GPG key ${gpg_key}"
+ gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "${gpg_key}"
+ gpg --batch --verify python.tar.xz.asc python.tar.xz;
+ gpgconf --kill all
+ rm -rf "$GNUPGHOME" python.tar.xz.asc
+ mkdir -p /usr/src/python
+ tar --extract --directory /usr/src/python --strip-components=1 --file
python.tar.xz
+ rm python.tar.xz
+ cd /usr/src/python
+ arch="$(dpkg --print-architecture)"; arch="${arch##*-}"
+ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"
+ EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"
+ EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer
-mno-omit-leaf-frame-pointer";
+ LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
+ LDFLAGS="${LDFLAGS:--Wl},--strip-all"
+ ./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip
--build="$gnuArch" \
+ --enable-loadable-sqlite-extensions --enable-option-checking=fatal
--enable-shared --with-lto
+ make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+ "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python
+ make -s -j "$(nproc)" install
+ cd /
+ rm -rf /usr/src/python
+ find /usr/python -depth \
+ \( \
+ \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+ -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name
'libpython*.a' \) \) \
+ \) -exec rm -rf '{}' +
+ link_python
+}
+
+function install_golang() {
+ curl
"https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg
--print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
+ rm -rf /usr/local/go && tar -C /usr/local -xzf
go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
+}
+
+function apt_clean() {
+ apt-get purge -y --auto-remove -o
APT::AutoRemove::RecommendsImportant=false
rm -rf /var/lib/apt/lists/* /var/log/*
}
@@ -129,9 +222,14 @@ if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
get_runtime_apt_deps
install_debian_runtime_dependencies
install_docker_cli
-
+ apt_clean
else
get_dev_apt_deps
install_debian_dev_dependencies
+ install_python
+ if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
+ install_golang
+ fi
install_docker_cli
+ apt_clean
fi
diff --git a/scripts/docker/install_os_dependencies_ci.sh
b/scripts/docker/install_os_dependencies_ci.sh
deleted file mode 100644
index 0873ac69c8c..00000000000
--- a/scripts/docker/install_os_dependencies_ci.sh
+++ /dev/null
@@ -1,167 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck shell=bash
-set -euo pipefail
-
-if [[ "$#" != 1 ]]; then
- echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
- exit 1
-fi
-
-AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10}
-GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
-
-if [[ "${1}" == "runtime" ]]; then
- INSTALLATION_TYPE="RUNTIME"
-elif [[ "${1}" == "dev" ]]; then
- INSTALLATION_TYPE="DEV"
-elif [[ "${1}" == "ci" ]]; then
- INSTALLATION_TYPE="CI"
-else
- echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
- exit 1
-fi
-
-function get_dev_apt_deps() {
- if [[ "${DEV_APT_DEPS=}" == "" ]]; then
- DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
-freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
-libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
-libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \
-gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
-libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
-libzstd-dev"
- export DEV_APT_DEPS
- fi
-}
-
-function get_runtime_apt_deps() {
- local debian_version
- local debian_version_apt_deps
- # Get debian version without installing lsb_release
- # shellcheck disable=SC1091
- debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
- echo
- echo "DEBIAN CODENAME: ${debian_version}"
- echo
- debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
- echo
- echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
- echo
- if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
- RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
-curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
-ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
-lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
- export RUNTIME_APT_DEPS
- fi
-}
-
-function install_docker_cli() {
- apt-get update
- apt-get install ca-certificates curl
- install -m 0755 -d /etc/apt/keyrings
- curl -fsSL https://download.docker.com/linux/debian/gpg -o
/etc/apt/keyrings/docker.asc
- chmod a+r /etc/apt/keyrings/docker.asc
- # shellcheck disable=SC1091
- echo \
- "deb [arch=$(dpkg --print-architecture)
signed-by=/etc/apt/keyrings/docker.asc]
https://download.docker.com/linux/debian \
- $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
- tee /etc/apt/sources.list.d/docker.list > /dev/null
- apt-get update
- apt-get install -y --no-install-recommends docker-ce-cli
-}
-
-function install_debian_dev_dependencies() {
- apt-get update
- apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
- # shellcheck disable=SC2086
- export ${ADDITIONAL_DEV_APT_ENV?}
- if [[ ${DEV_APT_COMMAND} != "" ]]; then
- bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}"
- fi
- if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then
- bash -o pipefail -o errexit -o nounset -o nolog -c
"${ADDITIONAL_DEV_APT_COMMAND}"
- fi
- apt-get update
- local debian_version
- local debian_version_apt_deps
- # Get debian version without installing lsb_release
- # shellcheck disable=SC1091
- debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
- echo
- echo "DEBIAN CODENAME: ${debian_version}"
- echo
- # shellcheck disable=SC2086
- apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
-}
-
-function install_debian_runtime_dependencies() {
- apt-get update
- apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
- apt-get install -y --no-install-recommends curl gnupg2 lsb-release
- # shellcheck disable=SC2086
- export ${ADDITIONAL_RUNTIME_APT_ENV?}
- if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then
- bash -o pipefail -o errexit -o nounset -o nolog -c
"${RUNTIME_APT_COMMAND}"
- fi
- if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then
- bash -o pipefail -o errexit -o nounset -o nolog -c
"${ADDITIONAL_RUNTIME_APT_COMMAND}"
- fi
- apt-get update
- # shellcheck disable=SC2086
- apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS}
${ADDITIONAL_RUNTIME_APT_DEPS}
- apt-get autoremove -yqq --purge
- apt-get clean
- rm -rf /var/lib/apt/lists/* /var/log/*
-}
-
-function install_python() {
- git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1
https://github.com/python/cpython.git
- cd cpython
- ./configure --enable-optimizations
- make -s -j "$(nproc)" all
- make -s -j "$(nproc)" install
- ln -s /usr/local/bin/python3 /usr/local/bin/python
- ln -s /usr/local/bin/pip3 /usr/local/bin/pip
- cd ..
- rm -rf cpython
-}
-
-function install_golang() {
- curl
"https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg
--print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
- rm -rf /usr/local/go && tar -C /usr/local -xzf
go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
-}
-
-if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
- get_runtime_apt_deps
- install_debian_runtime_dependencies
- install_docker_cli
-
-else
-
- get_dev_apt_deps
- install_debian_dev_dependencies
- install_python
- if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
- install_golang
- fi
- install_docker_cli
-fi
diff --git a/scripts/in_container/install_airflow_and_providers.py
b/scripts/in_container/install_airflow_and_providers.py
index 758f8d0be8d..b61b3f39aee 100755
--- a/scripts/in_container/install_airflow_and_providers.py
+++ b/scripts/in_container/install_airflow_and_providers.py
@@ -603,6 +603,17 @@ def install_airflow_and_providers(
use_distributions_from_dist: bool,
install_airflow_with_constraints: bool,
):
+ if mount_sources in ["tests", "remove"]:
+ console.print("[bright_blue]Uninstall editable packages installed in
CI image")
+ command = [
+ "uv pip freeze | grep -v '@ file://' | grep '-e file' | sed s'/-e
//' | xargs -r uv pip uninstall"
+ ]
+ run_command(
+ command,
+ github_actions=github_actions,
+ shell=True,
+ check=False,
+ )
console.print("[bright_blue]Installing Airflow and Providers")
installation_spec = find_installation_spec(
airflow_constraints_mode=airflow_constraints_mode,
@@ -629,17 +640,6 @@ def install_airflow_and_providers(
_install_airflow_ctl_with_constraints(installation_spec,
github_actions)
if installation_spec.provider_distributions or not
install_airflow_with_constraints:
_install_airflow_and_optionally_providers_together(installation_spec,
github_actions)
- if mount_sources in ["tests", "remove"]:
- console.print("[bright_blue]Uninstall editable packages installed in
CI image")
- command = [
- "uv pip freeze | grep -v '@ file://' | grep '-e file' | sed s'/-e
//' | xargs -r uv pip uninstall"
- ]
- run_command(
- command,
- github_actions=github_actions,
- shell=True,
- check=False,
- )
if mount_sources == "providers-and-tests":
if (
use_airflow_version
@@ -716,7 +716,7 @@ def _install_airflow_and_optionally_providers_together(
):
console.print("[bright_blue]Installing airflow and optionally providers
together")
base_install_cmd = [
- "/usr/local/bin/uv",
+ "uv",
"pip",
"install",
]
@@ -772,7 +772,7 @@ def
_install_airflow_ctl_with_constraints(installation_spec: InstallationSpec, g
f"{installation_spec.airflow_ctl_distribution} with constraints"
)
base_install_airflow_ctl_cmd = [
- "/usr/local/bin/uv",
+ "uv",
"pip",
"install",
]
@@ -812,7 +812,7 @@ def
_install_only_airflow_airflow_core_task_sdk_with_constraints(
installation_spec.airflow_ctl_constraints_location,
)
base_install_airflow_cmd = [
- "/usr/local/bin/uv",
+ "uv",
"pip",
"install",
]
diff --git a/scripts/in_container/install_airflow_python_client.py
b/scripts/in_container/install_airflow_python_client.py
index d3e62ce152d..36657f8768a 100644
--- a/scripts/in_container/install_airflow_python_client.py
+++ b/scripts/in_container/install_airflow_python_client.py
@@ -79,7 +79,7 @@ def install_airflow_python_client(
extension = "whl" if distribution_format == "wheel" else "tar.gz"
install_airflow_python_client_cmd = [
- "/usr/local/bin/uv",
+ "uv",
"pip",
"install",
find_airflow_python_client(extension),
diff --git a/scripts/in_container/run_check_imports_in_providers.py
b/scripts/in_container/run_check_imports_in_providers.py
index 974a9354a08..9a0ae942e10 100755
--- a/scripts/in_container/run_check_imports_in_providers.py
+++ b/scripts/in_container/run_check_imports_in_providers.py
@@ -36,7 +36,7 @@ def check_imports():
"analyze",
"graph",
"--python",
- "/usr/local/bin/python",
+ "/usr/python/bin/python",
]
console.print("Cmd", cmd)
import_tree_str = subprocess.check_output(cmd)
