This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 04278f1e206 Install python from source again (#53150) (#53202)
04278f1e206 is described below
commit 04278f1e206fe5cb3dcfe6f7fc9f85e2ede7566a
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Jul 12 07:35:17 2025 +0200
Install python from source again (#53150) (#53202)
* Draft: Build python from source
Builds python from source, also installs
golang from official distribution. Does
both of these for the ci image only.
* Updates path
* Adds version upgrade check for python version
Adds support for using the airflow api to fetch
the newest python patch version available for specific
major_minor pair
* Updated to use args in dockerfile for python
* Added support for golang upgrade
* Fixed go version sorting in pre_commit install
* Added github token usage and fixed version regex
Updated python fetch request during upgrade to use
github token and fixed the regex
* Updated dockerfile.ci file
* Added support for multiple python versions
Adds python version from global consts into
build args now for the docker ci.
* Updated python install
* Increases timeout for ci image build
(cherry picked from commit 5aec2d5ea87bf110e384187851acfb9a856da38f)
Co-authored-by: Aritra Basu <[email protected]>
---
.dockerignore | 1 +
.github/workflows/additional-ci-image-checks.yml | 2 +-
.github/workflows/basic-tests.yml | 6 +
.pre-commit-config.yaml | 2 +-
Dockerfile.ci | 67 +++++++--
dev/breeze/src/airflow_breeze/global_constants.py | 10 ++
.../src/airflow_breeze/params/build_ci_params.py | 4 +
.../pre_commit/update_installers_and_pre_commit.py | 68 +++++++++
scripts/docker/entrypoint_ci.sh | 2 +-
scripts/docker/install_os_dependencies_ci.sh | 167 +++++++++++++++++++++
10 files changed, 310 insertions(+), 19 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index c50ed5ae24e..75e6291445a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -38,6 +38,7 @@
!providers/
!task-sdk/
!airflow-ctl/
+!go-sdk/
# Add all "test" distributions
!tests
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 024f4a4ea0a..5709e658bfd 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -150,4 +150,4 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor"
--password-stdin
- name: "Check that image builds quickly"
- run: breeze shell --max-time 600 --platform "${PLATFORM}"
+ run: breeze shell --max-time 900 --platform "${PLATFORM}"
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index bc23674f442..24c0e8eb02e 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -293,6 +293,9 @@ jobs:
--hook-stage manual update-installers-and-pre-commit || true
if: always()
env:
+ UPGRADE_UV: "true"
+ UPGRADE_PYTHON: "false"
+ UPGRADE_GOLANG: "true"
UPGRADE_PIP: "false"
UPGRADE_PRE_COMMIT: "false"
UPGRADE_NODE_LTS: "false"
@@ -303,6 +306,9 @@ jobs:
--hook-stage manual update-installers-and-pre-commit
if: always()
env:
+ UPGRADE_UV: "false"
+ UPGRADE_PYTHON: "true"
+ UPGRADE_GOLANG: "false"
UPGRADE_PIP: "true"
UPGRADE_PRE_COMMIT: "true"
UPGRADE_NODE_LTS: "true"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3adeefd79b6..bdafd8421e5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -204,7 +204,7 @@ repos:
files:
^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/update_installers_and_pre_commit\.py$
pass_filenames: false
require_serial: true
- additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4',
'requests>=2.31.0']
+ additional_dependencies: ['pyyaml>=6.0.2', 'rich>=12.4.4',
'requests>=2.31.0',"packaging>=25"]
- id: update-chart-dependencies
name: Update chart dependencies to latest (manual)
entry: ./scripts/ci/pre_commit/update_chart_dependencies.py
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 066f5a98f55..82f370d7389 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -16,13 +16,13 @@
#
# WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
#
-ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
+ARG BASE_IMAGE="debian:bookworm-slim"
##############################################################################################
# This is the script image where we keep all inlined bash scripts needed in
other segments
-# We use PYTHON_BASE_IMAGE to make sure that the scripts are different for
different platforms.
+# We use BASE_IMAGE to make sure that the scripts are different for different
platforms.
##############################################################################################
-FROM ${PYTHON_BASE_IMAGE} as scripts
+FROM ${BASE_IMAGE} as scripts
##############################################################################################
# Please DO NOT modify the inlined scripts manually. The content of those
files will be
@@ -31,22 +31,27 @@ FROM ${PYTHON_BASE_IMAGE} as scripts
# make the PROD Dockerfile standalone
##############################################################################################
-# The content below is automatically copied from
scripts/docker/install_os_dependencies.sh
-COPY <<"EOF" /install_os_dependencies.sh
+# The content below is automatically copied from
scripts/docker/install_os_dependencies_ci.sh
+COPY <<"EOF" /install_os_dependencies_ci.sh
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" != 1 ]]; then
- echo "ERROR! There should be 'runtime' or 'dev' parameter passed as
argument.".
+ echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
exit 1
fi
+AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10}
+GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
+
if [[ "${1}" == "runtime" ]]; then
INSTALLATION_TYPE="RUNTIME"
elif [[ "${1}" == "dev" ]]; then
- INSTALLATION_TYPE="dev"
+ INSTALLATION_TYPE="DEV"
+elif [[ "${1}" == "ci" ]]; then
+ INSTALLATION_TYPE="CI"
else
- echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime'
or 'dev'.".
+ echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
exit 1
fi
@@ -56,7 +61,10 @@ function get_dev_apt_deps() {
freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
-software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev"
+software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \
+gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
+libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
+libzstd-dev"
export DEV_APT_DEPS
fi
}
@@ -143,14 +151,36 @@ function install_debian_runtime_dependencies() {
rm -rf /var/lib/apt/lists/* /var/log/*
}
+function install_python() {
+ git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1
https://github.com/python/cpython.git
+ cd cpython
+ ./configure --enable-optimizations
+ make -s -j "$(nproc)" all
+ make -s -j "$(nproc)" install
+ ln -s /usr/local/bin/python3 /usr/local/bin/python
+ ln -s /usr/local/bin/pip3 /usr/local/bin/pip
+ cd ..
+ rm -rf cpython
+}
+
+function install_golang() {
+ curl
"https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg
--print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
+ rm -rf /usr/local/go && tar -C /usr/local -xzf
go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
+}
+
if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
get_runtime_apt_deps
install_debian_runtime_dependencies
install_docker_cli
else
+
get_dev_apt_deps
install_debian_dev_dependencies
+ install_python
+ if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
+ install_golang
+ fi
install_docker_cli
fi
EOF
@@ -936,7 +966,7 @@ function environment_initialization() {
CI=${CI:="false"}
# Added to have run-tests on path
- export PATH=${PATH}:${AIRFLOW_SOURCES}
+ export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/
mkdir -pv "${AIRFLOW_HOME}/logs/"
@@ -1240,13 +1270,13 @@ COPY <<"EOF" /entrypoint_exec.sh
exec /bin/bash "${@}"
EOF
-FROM ${PYTHON_BASE_IMAGE} as main
+FROM ${BASE_IMAGE} as main
# Nolog bash flag is currently ignored - but you can replace it with other
flags (for example
# xtrace - to show commands executed)
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o",
"nolog", "-c"]
-ARG PYTHON_BASE_IMAGE
+ARG BASE_IMAGE
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow";
# By increasing this number we can do force build of all dependencies.
@@ -1256,7 +1286,7 @@ ARG
AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow";
ARG DEPENDENCIES_EPOCH_NUMBER="15"
# Make sure noninteractive debian install is used and language variables set
-ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
+ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
@@ -1267,7 +1297,7 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
UV_CACHE_DIR=/root/.cache/uv
-RUN echo "Base image version: ${PYTHON_BASE_IMAGE}"
+RUN echo "Base image version: ${BASE_IMAGE}"
ARG DEV_APT_COMMAND=""
ARG ADDITIONAL_DEV_APT_COMMAND=""
@@ -1282,8 +1312,13 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}
-COPY --from=scripts install_os_dependencies.sh /scripts/docker/
-RUN bash /scripts/docker/install_os_dependencies.sh dev
+ARG AIRFLOW_PYTHON_VERSION=v3.10.18
+ENV AIRFLOW_PYTHON_VERSION=$AIRFLOW_PYTHON_VERSION
+ENV GOLANG_MAJOR_MINOR_VERSION=1.24.4
+
+COPY --from=scripts install_os_dependencies_ci.sh /scripts/docker/
+
+RUN bash /scripts/docker/install_os_dependencies_ci.sh ci
COPY --from=scripts common.sh /scripts/docker/
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 7e0f5fff5f9..c680ddca5a8 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -753,6 +753,16 @@ PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str |
list[str]]] = [
},
]
+ALL_PYTHON_VERSION_TO_PATCH_VERSION: dict[str, str] = {
+ "3.6": "v3.6.1",
+ "3.7": "v3.7.1",
+ "3.8": "v3.8.1",
+ "3.9": "v3.9.23",
+ "3.10": "v3.10.18",
+ "3.11": "v3.11.13",
+ "3.12": "v3.12.11",
+}
+
# Number of slices for low dep tests
NUMBER_OF_LOW_DEP_SLICES = 5
diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py
b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
index 65adcf6efde..b4c42c341de 100644
--- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
@@ -21,6 +21,7 @@ from dataclasses import dataclass
from pathlib import Path
from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
+from airflow_breeze.global_constants import ALL_PYTHON_VERSION_TO_PATCH_VERSION
from airflow_breeze.params.common_build_params import CommonBuildParams
from airflow_breeze.utils.path_utils import BUILD_CACHE_PATH
@@ -68,6 +69,9 @@ class BuildCiParams(CommonBuildParams):
self._opt_arg("UV_HTTP_TIMEOUT", get_uv_timeout(self))
self._req_arg("AIRFLOW_VERSION", self.airflow_version)
self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image)
+ self._req_arg(
+ "AIRFLOW_PYTHON_VERSION",
ALL_PYTHON_VERSION_TO_PATCH_VERSION.get(self.python, self.python)
+ )
if self.upgrade_to_newer_dependencies:
self._opt_arg("UPGRADE_RANDOM_INDICATOR_STRING",
f"{random.randrange(2**32):x}")
# optional build args
diff --git a/scripts/ci/pre_commit/update_installers_and_pre_commit.py
b/scripts/ci/pre_commit/update_installers_and_pre_commit.py
index 612acdaaa2f..eed555e93fe 100755
--- a/scripts/ci/pre_commit/update_installers_and_pre_commit.py
+++ b/scripts/ci/pre_commit/update_installers_and_pre_commit.py
@@ -24,6 +24,7 @@ from enum import Enum
from pathlib import Path
import requests
+from packaging.version import Version
sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure
common_precommit_utils is imported
from common_precommit_utils import AIRFLOW_CORE_ROOT_PATH, AIRFLOW_ROOT_PATH,
console
@@ -65,6 +66,35 @@ def get_latest_pypi_version(package_name: str) -> str:
return latest_version
+def get_latest_python_version(python_major_minor: str, github_token: str |
None) -> str | None:
+ latest_version = None
+ # Matches versions of vA.B.C and vA.B where C can only be numeric and v is
optional
+ version_match = re.compile(rf"^v?{python_major_minor}\.?\d*$")
+ headers = {"User-Agent": "Python requests"}
+ if github_token:
+ headers["Authorization"] = f"Bearer {github_token}"
+ for i in range(5):
+ response = requests.get(
+
f"https://api.github.com/repos/python/cpython/tags?per_page=100&page={i + 1}",
+ headers=headers,
+ )
+ response.raise_for_status() # Ensure we got a successful response
+ data = response.json()
+ versions = [str(tag["name"]) for tag in data if
version_match.match(tag.get("name", ""))]
+ if versions:
+ latest_version = sorted(versions, key=Version, reverse=True)[0]
+ break
+ return latest_version
+
+
+def get_latest_golang_version() -> str:
+ response = requests.get("https://go.dev/dl/?mode=json";)
+ response.raise_for_status() # Ensure we got a successful response
+ versions = response.json()
+ stable_versions = [release["version"].replace("go", "") for release in
versions if release["stable"]]
+ return sorted(stable_versions, key=Version, reverse=True)[0]
+
+
def get_latest_lts_node_version() -> str:
response = requests.get("https://nodejs.org/dist/index.json";)
response.raise_for_status() # Ensure we got a successful response
@@ -92,6 +122,15 @@ PIP_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
(re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"),
Quoting.REVERSE_SINGLE_QUOTED),
]
+PYTHON_PATTERNS: list[tuple[str, Quoting]] = [
+ (r"(\"{python_major_minor}\": \")(v[0-9.]+)(\")", Quoting.UNQUOTED),
+]
+
+GOLANG_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
+ (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
+ (re.compile(r"(\| *`GOLANG_MAJOR_MINOR_VERSION` *\| *)(`[0-9.]+`)( *\|)"),
Quoting.REVERSE_SINGLE_QUOTED),
+]
+
UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [
(re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED),
(re.compile(r"(uv>=)([0-9.]+)"), Quoting.UNQUOTED),
@@ -167,6 +206,8 @@ def get_replacement(value: str, quoting: Quoting) -> str:
UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true"
UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true"
+UPGRADE_PYTHON: bool = os.environ.get("UPGRADE_PYTHON", "true").lower() ==
"true"
+UPGRADE_GOLANG: bool = os.environ.get("UPGRADE_GOLANG", "true").lower() ==
"true"
UPGRADE_SETUPTOOLS: bool = os.environ.get("UPGRADE_SETUPTOOLS",
"true").lower() == "true"
UPGRADE_PRE_COMMIT: bool = os.environ.get("UPGRADE_PRE_COMMIT",
"true").lower() == "true"
UPGRADE_NODE_LTS: bool = os.environ.get("UPGRADE_NODE_LTS", "true").lower() ==
"true"
@@ -175,6 +216,10 @@ UPGRADE_PYYAML: bool = os.environ.get("UPGRADE_PYYAML",
"true").lower() == "true
UPGRADE_GITPYTHON: bool = os.environ.get("UPGRADE_GITPYTHON", "true").lower()
== "true"
UPGRADE_RICH: bool = os.environ.get("UPGRADE_RICH", "true").lower() == "true"
+ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11",
"3.12"]
+
+GITHUB_TOKEN: str | None = os.environ.get("GITHUB_TOKEN")
+
def replace_version(pattern: re.Pattern[str], version: str, text: str,
keep_total_length: bool = True) -> str:
# Assume that the pattern has up to 3 replacement groups:
@@ -205,6 +250,7 @@ def replace_version(pattern: re.Pattern[str], version: str,
text: str, keep_tota
if __name__ == "__main__":
changed = False
+ golang_version = get_latest_golang_version()
pip_version = get_latest_pypi_version("pip")
uv_version = get_latest_pypi_version("uv")
setuptools_version = get_latest_pypi_version("setuptools")
@@ -225,6 +271,28 @@ if __name__ == "__main__":
new_content = replace_version(
line_pattern, get_replacement(pip_version, quoting),
new_content, keep_length
)
+ if UPGRADE_PYTHON:
+ for python_version in ALL_PYTHON_MAJOR_MINOR_VERSIONS:
+ latest_python_version =
get_latest_python_version(python_version, GITHUB_TOKEN)
+ if latest_python_version:
+ console.print(
+ f"[bright_blue]Latest python {python_version} version:
{latest_python_version}"
+ )
+ for line_format, quoting in PYTHON_PATTERNS:
+ line_pattern =
re.compile(line_format.format(python_major_minor=python_version))
+ console.print(line_pattern)
+ new_content = replace_version(
+ line_pattern,
+ get_replacement(latest_python_version, quoting),
+ new_content,
+ keep_length,
+ )
+ if UPGRADE_GOLANG:
+ console.print(f"[bright_blue]Latest golang version:
{golang_version}")
+ for line_pattern, quoting in GOLANG_PATTERNS:
+ new_content = replace_version(
+ line_pattern, get_replacement(golang_version, quoting),
new_content, keep_length
+ )
if UPGRADE_SETUPTOOLS:
console.print(f"[bright_blue]Latest setuptools version:
{setuptools_version}")
for line_pattern, quoting in SETUPTOOLS_PATTERNS:
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index b8a4e994ceb..831be7e241f 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -130,7 +130,7 @@ function environment_initialization() {
CI=${CI:="false"}
# Added to have run-tests on path
- export PATH=${PATH}:${AIRFLOW_SOURCES}
+ export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/
mkdir -pv "${AIRFLOW_HOME}/logs/"
diff --git a/scripts/docker/install_os_dependencies_ci.sh
b/scripts/docker/install_os_dependencies_ci.sh
new file mode 100644
index 00000000000..0873ac69c8c
--- /dev/null
+++ b/scripts/docker/install_os_dependencies_ci.sh
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck shell=bash
+set -euo pipefail
+
+if [[ "$#" != 1 ]]; then
+ echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as
argument.".
+ exit 1
+fi
+
+AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-v3.10.10}
+GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
+
+if [[ "${1}" == "runtime" ]]; then
+ INSTALLATION_TYPE="RUNTIME"
+elif [[ "${1}" == "dev" ]]; then
+ INSTALLATION_TYPE="DEV"
+elif [[ "${1}" == "ci" ]]; then
+ INSTALLATION_TYPE="CI"
+else
+ echo "ERROR! Wrong argument. Passed ${1} and it should be one of
'runtime', 'ci' or 'dev'.".
+ exit 1
+fi
+
+function get_dev_apt_deps() {
+ if [[ "${DEV_APT_DEPS=}" == "" ]]; then
+ DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
+freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
+libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
+libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
+software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev \
+gdb lcov pkg-config libbz2-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
+libncurses5-dev libreadline6-dev libsqlite3-dev lzma lzma-dev tk-dev uuid-dev \
+libzstd-dev"
+ export DEV_APT_DEPS
+ fi
+}
+
+function get_runtime_apt_deps() {
+ local debian_version
+ local debian_version_apt_deps
+ # Get debian version without installing lsb_release
+ # shellcheck disable=SC1091
+ debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
+ echo
+ echo "DEBIAN CODENAME: ${debian_version}"
+ echo
+ debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
+ echo
+ echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION:
${debian_version}"
+ echo
+ if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
+ RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
+curl dumb-init freetds-bin git krb5-user libev4 libgeos-dev \
+ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
+lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
+ export RUNTIME_APT_DEPS
+ fi
+}
+
+function install_docker_cli() {
+ apt-get update
+ apt-get install ca-certificates curl
+ install -m 0755 -d /etc/apt/keyrings
+ curl -fsSL https://download.docker.com/linux/debian/gpg -o
/etc/apt/keyrings/docker.asc
+ chmod a+r /etc/apt/keyrings/docker.asc
+ # shellcheck disable=SC1091
+ echo \
+ "deb [arch=$(dpkg --print-architecture)
signed-by=/etc/apt/keyrings/docker.asc]
https://download.docker.com/linux/debian \
+ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+ tee /etc/apt/sources.list.d/docker.list > /dev/null
+ apt-get update
+ apt-get install -y --no-install-recommends docker-ce-cli
+}
+
+function install_debian_dev_dependencies() {
+ apt-get update
+ apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
+ apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ # shellcheck disable=SC2086
+ export ${ADDITIONAL_DEV_APT_ENV?}
+ if [[ ${DEV_APT_COMMAND} != "" ]]; then
+ bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}"
+ fi
+ if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then
+ bash -o pipefail -o errexit -o nounset -o nolog -c
"${ADDITIONAL_DEV_APT_COMMAND}"
+ fi
+ apt-get update
+ local debian_version
+ local debian_version_apt_deps
+ # Get debian version without installing lsb_release
+ # shellcheck disable=SC1091
+ debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
+ echo
+ echo "DEBIAN CODENAME: ${debian_version}"
+ echo
+ # shellcheck disable=SC2086
+ apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
${ADDITIONAL_DEV_APT_DEPS}
+}
+
+function install_debian_runtime_dependencies() {
+ apt-get update
+ apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
+ apt-get install -y --no-install-recommends curl gnupg2 lsb-release
+ # shellcheck disable=SC2086
+ export ${ADDITIONAL_RUNTIME_APT_ENV?}
+ if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then
+ bash -o pipefail -o errexit -o nounset -o nolog -c
"${RUNTIME_APT_COMMAND}"
+ fi
+ if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then
+ bash -o pipefail -o errexit -o nounset -o nolog -c
"${ADDITIONAL_RUNTIME_APT_COMMAND}"
+ fi
+ apt-get update
+ # shellcheck disable=SC2086
+ apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS}
${ADDITIONAL_RUNTIME_APT_DEPS}
+ apt-get autoremove -yqq --purge
+ apt-get clean
+ rm -rf /var/lib/apt/lists/* /var/log/*
+}
+
+function install_python() {
+ git clone --branch "${AIRFLOW_PYTHON_VERSION}" --depth 1
https://github.com/python/cpython.git
+ cd cpython
+ ./configure --enable-optimizations
+ make -s -j "$(nproc)" all
+ make -s -j "$(nproc)" install
+ ln -s /usr/local/bin/python3 /usr/local/bin/python
+ ln -s /usr/local/bin/pip3 /usr/local/bin/pip
+ cd ..
+ rm -rf cpython
+}
+
+function install_golang() {
+ curl
"https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg
--print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
+ rm -rf /usr/local/go && tar -C /usr/local -xzf
go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
+}
+
+if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
+ get_runtime_apt_deps
+ install_debian_runtime_dependencies
+ install_docker_cli
+
+else
+
+ get_dev_apt_deps
+ install_debian_dev_dependencies
+ install_python
+ if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
+ install_golang
+ fi
+ install_docker_cli
+fi
