This is an automated email from the ASF dual-hosted git repository. vatsrahul1001 pushed a commit to branch v3-4-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4826de34b8e7bf1ad0baab4274bd02b2e1b4262c Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Oct 31 21:02:28 2026 +0100 [v3-4-test] Reuse locked CI dependencies across workspace source changes (#74466) (#76177) * Reuse locked CI dependencies across workspace source changes * Skip the locked dependency pre-install when uv.lock does not fit A frozen sync fails when uv.lock is missing or lacks a workspace member, which would fail the whole image build even though the full sync after COPY already re-resolves in that case. * Run the locked dependency pre-install through the install script The pre-install repeated the uv sync flags of install_from_sources, so the two layers could drift apart and install different builds of the same packages. INSTALL_LOCKED_THIRD_PARTY_ONLY makes the install script add --no-install-workspace, skip the re-resolving fallback and skip the steps that need workspace packages, so both layers run the same uv sync. The Dockerfile.ci step stays best-effort and is still skipped for upgrade builds. * Skip the locked dependency pre-install only when uv.lock does not fit A frozen dry run now decides whether to skip: it fails when uv.lock is missing or lacks a workspace member, and only then is the warm-up skipped. Any other failure, such as a network error or an lxml/xmlsec build error, would fail the full sync as well, so it now fails the build instead of leaving an empty layer in the cache. The install script also refuses the locked mode for upgrade builds itself instead of relying on the caller. --------- (cherry picked from commit 9b114c8deeeb0ae02d462cf2464350ad09af35fa) Generated-by: Claude Opus 5 Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: Shahar Epstein <[email protected]> --- Dockerfile | 43 ++++++++++--- Dockerfile.ci | 71 +++++++++++++++++++--- .../docker/install_airflow_when_building_images.sh | 48 ++++++++++++--- 3 files changed, 140 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2626641d460..bb5b1b35e2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1540,6 +1540,15 @@ function install_from_sources() { if [[ ${VIRTUAL_ENV=} != "" ]]; then extra_sync_flags="--active" fi + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then + echo + echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + extra_sync_flags="${extra_sync_flags} --no-install-workspace" + fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then set +x @@ -1560,19 +1569,35 @@ function install_from_sources() { ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python else - set +x - echo - echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" - echo # Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving. # --no-binary-package is needed in order to avoid libxml and xmlsec using different version of # libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 + # shellcheck disable=SC2206 # extra_sync_flags holds several flags + local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image + ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec + --no-python-downloads --no-managed-python) + set +x + # Without the workspace sources a lock that does not match the manifests cannot be re-resolved; + # the installation that adds the workspace packages does that later. + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then + echo + echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + echo + echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" + echo set -x - if ! uv sync --all-packages --frozen --group ci-image \ - ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ - --no-python-downloads --no-managed-python; then + if ! "${frozen_sync[@]}"; then set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + echo + echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}" + echo + exit 1 + fi if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}" @@ -1662,6 +1687,10 @@ function install_airflow_when_building_images() { install_from_external_spec fi set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + # Packaging tools and 'pip check' are handled by the installation that adds the workspace packages + return + fi common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" diff --git a/Dockerfile.ci b/Dockerfile.ci index da16c347219..3a0d8758d56 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1233,6 +1233,15 @@ function install_from_sources() { if [[ ${VIRTUAL_ENV=} != "" ]]; then extra_sync_flags="--active" fi + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then + echo + echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + extra_sync_flags="${extra_sync_flags} --no-install-workspace" + fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then set +x @@ -1253,19 +1262,35 @@ function install_from_sources() { ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python else - set +x - echo - echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" - echo # Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving. # --no-binary-package is needed in order to avoid libxml and xmlsec using different version of # libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 + # shellcheck disable=SC2206 # extra_sync_flags holds several flags + local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image + ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec + --no-python-downloads --no-managed-python) + set +x + # Without the workspace sources a lock that does not match the manifests cannot be re-resolved; + # the installation that adds the workspace packages does that later. + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then + echo + echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + echo + echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" + echo set -x - if ! uv sync --all-packages --frozen --group ci-image \ - ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ - --no-python-downloads --no-managed-python; then + if ! "${frozen_sync[@]}"; then set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + echo + echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}" + echo + exit 1 + fi if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}" @@ -1355,6 +1380,10 @@ function install_airflow_when_building_images() { install_from_external_spec fi set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + # Packaging tools and 'pip check' are handled by the installation that adds the workspace packages + return + fi common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -1992,6 +2021,22 @@ fi exec /bin/bash "${@}" EOF +# Extract manifests separately so COPY below depends on their contents, not all source files. +FROM ${BASE_IMAGE} as dependency-manifests +WORKDIR /source +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] +RUN --mount=type=bind,target=/source \ + manifest_list="$(mktemp)" && \ + trap 'rm -f "${manifest_list}"' EXIT && \ + find . -name pyproject.toml -type f -print0 > "${manifest_list}" && \ + mkdir -p /dependency-manifests && \ + while IFS= read -r -d '' manifest; do \ + relative_path="${manifest#./}"; \ + mkdir -p "/dependency-manifests/$(dirname "${relative_path}")" || exit 1; \ + cp "${manifest}" "/dependency-manifests/${relative_path}" || exit 1; \ + done < "${manifest_list}" && \ + if [ -f uv.lock ]; then cp uv.lock /dependency-manifests/ || exit 1; fi + FROM ${BASE_IMAGE} as main # Nolog bash flag is currently ignored - but you can replace it with other flags (for example @@ -2233,13 +2278,23 @@ RUN bash /scripts/docker/install_packaging_tools.sh COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/ +COPY --from=dependency-manifests /dependency-manifests/ ${AIRFLOW_SOURCES}/ +ARG UPGRADE_RANDOM_INDICATOR_STRING="" +# Only preinstall locked third-party dependencies, with the same uv sync as the full installation. +# A uv.lock that does not match the manifests only skips this warm-up: the full sync after COPY still +# installs workspace packages from the current checkout and retains its resolution fallback. Any other +# failure would fail the full sync as well, so it fails the build here. +RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \ + if [[ -z "${UPGRADE_RANDOM_INDICATOR_STRING}" ]]; then \ + INSTALL_LOCKED_THIRD_PARTY_ONLY="true" bash /scripts/docker/install_airflow_when_building_images.sh; \ + fi + # We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not # copying over stuff that is accidentally generated or that we do not need (such as egg-info) # if you want to add something that is missing and you expect to see it in the image you can # add it with ! in .dockerignore next to the airflow, test etc. directories there COPY . ${AIRFLOW_SOURCES}/ -ARG UPGRADE_RANDOM_INDICATOR_STRING="" ARG VERSION_SUFFIX="" ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \ diff --git a/scripts/docker/install_airflow_when_building_images.sh b/scripts/docker/install_airflow_when_building_images.sh index 772465678a8..45e60d5fdbf 100644 --- a/scripts/docker/install_airflow_when_building_images.sh +++ b/scripts/docker/install_airflow_when_building_images.sh @@ -30,6 +30,11 @@ # UPGRADE_RANDOM_INDICATOR_STRING - if set with random value determines that dependency upgrade should be done # with highest resolution. It is random in order to trigger cache # invalidation when building images +# INSTALL_LOCKED_THIRD_PARTY_ONLY - if "true", only the third-party dependencies locked in uv.lock are installed +# from sources (--no-install-workspace), so only the pyproject.toml files +# and uv.lock are needed. When a frozen dry run shows that uv.lock does not +# match them, or for upgrade builds, nothing is installed; any other +# failure fails. The steps that need the workspace packages are skipped. # # shellcheck shell=bash disable=SC2086 # shellcheck source=scripts/docker/common.sh @@ -41,6 +46,15 @@ function install_from_sources() { if [[ ${VIRTUAL_ENV=} != "" ]]; then extra_sync_flags="--active" fi + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then + echo + echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + extra_sync_flags="${extra_sync_flags} --no-install-workspace" + fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then set +x @@ -61,19 +75,35 @@ function install_from_sources() { ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python else - set +x - echo - echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" - echo # Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving. # --no-binary-package is needed in order to avoid libxml and xmlsec using different version of # libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 + # shellcheck disable=SC2206 # extra_sync_flags holds several flags + local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image + ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec + --no-python-downloads --no-managed-python) + set +x + # Without the workspace sources a lock that does not match the manifests cannot be re-resolved; + # the installation that adds the workspace packages does that later. + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then + echo + echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + echo + echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" + echo set -x - if ! uv sync --all-packages --frozen --group ci-image \ - ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ - --no-python-downloads --no-managed-python; then + if ! "${frozen_sync[@]}"; then set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + echo + echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}" + echo + exit 1 + fi if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}" @@ -163,6 +193,10 @@ function install_airflow_when_building_images() { install_from_external_spec fi set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + # Packaging tools and 'pip check' are handled by the installation that adds the workspace packages + return + fi common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
