This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f43873bbe5d Recreate and auto-prune caches used in CI (#48115)
f43873bbe5d is described below
commit f43873bbe5d42bd950ed6353bea5e7f7bacfd404
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Mar 23 20:36:29 2025 +0100
Recreate and auto-prune caches used in CI (#48115)
The uv and mount cache for our images grows uncontrollably over
time - because it keeps on getting more versions of packages stored
in the cache - the more of those we upgrade over time, the more it
older version it contains.
The uv team recommends runnig `uv cache prune --ci` in the ci
jobs to decrease the size of cache - what it does it keeps all
the packages that were build from sources while removing the
downloaded binary wheels - assuming that downloading the wheels
will be even faster when done by `uv` rather than when cache is
downloaded. See:
https://docs.astral.sh/uv/concepts/cache/#caching-in-continuous-integration
That should keep the cache much smaller and builds much faster for
much longer time. Also `uv cache prune --ci` removes all cache
entries that are unused because of uv version upgrades, which
means that most likely the cache will be self-cleaning over time.
---
.github/actions/prepare_breeze_and_image/action.yml | 2 +-
.github/actions/prepare_single_ci_image/action.yml | 2 +-
.github/workflows/ci-image-build.yml | 14 +++++++-------
.github/workflows/prod-image-build.yml | 2 +-
dev/breeze/doc/images/output_ci-image_load.svg | 2 +-
dev/breeze/doc/images/output_ci-image_load.txt | 2 +-
.../src/airflow_breeze/commands/ci_image_commands.py | 17 ++++++++++-------
7 files changed, 22 insertions(+), 19 deletions(-)
diff --git a/.github/actions/prepare_breeze_and_image/action.yml
b/.github/actions/prepare_breeze_and_image/action.yml
index 01f5a13b60a..3254254a865 100644
--- a/.github/actions/prepare_breeze_and_image/action.yml
+++ b/.github/actions/prepare_breeze_and_image/action.yml
@@ -49,7 +49,7 @@ runs:
- name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform
}}:${{ inputs.python }}"
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
- key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{
inputs.python }}
+ key: ${{ inputs.image-type }}-image-save-v3-${{ inputs.platform }}-${{
inputs.python }}
path: "/mnt/"
only-current-branch: 'true'
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{
inputs.python }}"
diff --git a/.github/actions/prepare_single_ci_image/action.yml
b/.github/actions/prepare_single_ci_image/action.yml
index d9602c732b4..f8046de085b 100644
--- a/.github/actions/prepare_single_ci_image/action.yml
+++ b/.github/actions/prepare_single_ci_image/action.yml
@@ -38,7 +38,7 @@ runs:
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python
}}"
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
- key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }}
+ key: ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}
path: "/mnt/"
only-current-branch: 'true'
if: contains(inputs.python-versions-list-as-string, inputs.python)
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 62a5caf5424..c695778b87b 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -138,13 +138,13 @@ jobs:
- name: "Restore ci-cache mount image ${{ inputs.platform }}:${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
- key: "ci-cache-mount-save-v2-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ key: "ci-cache-mount-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
path: "/tmp/"
id: restore-cache-mount
- name: "Verify ci-cache file exists"
if: steps.restore-cache-mount.outputs.stash-hit == 'true'
env:
- CACHE_FILE: "/tmp/ci-cache-mount-save-v2-${{
env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz"
+ CACHE_FILE: "/tmp/ci-cache-mount-save-v3-${{
env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz"
run: |
if [ ! -f "$CACHE_FILE" ]; then
echo "Cache file not found:"
@@ -157,7 +157,7 @@ jobs:
PYTHON_MAJOR_MINOR_VERSION: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
run: >
breeze ci-image import-mount-cache
- --cache-file
/tmp/ci-cache-mount-save-v2-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
+ --cache-file
/tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
if: steps.restore-cache-mount.outputs.stash-hit == 'true'
- name: "Login to ghcr.io"
env:
@@ -194,7 +194,7 @@ jobs:
- name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
- key: ci-image-save-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}
+ key: ci-image-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}
path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION
}}.tar"
if-no-files-found: 'error'
retention-days: '2'
@@ -204,13 +204,13 @@ jobs:
PYTHON_MAJOR_MINOR_VERSION: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
run: >
breeze ci-image export-mount-cache
- --cache-file
/tmp/ci-cache-mount-save-v2-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
+ --cache-file
/tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
if: inputs.upload-mount-cache-artifact == 'true'
- name: "Stash cache mount ${{ inputs.platform }}:${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
- key: "ci-cache-mount-save-v2-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
- path: "/tmp/ci-cache-mount-save-v2-${{
env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz"
+ key: "ci-cache-mount-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}"
+ path: "/tmp/ci-cache-mount-save-v3-${{
env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz"
if-no-files-found: 'error'
retention-days: 2
if: inputs.upload-mount-cache-artifact == 'true'
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 749330c0d35..e6b0a799d1f 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -292,7 +292,7 @@ jobs:
- name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
- key: prod-image-save-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}
+ key: prod-image-save-v3-${{ inputs.platform }}-${{
env.PYTHON_MAJOR_MINOR_VERSION }}
path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION
}}.tar"
if-no-files-found: 'error'
retention-days: '2'
diff --git a/dev/breeze/doc/images/output_ci-image_load.svg
b/dev/breeze/doc/images/output_ci-image_load.svg
index cb7036af1c9..cc24b400602 100644
--- a/dev/breeze/doc/images/output_ci-image_load.svg
+++ b/dev/breeze/doc/images/output_ci-image_load.svg
@@ -172,7 +172,7 @@
</text><text class="breeze-ci-image-load-r5" x="0" y="215.2" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-8)">│</text><text
class="breeze-ci-image-load-r5" x="451.4" y="215.2" textLength="732"
clip-path="url(#breeze-ci-image-load-line-8)">[default: 3.9]                                    
[...]
</text><text class="breeze-ci-image-load-r5" x="0" y="239.6" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-9)">│</text><text
class="breeze-ci-image-load-r4" x="61" y="239.6" textLength="122"
clip-path="url(#breeze-ci-image-load-line-9)">--platform</text><text
class="breeze-ci-image-load-r1" x="451.4" y="239.6" textLength="329.4"
clip-path="url(#breeze-ci-image-load-line-9)">Platform for Airflow image.</text><text
class="breeze-ci-image-load-r7" x="793" y="239 [...]
</text><text class="breeze-ci-image-load-r5" x="0" y="264" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-10)">│</text><text
class="breeze-ci-image-load-r4" x="61" y="264" textLength="146.4"
clip-path="url(#breeze-ci-image-load-line-10)">--image-file</text><text
class="breeze-ci-image-load-r1" x="451.4" y="264" textLength="988.2"
clip-path="url(#breeze-ci-image-load-line-10)">Optional file name to load the image from - name m
[...]
-</text><text class="breeze-ci-image-load-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-11)">│</text><text
class="breeze-ci-image-load-r1" x="451.4" y="288.4" textLength="988.2"
clip-path="url(#breeze-ci-image-load-line-11)">convention:`ci-image-save-{escaped_platform}-*-{python_version}.tar`. where      </text><text
class="breeze-ci-image-load-r5" x="1451.8" y="288.4" textLength="12.2"
clip-path="url(#breeze-ci-image-lo [...]
+</text><text class="breeze-ci-image-load-r5" x="0" y="288.4" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-11)">│</text><text
class="breeze-ci-image-load-r1" x="451.4" y="288.4" textLength="988.2"
clip-path="url(#breeze-ci-image-load-line-11)">convention:`ci-image-save-v3-{escaped_platform}-*-{python_version}.tar`. where   </text><text
class="breeze-ci-image-load-r5" x="1451.8" y="288.4" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-11)">│< [...]
</text><text class="breeze-ci-image-load-r5" x="0" y="312.8" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-12)">│</text><text
class="breeze-ci-image-load-r1" x="451.4" y="312.8" textLength="988.2"
clip-path="url(#breeze-ci-image-load-line-12)">escaped_platform is one of linux_amd64 or linux_arm64. If it does not exist in   </text><text
class="breeze-ci-image-load-r5" x="1451.8" y="312.8" textLength=" [...]
</text><text class="breeze-ci-image-load-r5" x="0" y="337.2" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-13)">│</text><text
class="breeze-ci-image-load-r1" x="451.4" y="337.2" textLength="988.2"
clip-path="url(#breeze-ci-image-load-line-13)">current working dir and if you do not specify absolute file, it will be searched </text><text
class="breeze-ci-image-load-r5" x="1451.8" y="337.2" textLength=" [...]
</text><text class="breeze-ci-image-load-r5" x="0" y="361.6" textLength="12.2"
clip-path="url(#breeze-ci-image-load-line-14)">│</text><text
class="breeze-ci-image-load-r1" x="451.4" y="361.6" textLength="134.2"
clip-path="url(#breeze-ci-image-load-line-14)">for in the </text><text
class="breeze-ci-image-load-r4" x="585.6" y="361.6" textLength="195.2"
clip-path="url(#breeze-ci-image-load-line-14)">--image-file-dir</text><text
class="breeze-ci-image-load-r1" x="780.8" y="361 [...]
diff --git a/dev/breeze/doc/images/output_ci-image_load.txt
b/dev/breeze/doc/images/output_ci-image_load.txt
index 9ab6e91ff9d..c48e9e7439a 100644
--- a/dev/breeze/doc/images/output_ci-image_load.txt
+++ b/dev/breeze/doc/images/output_ci-image_load.txt
@@ -1 +1 @@
-c7792f39775f01082f867e0ce0f38ecb
+55eff05e9dfa48cff628b818e7110171
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
index f7cf76dca4d..eddcd7ff875 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
@@ -82,6 +82,7 @@ from
airflow_breeze.commands.common_package_installation_options import (
option_airflow_constraints_location,
option_airflow_constraints_mode_ci,
)
+from airflow_breeze.global_constants import UV_VERSION
from airflow_breeze.params.build_ci_params import BuildCiParams
from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.click_utils import BreezeGroup
@@ -294,7 +295,7 @@ option_ci_image_file_to_load = click.option(
type=click.Path(dir_okay=False, readable=True, path_type=Path,
resolve_path=True),
envvar="IMAGE_FILE",
help="Optional file name to load the image from - name must follow the
convention:"
- "`ci-image-save-{escaped_platform}-*-{python_version}.tar`. where
escaped_platform is one of "
+ "`ci-image-save-v3-{escaped_platform}-*-{python_version}.tar`. where
escaped_platform is one of "
"linux_amd64 or linux_arm64. If it does not exist in current working dir
and if you do not specify "
"absolute file, it will be searched for in the --image-file-dir.",
)
@@ -627,7 +628,7 @@ def save(
run_command(["docker", "buildx", "du", "--verbose"], check=False)
escaped_platform = platform.replace("/", "_")
if not image_file:
- image_file_to_store = image_file_dir /
f"ci-image-save-{escaped_platform}-{python}.tar"
+ image_file_to_store = image_file_dir /
f"ci-image-save-v3-{escaped_platform}-{python}.tar"
elif image_file.is_absolute():
image_file_to_store = image_file
else:
@@ -673,7 +674,7 @@ def load(
escaped_platform = platform.replace("/", "_")
if not image_file:
- image_file_to_load = image_file_dir /
f"ci-image-save-{escaped_platform}-{python}.tar"
+ image_file_to_load = image_file_dir /
f"ci-image-save-v3-{escaped_platform}-{python}.tar"
elif image_file.is_absolute() or image_file.exists():
image_file_to_load = image_file
else:
@@ -684,10 +685,10 @@ def load(
f"[error]The image file {image_file_to_load} does not end with
'-{python}.tar'. Exiting.[/]"
)
sys.exit(1)
- if not
image_file_to_load.name.startswith(f"ci-image-save-{escaped_platform}"):
+ if not
image_file_to_load.name.startswith(f"ci-image-save-v3-{escaped_platform}"):
get_console().print(
f"[error]The image file {image_file_to_load} does not start with "
- f"'ci-image-save-{escaped_platform}'. Exiting.[/]"
+ f"'ci-image-save-v3-{escaped_platform}'. Exiting.[/]"
)
sys.exit(1)
@@ -1021,12 +1022,14 @@ def export_mount_cache(
"""
perform_environment_checks()
make_sure_builder_configured(params=BuildCiParams(builder=builder))
- dockerfile = """
+ dockerfile = f"""
# syntax=docker/dockerfile:1.4
- FROM python:3.9-slim-bookworm
+ FROM ghcr.io/astral-sh/uv:{UV_VERSION}-bookworm-slim
ARG TARGETARCH
ARG DEPENDENCY_CACHE_EPOCH=<REPLACE_FROM_DOCKER_CI>
RUN
--mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/
\\
+ uv cache prune --ci
+ RUN
--mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/
\\
tar -C /root/.cache/ -czf /root/.cache.tar.gz .
"""
