This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b86fa49a15d Optimize CI disk cleanup (#73184)
b86fa49a15d is described below
commit b86fa49a15dece4a0b2594c0bc645fdc0d6c1338
Author: Yossi Eliaz <[email protected]>
AuthorDate: Mon Sep 28 16:14:44 2026 +0300
Optimize CI disk cleanup (#73184)
* Optimize CI disk cleanup
* Test CI disk cleanup behavior
* Format CI optimization tests
* Limit disk cleanup changes to bounded parallel deletion
* Keep disk cleanup best-effort and test it next to the script
The script runs on the hot path of most CI jobs. Since #45322 a failed
deletion or apt cache cleanup only meant less space was freed; with
`set -euo pipefail` the same failure would fail the whole job, and the
initial `df -H` too. The bounded parallel deletion does not need it, so
the script keeps the best-effort behaviour and the test guards it:
one failing target must not stop the remaining targets or `apt-get clean`.
The test moves to `scripts/tests/tools/` to mirror the script path, and
no longer asserts on the script's source text.
Generated-by: Claude Fable 5.1
---------
Co-authored-by: Shahar Epstein <[email protected]>
---
.../tools/__init__.py} | 21 -----
scripts/tests/tools/test_free_up_disk_space.py | 98 ++++++++++++++++++++++
scripts/tools/free_up_disk_space.sh | 25 ++++--
3 files changed, 114 insertions(+), 30 deletions(-)
diff --git a/scripts/tools/free_up_disk_space.sh
b/scripts/tests/tools/__init__.py
old mode 100755
new mode 100644
similarity index 58%
copy from scripts/tools/free_up_disk_space.sh
copy to scripts/tests/tools/__init__.py
index e6ae6b16274..13a83393a91
--- a/scripts/tools/free_up_disk_space.sh
+++ b/scripts/tests/tools/__init__.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,23 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-COLOR_BLUE=$'\e[34m'
-COLOR_RESET=$'\e[0m'
-
-echo "${COLOR_BLUE}Disk space before cleanup${COLOR_RESET}"
-df -H
-
-echo "${COLOR_BLUE}Freeing up disk space${COLOR_RESET}"
-sudo rm -rf /usr/share/dotnet/
-sudo rm -rf /usr/local/graalvm/
-sudo rm -rf /usr/local/.ghcup/
-sudo rm -rf /usr/local/share/powershell
-sudo rm -rf /usr/local/share/chromium
-sudo rm -rf /usr/local/share/boost
-sudo rm -rf /usr/local/lib/android
-sudo rm -rf /opt/hostedtoolcache
-sudo rm -rf /opt/ghc
-sudo apt-get clean
-echo "${COLOR_BLUE}Disk space after cleanup${COLOR_RESET}"
-df -H
diff --git a/scripts/tests/tools/test_free_up_disk_space.py
b/scripts/tests/tools/test_free_up_disk_space.py
new file mode 100644
index 00000000000..f73fc81fbc0
--- /dev/null
+++ b/scripts/tests/tools/test_free_up_disk_space.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import os
+import shlex
+import subprocess
+from pathlib import Path
+
+import pytest
+
+SCRIPT = Path(__file__).resolve().parents[2] / "tools" /
"free_up_disk_space.sh"
+
+EXPECTED_TARGETS = [
+ "/usr/share/dotnet/",
+ "/usr/local/graalvm/",
+ "/usr/local/.ghcup/",
+ "/usr/local/share/powershell",
+ "/usr/local/share/chromium",
+ "/usr/local/share/boost",
+ "/usr/local/lib/android",
+ "/opt/hostedtoolcache",
+ "/opt/ghc",
+]
+
+
[email protected]
+def fake_tools(tmp_path):
+ """Shadow ``sudo`` and ``df`` with a logger that can be told to fail for a
matching command."""
+ tools = tmp_path / "bin"
+ tools.mkdir()
+ command = tools / "command"
+ command.write_text(
+ r"""#!/usr/bin/env bash
+name="${0##*/}"
+printf '%s\n' "${name} $*" >> "${COMMAND_LOG}"
+if [[ -n "${FAIL_MATCH:-}" && "${name} $*" == *"${FAIL_MATCH}"* ]]; then
+ exit 42
+fi
+"""
+ )
+ command.chmod(0o755)
+ for name in ("sudo", "df"):
+ (tools / name).symlink_to(command)
+ return {"PATH": f"{tools}:{os.environ['PATH']}", "COMMAND_LOG":
str(tmp_path / "commands.log")}
+
+
+def run_script(env):
+ return subprocess.run(
+ ["bash", "--noprofile", "--norc", str(SCRIPT)],
+ env={**os.environ, **env},
+ capture_output=True,
+ text=True,
+ timeout=15,
+ check=False,
+ )
+
+
+def read_commands(env):
+ return [shlex.split(line) for line in
Path(env["COMMAND_LOG"]).read_text().splitlines()]
+
+
+def get_deleted_paths(commands):
+ return [command[-1] for command in commands if command[:3] == ["sudo",
"rm", "-rf"]]
+
+
+def test_deletes_expected_targets_before_apt_clean(fake_tools):
+ result = run_script(fake_tools)
+
+ assert result.returncode == 0, result.stderr
+ commands = read_commands(fake_tools)
+ assert sorted(get_deleted_paths(commands)) == sorted(EXPECTED_TARGETS)
+ apt_clean_position = commands.index(["sudo", "apt-get", "clean"])
+ assert not get_deleted_paths(commands[apt_clean_position:])
+
+
+def test_failed_deletion_does_not_stop_remaining_cleanup(fake_tools):
+ result = run_script({**fake_tools, "FAIL_MATCH": "/usr/local/lib/android"})
+
+ assert result.returncode == 0, result.stderr
+ commands = read_commands(fake_tools)
+ assert sorted(get_deleted_paths(commands)) == sorted(EXPECTED_TARGETS)
+ assert ["sudo", "apt-get", "clean"] in commands
diff --git a/scripts/tools/free_up_disk_space.sh
b/scripts/tools/free_up_disk_space.sh
index e6ae6b16274..63d9df40849 100755
--- a/scripts/tools/free_up_disk_space.sh
+++ b/scripts/tools/free_up_disk_space.sh
@@ -23,15 +23,22 @@ echo "${COLOR_BLUE}Disk space before cleanup${COLOR_RESET}"
df -H
echo "${COLOR_BLUE}Freeing up disk space${COLOR_RESET}"
-sudo rm -rf /usr/share/dotnet/
-sudo rm -rf /usr/local/graalvm/
-sudo rm -rf /usr/local/.ghcup/
-sudo rm -rf /usr/local/share/powershell
-sudo rm -rf /usr/local/share/chromium
-sudo rm -rf /usr/local/share/boost
-sudo rm -rf /usr/local/lib/android
-sudo rm -rf /opt/hostedtoolcache
-sudo rm -rf /opt/ghc
+
+# These directories are independent; cap concurrent deletions to limit disk
contention.
+cleanup_targets=(
+ /usr/share/dotnet/
+ /usr/local/graalvm/
+ /usr/local/.ghcup/
+ /usr/local/share/powershell
+ /usr/local/share/chromium
+ /usr/local/share/boost
+ /usr/local/lib/android
+ /opt/hostedtoolcache
+ /opt/ghc
+)
+printf '%s\0' "${cleanup_targets[@]}" | xargs -0 -r -n 1 -P 4 sudo rm -rf --
+
sudo apt-get clean
+
echo "${COLOR_BLUE}Disk space after cleanup${COLOR_RESET}"
df -H
