This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 51a84ce44e9 More diagnostics and more logical name for cleaning up the
runner (#50540)
51a84ce44e9 is described below
commit 51a84ce44e919d7368cd2f6f182dbb1c2f8f1fd3
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue May 13 16:15:02 2025 +0200
More diagnostics and more logical name for cleaning up the runner (#50540)
When we are starting the runner on CI we are cleaning it up and
preparing the /mnt directory to be used for all kind of disk
storage we need during CI build. This is happening in the step
that was named "cleanup docker" but it did more than that.
The script and all references to it are renamed, also more diagnostics
is printed just before we restore CI image from cache to see how
much space we have available, because for some reason we seem to
have problems with disk space there in some jobs
---
.github/actions/prepare_breeze_and_image/action.yml | 9 +++++++--
.github/workflows/additional-ci-image-checks.yml | 4 ++--
.github/workflows/additional-prod-image-tests.yml | 2 --
.github/workflows/basic-tests.yml | 20 ++++++++++----------
.github/workflows/ci-amd.yml | 8 ++++----
.github/workflows/ci-arm.yml | 4 ++--
.github/workflows/ci-image-build.yml | 4 ++--
.github/workflows/ci-image-checks.yml | 10 ++++------
.github/workflows/finalize-tests.yml | 4 ++--
.github/workflows/generate-constraints.yml | 4 ++--
.github/workflows/helm-tests.yml | 4 ++--
.github/workflows/prod-image-build.yml | 8 ++++----
.github/workflows/publish-docs-to-s3.yml | 8 ++++----
.github/workflows/push-image-cache.yml | 8 ++++----
.github/workflows/release_dockerhub_image.yml | 8 +++++---
.github/workflows/release_single_dockerhub_image.yml | 8 ++++----
...eanup_docker.sh => prepare_and_cleanup_runner.sh} | 4 ++--
scripts/tools/free_up_disk_space.sh | 4 ++--
18 files changed, 62 insertions(+), 59 deletions(-)
diff --git a/.github/actions/prepare_breeze_and_image/action.yml
b/.github/actions/prepare_breeze_and_image/action.yml
index 3254254a865..4a2685cc4eb 100644
--- a/.github/actions/prepare_breeze_and_image/action.yml
+++ b/.github/actions/prepare_breeze_and_image/action.yml
@@ -38,14 +38,19 @@ outputs:
runs:
using: "composite"
steps:
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
shell: bash
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
use-uv: ${{ inputs.use-uv }}
id: breeze
+ - name: "Check free space"
+ shell: bash
+ run: |
+ echo "Checking free space!"
+ df -H
- name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform
}}:${{ inputs.python }}"
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 296e58fa23a..9e7edaee0ee 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -138,8 +138,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index 489b2967af3..f8a5d7cb4ec 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -113,8 +113,6 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
- name: "Prepare breeze & PROD image: ${{ inputs.default-python-version
}}"
uses: ./.github/actions/prepare_breeze_and_image
with:
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index 5d931b00186..52983ad65d5 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -76,8 +76,8 @@ jobs:
# Need to fetch all history for selective checks tests
fetch-depth: 0
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -97,8 +97,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: Setup pnpm
uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 #
v4.0.0
with:
@@ -194,8 +194,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -249,8 +249,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -322,8 +322,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index a788319e496..51e17063a8a 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -137,8 +137,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
@@ -864,8 +864,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index 5b8cee0a432..643d5364c60 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -130,8 +130,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index d531ffebebf..b52fdb906fa 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -120,8 +120,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index 64f197f7ebf..f9bc61bd2b5 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -330,15 +330,15 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Download docs prepared as artifacts"
uses: actions/download-artifact@v4
with:
name: airflow-docs
path: './generated/_build'
- name: Check disk space available
- run: df -h
+ run: df -H
- name: Create /mnt/airflow-site directory
run: sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}"
/mnt/airflow-site
- name: "Clone airflow-site"
@@ -358,7 +358,7 @@ jobs:
breeze release-management publish-docs --override-versioned
--run-in-parallel
${DOCS_LIST_AS_STRING}
- name: Check disk space available
- run: df -h
+ run: df -H
- name: "Generate back references for providers"
run: breeze release-management add-back-references all-providers
- name: "Generate back references for apache-airflow"
@@ -408,8 +408,6 @@ jobs:
with:
fetch-depth: 2
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index 814b65906bb..6afcbd08122 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -103,8 +103,8 @@ jobs:
with:
# Needed to perform push action
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >>
${GITHUB_OUTPUT}
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 363888ff297..35b31db98e5 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -67,8 +67,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
shell: bash
- name: "Install Breeze"
uses: ./.github/actions/breeze
diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index e00ac14506a..26bf28cce1b 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -96,8 +96,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 721158f0978..4b0ff29714f 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -127,8 +127,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
if: inputs.upload-package-artifact == 'true'
- name: "Cleanup dist and context file"
shell: bash
@@ -214,8 +214,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index d3e80a98c56..7740773c107 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -179,8 +179,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Checkout ${{ inputs.ref }}"
run: |
git clone https://github.com/apache/airflow.git
"${AIRFLOW_ROOT_PATH}"
@@ -193,7 +193,7 @@ jobs:
name: airflow-docs
path: ${{ env.AIRFLOW_ROOT_PATH }}/generated/_build
- name: Check disk space available
- run: df -h
+ run: df -H
# Here we will create temp airflow-site dir to publish
# docs and for back-references
- name: Create /mnt/airflow-site directory
@@ -213,7 +213,7 @@ jobs:
breeze release-management publish-docs --override-versioned
--run-in-parallel
${INCLUDE_DOCS}
- name: Check disk space available
- run: df -h
+ run: df -H
- name: "Generate back references for providers"
run: breeze release-management add-back-references all-providers
- name: "Generate back references for apache-airflow"
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index d562f798c77..f2258c13b77 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -116,8 +116,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -187,8 +187,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index 08cdd0bebf2..fe1ce300fbf 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -79,13 +79,15 @@ jobs:
echo "Limit python versions: '${LIMIT_PYTHON_VERSIONS}'"
- name: "Cleanup repo"
shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ run: >
+ docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install uv"
run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: "Check airflow version"
diff --git a/.github/workflows/release_single_dockerhub_image.yml
b/.github/workflows/release_single_dockerhub_image.yml
index dd9b4596565..55a8c2cc004 100644
--- a/.github/workflows/release_single_dockerhub_image.yml
+++ b/.github/workflows/release_single_dockerhub_image.yml
@@ -79,8 +79,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -177,8 +177,8 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
+ - name: "Prepare and cleanup runner"
+ run: ./scripts/ci/prepare_and_cleanup_runner.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/scripts/ci/cleanup_docker.sh
b/scripts/ci/prepare_and_cleanup_runner.sh
similarity index 95%
rename from scripts/ci/cleanup_docker.sh
rename to scripts/ci/prepare_and_cleanup_runner.sh
index 3327145c8f0..9f274c91898 100755
--- a/scripts/ci/cleanup_docker.sh
+++ b/scripts/ci/prepare_and_cleanup_runner.sh
@@ -17,7 +17,7 @@
# under the License.
-function cleanup_docker {
+function prepare_and_cleanup_runner {
local target_docker_volume_location="/mnt/var-lib-docker"
echo "Checking free space!"
df -H
@@ -34,4 +34,4 @@ function cleanup_docker {
sudo systemctl start docker
}
-cleanup_docker
+prepare_and_cleanup_runner
diff --git a/scripts/tools/free_up_disk_space.sh
b/scripts/tools/free_up_disk_space.sh
index f959f65e9e1..8ba7cfe4220 100755
--- a/scripts/tools/free_up_disk_space.sh
+++ b/scripts/tools/free_up_disk_space.sh
@@ -20,7 +20,7 @@ COLOR_BLUE=$'\e[34m'
COLOR_RESET=$'\e[0m'
echo "${COLOR_BLUE}Disk space before cleanup${COLOR_RESET}"
-df -h
+df -H
echo "${COLOR_BLUE}Freeing up disk space${COLOR_RESET}"
sudo rm -rf /usr/share/dotnet/
@@ -36,4 +36,4 @@ sudo rm -rf /opt/hostedtoolcache/go
sudo rm -rf /opt/ghc
sudo apt-get clean
echo "${COLOR_BLUE}Disk space after cleanup${COLOR_RESET}"
-df -h
+df -H
