This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7a718dd5aa7 Fix ownership of files that docker uses in mounted
directories (#46428)
7a718dd5aa7 is described below
commit 7a718dd5aa79b6bc004b2daadd8a9e528be20100
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Feb 4 16:59:20 2025 +0100
Fix ownership of files that docker uses in mounted directories (#46428)
The #46358 moved docker to another mounted directory - but this
directory and all files in it are owned by host user. The directory
and all files inside should be owned by root in order to properly
reflect permissions of the files when building docker images.
The change is now simplified. Rather than passing mount directory
by variable and passing it through GitHub Actions, we hard-code
the location of docker in cleanup_docker.sh script - we also
incorporate changing ownership and showing disk space in the same
cleanup_docker.sh script and make sure that script is only called
in the "real" (not composite) actions at the beginning - right
after the repository is checked out - previously that script
was also called in composite actions and changing the repo to be
writeable was done AFTER cleanup_docker.sh - which would not
work as we want the /mnt directory to be still owned by the
host user, but the docker storage should be still owned by root.
---
.github/actions/prepare_all_ci_images/action.yml | 5 -----
.github/actions/prepare_breeze_and_image/action.yml | 9 ---------
.github/actions/prepare_single_ci_image/action.yml | 6 ------
.github/workflows/additional-ci-image-checks.yml | 3 ---
.github/workflows/additional-prod-image-tests.yml | 3 ---
.github/workflows/basic-tests.yml | 18 ------------------
.github/workflows/ci-image-build.yml | 9 ---------
.github/workflows/ci-image-checks.yml | 6 ------
.github/workflows/ci.yml | 3 ---
.github/workflows/finalize-tests.yml | 6 ------
.github/workflows/generate-constraints.yml | 3 +++
.github/workflows/helm-tests.yml | 3 ---
.github/workflows/prod-image-build.yml | 12 ------------
.github/workflows/push-image-cache.yml | 6 ------
.github/workflows/release_dockerhub_image.yml | 6 ------
scripts/ci/cleanup_docker.sh | 18 ++++++++++++------
16 files changed, 15 insertions(+), 101 deletions(-)
diff --git a/.github/actions/prepare_all_ci_images/action.yml
b/.github/actions/prepare_all_ci_images/action.yml
index 25a695b4f97..76c00a72a39 100644
--- a/.github/actions/prepare_all_ci_images/action.yml
+++ b/.github/actions/prepare_all_ci_images/action.yml
@@ -31,11 +31,6 @@ inputs:
runs:
using: "composite"
steps:
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: ${{ inputs.docker-volume-location }}
- shell: bash
# TODO: Currently we cannot loop through the list of python versions and
have dynamic list of
# tasks. Instead we hardcode all possible python versions and they -
but
# this should be implemented in stash action as list of keys to
download.
diff --git a/.github/actions/prepare_breeze_and_image/action.yml
b/.github/actions/prepare_breeze_and_image/action.yml
index b29e1f7afa3..01f5a13b60a 100644
--- a/.github/actions/prepare_breeze_and_image/action.yml
+++ b/.github/actions/prepare_breeze_and_image/action.yml
@@ -39,22 +39,13 @@ runs:
using: "composite"
steps:
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
shell: bash
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
use-uv: ${{ inputs.use-uv }}
id: breeze
- - name: Check free space
- run: df -H
- shell: bash
- - name: Make /mnt/ directory writeable
- run: sudo chown -R ${USER} /mnt
- shell: bash
- name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform
}}:${{ inputs.python }}"
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
diff --git a/.github/actions/prepare_single_ci_image/action.yml
b/.github/actions/prepare_single_ci_image/action.yml
index d8041c15937..d9602c732b4 100644
--- a/.github/actions/prepare_single_ci_image/action.yml
+++ b/.github/actions/prepare_single_ci_image/action.yml
@@ -35,12 +35,6 @@ inputs:
runs:
using: "composite"
steps:
- - name: Check free space
- run: df -H
- shell: bash
- - name: Make /mnt/ directory writeable
- run: sudo chown -R ${USER} /mnt
- shell: bash
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python
}}"
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 3dba80a024d..a6b7bdafcb5 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -142,10 +142,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index 86c1c19738a..7b551215714 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -116,10 +116,7 @@ jobs:
fetch-depth: 2
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Prepare breeze & PROD image: ${{ inputs.default-python-version
}}"
uses: ./.github/actions/prepare_breeze_and_image
with:
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index f23b9c72463..62ac87721a3 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -77,10 +77,7 @@ jobs:
fetch-depth: 0
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -101,10 +98,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: Setup pnpm
uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 #
v4.0.0
with:
@@ -171,10 +165,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Setup node"
uses: actions/setup-node@v4
with:
@@ -242,10 +233,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -300,10 +288,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -382,10 +367,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 39ef0a44f19..62a5caf5424 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -130,10 +130,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -189,12 +186,6 @@ jobs:
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
PLATFORM: ${{ inputs.platform }}
- - name: Check free space
- run: df -H
- shell: bash
- - name: Make /mnt/ directory writeable
- run: sudo chown -R ${USER} /mnt
- shell: bash
- name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
env:
PLATFORM: ${{ inputs.platform }}
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index bf4baa0af7f..faa386a2e98 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -324,10 +324,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Download docs prepared as artifacts"
uses: actions/download-artifact@v4
with:
@@ -408,10 +405,7 @@ jobs:
fetch-depth: 2
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0799ee8397f..3e41f1ce880 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -147,10 +147,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index e6af5339c52..ac13089caf6 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -104,10 +104,7 @@ jobs:
# Needed to perform push action
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >>
${GITHUB_OUTPUT}
@@ -195,10 +192,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
diff --git a/.github/workflows/generate-constraints.yml
b/.github/workflows/generate-constraints.yml
index 0e6cd9f11ae..4eb442d9421 100644
--- a/.github/workflows/generate-constraints.yml
+++ b/.github/workflows/generate-constraints.yml
@@ -68,6 +68,9 @@ jobs:
uses: actions/checkout@v4
with:
persist-credentials: false
+ - name: "Cleanup docker"
+ run: ./scripts/ci/cleanup_docker.sh
+ shell: bash
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index e45d86ae576..1b4aa19cbe5 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -97,10 +97,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 4bd13547b7f..a7880765907 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -137,10 +137,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
if: inputs.upload-package-artifact == 'true'
- name: "Cleanup dist and context file"
shell: bash
@@ -225,10 +222,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -289,12 +283,6 @@ jobs:
if: inputs.build-provider-packages != 'true'
- name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
run: breeze prod-image verify
- - name: Check free space
- run: df -H
- shell: bash
- - name: Make /mnt/ directory writeable
- run: sudo chown -R ${USER} /mnt
- shell: bash
- name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
env:
PLATFORM: ${{ inputs.platform }}
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index 5d0e3ec1d13..7698fc88e53 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -126,10 +126,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -205,10 +202,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/.github/workflows/release_dockerhub_image.yml
b/.github/workflows/release_dockerhub_image.yml
index 287c6fd9790..b8758146cc1 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -60,10 +60,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
@@ -102,10 +99,7 @@ jobs:
with:
persist-credentials: false
- name: "Cleanup docker"
- # Move docker space to second partition to have more space
run: ./scripts/ci/cleanup_docker.sh
- env:
- TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
diff --git a/scripts/ci/cleanup_docker.sh b/scripts/ci/cleanup_docker.sh
index 8f195fe7550..3327145c8f0 100755
--- a/scripts/ci/cleanup_docker.sh
+++ b/scripts/ci/cleanup_docker.sh
@@ -15,16 +15,22 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+
function cleanup_docker {
+ local target_docker_volume_location="/mnt/var-lib-docker"
+ echo "Checking free space!"
+ df -H
+ echo "Making sure that /mnt is writeable"
+ sudo chown -R "${USER}" /mnt
# This is faster than docker prune
+ echo "Stopping docker"
sudo systemctl stop docker
sudo rm -rf /var/lib/docker
- # If a path is provided in ENV, bind mount it to /var/lib/docker
- if [ -n "${TARGET_DOCKER_VOLUME_LOCATION}" ]; then
- echo "Mounting ${TARGET_DOCKER_VOLUME_LOCATION} to /var/lib/docker"
- sudo mkdir -p "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker
- sudo mount --bind "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker
- fi
+ echo "Mounting ${target_docker_volume_location} to /var/lib/docker"
+ sudo mkdir -p "${target_docker_volume_location}" /var/lib/docker
+ sudo mount --bind "${target_docker_volume_location}" /var/lib/docker
+ sudo chown -R 0:0 "${target_docker_volume_location}"
sudo systemctl start docker
}
