This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d108da4cf66 Push multi-arch Python SDK containers built by GH tests to
GCR and remove auto delete (#27840)
d108da4cf66 is described below
commit d108da4cf66d4cf736346857235aba768a43348e
Author: Celeste Zeng <61256376+celeste-z...@users.noreply.github.com>
AuthorDate: Thu Aug 3 17:31:45 2023 -0700
Push multi-arch Python SDK containers built by GH tests to GCR and remove
auto delete (#27840)
---
.../workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml | 2 +-
sdks/go/test/run_validatesrunner_tests.sh | 8 ++++++--
sdks/python/container/run_validatescontainer.sh | 9 +++++++--
3 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml
b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml
index 89e93758c67..468a9b9a1fb 100644
--- a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml
+++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml
@@ -108,7 +108,7 @@ jobs:
arguments: |
-PpythonVersion=${{ matrix.python_version }} \
-Pcontainer-architecture-list=arm64,amd64 \
- -Pdocker-repository=us.gcr.io/apache-beam-testing/github-actions \
+
-Pdocker-repository-root=us.gcr.io/apache-beam-testing/github-actions \
-Pdocker-tag=${{ env.MULTIARCH_TAG }} \
-Ppush-containers \
env:
diff --git a/sdks/go/test/run_validatesrunner_tests.sh
b/sdks/go/test/run_validatesrunner_tests.sh
index d9621d7fb4d..2440d0a48af 100755
--- a/sdks/go/test/run_validatesrunner_tests.sh
+++ b/sdks/go/test/run_validatesrunner_tests.sh
@@ -438,8 +438,12 @@ cd ../..
if [[ "$RUNNER" == "dataflow" ]]; then
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Built container image was not removed.
Possibly, it was not not saved locally."
- gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
-
+ # Note: we don't delete the multi-arch containers here because this command
only deletes the manifest list with the tag,
+ # the associated container images can't be deleted because they are not
tagged. However, the multi-arch containers
+ # are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs every 6
weeks.
+ if [[ "$USER" == "jenkins" ]]; then
+ gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
+ fi
if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n
"$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then
# Delete the java cross-language container locally and remotely
docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to remove container"
diff --git a/sdks/python/container/run_validatescontainer.sh
b/sdks/python/container/run_validatescontainer.sh
index 3ea9148ca67..a7fbac6cf49 100755
--- a/sdks/python/container/run_validatescontainer.sh
+++ b/sdks/python/container/run_validatescontainer.sh
@@ -104,7 +104,12 @@ function cleanup_container {
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container
image"
done
- gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
+ # Note: we don't delete the multi-arch containers here because this command
only deletes the manifest list with the tag,
+ # the associated container images can't be deleted because they are not
tagged. However, the multi-arch containers
+ # are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs every 6
weeks.
+ if [[ "$ARCH" == "x86" ]]; then
+ gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
+ fi
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
@@ -139,4 +144,4 @@ pytest -o junit_suite_name=$IMAGE_NAME \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
-echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
\ No newline at end of file
+echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
