This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 5e9ea74c85e Build multi-arch snapshot containers (#29419)
5e9ea74c85e is described below
commit 5e9ea74c85e2ce6e6fb3418a0abc984fbebaefb1
Author: Danny McCormick <[email protected]>
AuthorDate: Thu Nov 16 16:52:22 2023 -0800
Build multi-arch snapshot containers (#29419)
* Build multi-arch snapshot containers
* Add push-containers
* Setup buildx
* Remove push arg
* Broaden check
* Missing )
* Push with docker task
* Test out SOURCE_DATE_EPOCH
* Back to TagPush
* Save containers instead of pushing
* Try containerImageTags
* Test multiple tags
* Update docker plugin
* Fix
* Fix
* Turn off provenance
* Fix jenkins precommits
* Fix jenkins precommits
* Remove source_date_epoch
* Don't delete containers if they have dependencies that cant be deleted
---
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 8 +++++--
.../tools/stale_dataflow_prebuilt_image_cleaner.sh | 27 ++++++++++++++++++++--
.../org/apache/beam/gradle/BeamDockerPlugin.groovy | 19 ++++++++++++++-
3 files changed, 49 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
index cb9174f9104..32f65969e51 100644
--- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
+++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
@@ -78,6 +78,8 @@ jobs:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.container_task }})
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@v0
with:
@@ -101,8 +103,10 @@ jobs:
- name: run Publish Beam SDK Snapshots script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :sdks:${{ matrix.container_task }}:dockerTagPush
+ gradle-command: :sdks:${{ matrix.container_task }}:docker
arguments: |
-Pjava11Home=$JAVA_HOME_11_X64 \
-Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \
- -Pdocker-tag-list=${{ github.sha }},latest
\ No newline at end of file
+ -Pdocker-tag-list=${{ github.sha }},latest \
+ -Pcontainer-architecture-list=arm64,amd64 \
+ -Ppush-containers \
diff --git a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
index eb3acb809ff..2838d6245ce 100755
--- a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
+++ b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
@@ -70,8 +70,31 @@ for image_name in ${IMAGE_NAMES[@]}; do
# do not delete the one with latest label and the newest image without
latest label
# this make sure we leave at least one container under each image name,
either labelled "latest" or not
if [ "$LATEST_IN_TIME" != "$current" ]; then
- echo "Deleting image. Command: gcloud container images delete
${image_name}@"${current}" --force-delete-tags -q"
- gcloud container images delete ${image_name}@"${current}"
--force-delete-tags -q || FAILED_TO_DELETE+="${current} "
+ # Check to see if this image is built on top of earlier images. This
is the case for multiarch images,
+ # they will have a virtual size of 0 and a created date at the start
of the epoch, but their manifests will
+ # point to active images. These images should only be deleted when all
of their dependencies can be safely
+ # deleted.
+ MANIFEST=$(docker manifest inspect ${image_name}@"${current}")
+ SHOULD_DELETE=0
+ DIGEST=$(echo $MANIFEST | jq -r '.manifests[0].digest')
+ if [ "$DIGEST" != "null" ]
+ then
+ SHOULD_DELETE=1
+ for i in ${STALE_IMAGES_CURRENT[@]}
+ do
+ echo "$i"
+ if [ "$i" = "$DIGEST" ]
+ then
+ SHOULD_DELETE=0
+ fi
+ done
+ fi
+
+ if [ $SHOULD_DELETE = 0 ]
+ then
+ echo "Deleting image. Command: gcloud container images delete
${image_name}@"${current}" --force-delete-tags -q"
+ gcloud container images delete ${image_name}@"${current}"
--force-delete-tags -q || FAILED_TO_DELETE+="${current} "
+ fi
fi
done
fi
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy
index 442b35439ca..2aa37d89127 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy
@@ -240,6 +240,11 @@ class BeamDockerPlugin implements Plugin<Project> {
if (ext.builder != null) {
buildCommandLine.addAll('--builder', ext.builder)
}
+ // Jenkins dependencies aren't up to date enough to accept this flag.
+ // Temporarily exclude until we fully move to GHA.
+ if (!ext.project.jenkins.isCIBuild) {
+ buildCommandLine.addAll('--provenance=false')
+ }
} else {
buildCommandLine.add 'build'
}
@@ -267,7 +272,19 @@ class BeamDockerPlugin implements Plugin<Project> {
if (ext.pull) {
buildCommandLine.add '--pull'
}
- buildCommandLine.addAll(['-t', "${-> ext.name}", '.'])
+ if (!ext.tags.isEmpty() && ext.push) {
+ String[] repoParts = (ext.name as String).split(':')
+ String repo = repoParts[0]
+ for (int i = 1; i < repoParts.length - 1; i++) {
+ repo += ':' + repoParts[i]
+ }
+ for (tag in ext.getTags()) {
+ buildCommandLine.addAll(['-t', repo + ':' + tag])
+ }
+ buildCommandLine.add '.'
+ } else {
+ buildCommandLine.addAll(['-t', "${-> ext.name}", '.'])
+ }
logger.debug("${buildCommandLine}" as String)
return buildCommandLine
}
