This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new cb7329c3273 Use prebuilt Snapshots SDK images for PostCommit Python
Arm (#39498)
cb7329c3273 is described below
commit cb7329c3273efe16b9e628e2c8754120b9436a11
Author: Abdelrahman Ibrahim <[email protected]>
AuthorDate: Mon Jul 27 17:09:08 2026 +0300
Use prebuilt Snapshots SDK images for PostCommit Python Arm (#39498)
---
.../trigger_files/beam_PostCommit_Python_Arm.json | 4 ++++
.github/workflows/beam_PostCommit_Python_Arm.yml | 21 ++++++---------------
sdks/python/scripts/run_integration_test.sh | 11 +++++++++--
sdks/python/test-suites/dataflow/common.gradle | 8 +++++++-
4 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Python_Arm.json
b/.github/trigger_files/beam_PostCommit_Python_Arm.json
new file mode 100644
index 00000000000..1efc8e9e440
--- /dev/null
+++ b/.github/trigger_files/beam_PostCommit_Python_Arm.json
@@ -0,0 +1,4 @@
+{
+ "comment": "Modify this file in a trivial way to cause this test suite to
run",
+ "modification": 1
+}
diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml
b/.github/workflows/beam_PostCommit_Python_Arm.yml
index 961dfbb2c53..acefdcc44a9 100644
--- a/.github/workflows/beam_PostCommit_Python_Arm.yml
+++ b/.github/workflows/beam_PostCommit_Python_Arm.yml
@@ -54,8 +54,9 @@ jobs:
beam_PostCommit_Python_Arm:
name: ${{ matrix.job_name }} ${{ matrix.python_version }}
runs-on: [self-hosted, ubuntu-24.04, main]
- # Python 3.14 multiarch container builds take much longer than other
versions.
- timeout-minutes: ${{ matrix.python_version == '3.14' && 600 || 480 }}
+ # Python ARM PostCommit uses prebuilt Snapshots SDK images, so it no longer
+ # spends hours on multiarch container builds. Keep headroom for the IT
suite.
+ timeout-minutes: 180
strategy:
fail-fast: false
matrix:
@@ -93,20 +94,12 @@ jobs:
credentials_json: ${{ secrets.GCP_SA_KEY }}
- name: Set up Cloud SDK
uses:
google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db
- - name: Set up Docker Buildx
- uses:
docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c
- - name: GCloud Docker credential helper
- run: |
- gcloud auth configure-docker us.gcr.io
- name: Set PY_VER_CLEAN
id: set_py_ver_clean
run: |
PY_VER=${{ matrix.python_version }}
PY_VER_CLEAN=${PY_VER//.}
echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT
- - name: Generate TAG unique variable based on timestamp
- id: set_tag
- run: echo "TAG=$(date +'%Y%m%d-%H%M%S%N')" >> $GITHUB_OUTPUT
- name: run PostCommit Python ${{ matrix.python_version }} script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
@@ -114,12 +107,10 @@ jobs:
arguments: |
-PuseWheelDistribution \
-PpythonVersion=${{ matrix.python_version }} \
- -Pcontainer-architecture-list=arm64,amd64 \
-
-Pdocker-repository-root=us.gcr.io/apache-beam-testing/github-actions \
- -Pdocker-tag=${{ steps.set_tag.outputs.TAG }} \
- -Ppush-containers \
+ -PusePrebuiltSdkContainer \
env:
- MULTIARCH_TAG: ${{ steps.set_tag.outputs.TAG }}
+ # Use multiarch images published by Publish Beam SDK Snapshots.
+ SDK_CONTAINER_IMAGE:
gcr.io/apache-beam-testing/beam-sdk/beam_python${{ matrix.python_version
}}_sdk:latest
USER: github-actions
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
diff --git a/sdks/python/scripts/run_integration_test.sh
b/sdks/python/scripts/run_integration_test.sh
index 4206f9ee971..c1e0401793f 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -248,8 +248,15 @@ if [[ -z $PIPELINE_OPTS ]]; then
if [[ "$ARCH" == "ARM" ]]; then
opts+=("--machine_type=t2a-standard-1")
- IMAGE_NAME="beam_python${PY_VERSION}_sdk"
-
opts+=("--sdk_container_image=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME:$MULTIARCH_TAG")
+ # Prefer an explicit image (e.g. Snapshots latest) when provided so CI can
+ # skip rebuilding multiarch SDK containers. Otherwise use the image built
+ # and pushed by the caller under MULTIARCH_TAG.
+ if [[ -n "${SDK_CONTAINER_IMAGE:-}" ]]; then
+ opts+=("--sdk_container_image=$SDK_CONTAINER_IMAGE")
+ else
+ IMAGE_NAME="beam_python${PY_VERSION}_sdk"
+
opts+=("--sdk_container_image=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME:$MULTIARCH_TAG")
+ fi
fi
if [[ ! -z "$KMS_KEY_NAME" ]]; then
diff --git a/sdks/python/test-suites/dataflow/common.gradle
b/sdks/python/test-suites/dataflow/common.gradle
index f9fd691517c..cbc79585a5f 100644
--- a/sdks/python/test-suites/dataflow/common.gradle
+++ b/sdks/python/test-suites/dataflow/common.gradle
@@ -146,7 +146,13 @@ task postCommitIT {
task postCommitArmIT {
def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
dependsOn 'initializeForDataflowJob'
- dependsOn ":sdks:python:container:py${pyversion}:docker"
+ // When -PusePrebuiltSdkContainer is set, skip building/pushing a multiarch
+ // SDK image and use SDK_CONTAINER_IMAGE from the environment instead (e.g.
+ // gcr.io/apache-beam-testing/beam-sdk/beam_python3.x_sdk:latest from
Snapshots).
+ def usePrebuiltSdkContainer =
project.rootProject.hasProperty(['usePrebuiltSdkContainer'])
+ if (!usePrebuiltSdkContainer) {
+ dependsOn ":sdks:python:container:py${pyversion}:docker"
+ }
doLast {
def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"]
