This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new babf20c99dc Adds validates runner post commit tests for Java streaming
worker on streaming engine (#37872)
babf20c99dc is described below
commit babf20c99dcedc63fa1b3a2a91470644c16c0970
Author: Andrew Crites <[email protected]>
AuthorDate: Tue Mar 17 07:18:40 2026 -0700
Adds validates runner post commit tests for Java streaming worker on
streaming engine (#37872)
Adds a ValidatesRunner post-commit test suite using streaming engine.
Previously we only tested the Java worker on appliance.
---
..._ValidatesRunner_Dataflow_Streaming_Engine.json | 4 +
...a_ValidatesRunner_Dataflow_Streaming_Engine.yml | 96 ++++++++++++++++++++++
runners/google-cloud-dataflow-java/build.gradle | 11 +++
3 files changed, 111 insertions(+)
diff --git
a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.json
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.json
new file mode 100644
index 00000000000..e623d3373a9
--- /dev/null
+++
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.json
@@ -0,0 +1,4 @@
+{
+ "comment": "Modify this file in a trivial way to cause this test suite to
run!",
+ "modification": 1,
+}
diff --git
a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.yml
b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.yml
new file mode 100644
index 00000000000..99f67b3fb31
--- /dev/null
+++
b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.yml
@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PostCommit Java ValidatesRunner Dataflow Streaming Engine
+
+on:
+ schedule:
+ - cron: '30 4/8 * * *'
+ pull_request_target:
+ paths: ['release/trigger_all_tests.json',
'.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine.json']
+ workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+ group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha
|| github.head_ref || github.ref }}-${{ github.event.schedule ||
github.event.comment.id || github.event.sender.login }}'
+ cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions
which are `write-all` in case of pull_request_target event
+permissions:
+ actions: write
+ pull-requests: write
+ checks: write
+ contents: read
+ deployments: read
+ id-token: none
+ issues: write
+ discussions: read
+ packages: read
+ pages: read
+ repository-projects: read
+ security-events: read
+ statuses: read
+
+env:
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+ GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+ GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+ beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming:
+ name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+ runs-on: [self-hosted, ubuntu-20.04, main]
+ timeout-minutes: 720
+ strategy:
+ matrix:
+ job_name:
[beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_Engine]
+ job_phrase: [Run Dataflow Streaming Engine ValidatesRunner]
+ if: |
+ github.event_name == 'workflow_dispatch' ||
+ github.event_name == 'pull_request_target' ||
+ (github.event_name == 'schedule' && github.repository == 'apache/beam')
||
+ github.event.comment.body == 'Run Dataflow Streaming Engine
ValidatesRunner'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup repository
+ uses: ./.github/actions/setup-action
+ with:
+ comment_phrase: ${{ matrix.job_phrase }}
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+ - name: Setup environment
+ uses: ./.github/actions/setup-environment-action
+ with:
+ java-version: default
+ - name: run validatesRunnerStreaming script
+ uses: ./.github/actions/gradle-command-self-hosted-action
+ with:
+ gradle-command:
:runners:google-cloud-dataflow-java:validatesRunnerStreamingEngine
+ max-workers: 12
+ - name: Archive JUnit Test Results
+ uses: actions/upload-artifact@v4
+ if: ${{ !success() }}
+ with:
+ name: JUnit Test Results
+ path: "**/build/reports/tests/"
+ - name: Publish JUnit Test Results
+ uses: EnricoMi/publish-unit-test-result-action@v2
+ if: always()
+ with:
+ commit: '${{ env.prsha || env.GITHUB_SHA }}'
+ comment_mode: ${{ github.event_name == 'issue_comment' && 'always'
|| 'off' }}
+ files: '**/build/test-results/**/*.xml'
+ large_files: true
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index bca8847fd4a..49de59fac32 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -506,6 +506,17 @@ task validatesRunnerStreamingTagEncodingV2 {
]))
}
+task validatesRunnerStreamingEngine {
+ group = "Verification"
+ description "Validates Dataflow runner forcing streaming mode on streaming
engine"
+
dependsOn(createLegacyWorkerValidatesRunnerTest(validatesRunnerStreamingConfig
+ [
+ name: 'validatesRunnerLegacyWorkerTestStreamingEngine',
+ pipelineOptions: validatesRunnerStreamingConfig.pipelineOptions + [
+ '--experiments=enable_streaming_engine',
+ ],
+ ]))
+}
+
def setupXVR = tasks.register("setupXVR") {
dependsOn buildAndPushDockerJavaContainer
dependsOn buildAndPushDockerPythonContainer
