This is an automated email from the ASF dual-hosted git repository.
scwhittle pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0699f3594e6 Add post submit test for dataflow streaming with windmill
tag encoding v2 (#37603)
0699f3594e6 is described below
commit 0699f3594e65912a311c91c93920930b7f4779b7
Author: Arun Pandian <[email protected]>
AuthorDate: Wed Feb 18 02:07:46 2026 -0800
Add post submit test for dataflow streaming with windmill tag encoding v2
(#37603)
---
...tesRunner_Dataflow_Streaming_TagEncodingV2.json | 4 +
...atesRunner_Dataflow_Streaming_TagEncodingV2.yml | 97 ++++++++++++++++++++++
runners/google-cloud-dataflow-java/build.gradle | 62 +++++++++-----
3 files changed, 140 insertions(+), 23 deletions(-)
diff --git
a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json
new file mode 100644
index 00000000000..552bd813e7f
--- /dev/null
+++
b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json
@@ -0,0 +1,4 @@
+{
+ "comment": "Modify this file in a trivial way to cause this test suite to
run!",
+ "modification": 1
+}
diff --git
a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.yml
b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.yml
new file mode 100644
index 00000000000..cb5aaf92e84
--- /dev/null
+++
b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.yml
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PostCommit Java ValidatesRunner Dataflow Streaming TagEncodingV2
+
+on:
+ schedule:
+ - cron: '30 4/8 * * *'
+ pull_request_target:
+ paths: ['release/trigger_all_tests.json',
'.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json']
+ workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+ group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha
|| github.head_ref || github.ref }}-${{ github.event.schedule ||
github.event.comment.id || github.event.sender.login }}'
+ cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions
which are `write-all` in case of pull_request_target event
+permissions:
+ actions: write
+ pull-requests: write
+ checks: write
+ contents: read
+ deployments: read
+ id-token: none
+ issues: write
+ discussions: read
+ packages: read
+ pages: read
+ repository-projects: read
+ security-events: read
+ statuses: read
+
+env:
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+ GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+ GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+ beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2:
+ name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+ runs-on: [self-hosted, ubuntu-20.04, main]
+ timeout-minutes: 720
+ strategy:
+ matrix:
+ job_name:
[beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2]
+ job_phrase: [Run Java Dataflow Streaming TagEncodingV2 ValidatesRunner]
+ if: |
+ github.event_name == 'workflow_dispatch' ||
+ github.event_name == 'pull_request_target' ||
+ (github.event_name == 'schedule' && github.repository == 'apache/beam')
||
+ github.event.comment.body == 'Run Java Dataflow Streaming TagEncodingV2
ValidatesRunner'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup repository
+ uses: ./.github/actions/setup-action
+ with:
+ comment_phrase: ${{ matrix.job_phrase }}
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+ - name: Setup environment
+ uses: ./.github/actions/setup-environment-action
+ with:
+ java-version: default
+ - name: run validatesRunnerStreamingTagEncodingV2 script
+ uses: ./.github/actions/gradle-command-self-hosted-action
+ with:
+ gradle-command:
:runners:google-cloud-dataflow-java:validatesRunnerStreamingTagEncodingV2
+ max-workers: 12
+ - name: Archive JUnit Test Results
+ uses: actions/upload-artifact@v4
+ if: ${{ !success() }}
+ with:
+ name: JUnit Test Results
+ path: "**/build/reports/tests/"
+ - name: Publish JUnit Test Results
+ uses: EnricoMi/publish-unit-test-result-action@v2
+ if: always()
+ with:
+ commit: '${{ env.prsha || env.GITHUB_SHA }}'
+ comment_mode: ${{ github.event_name == 'issue_comment' && 'always'
|| 'off' }}
+ files: '**/build/test-results/**/*.xml'
+ large_files: true
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index 6e6fa4fd5fb..a4da00c2e14 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -458,34 +458,50 @@ task validatesRunner {
))
}
+def validatesRunnerStreamingConfig = [
+ pipelineOptions: legacyPipelineOptions + ['--streaming'],
+ excludedCategories: [
+ 'org.apache.beam.sdk.testing.UsesCommittedMetrics',
+ 'org.apache.beam.sdk.testing.UsesRequiresTimeSortedInput',
+ ],
+ excludedTests: [
+ // TODO(https://github.com/apache/beam/issues/21472)
+
'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerUsingState',
+ // GroupIntoBatches.withShardedKey not supported on streaming runner v1
+ // https://github.com/apache/beam/issues/22592
+
'org.apache.beam.sdk.transforms.GroupIntoBatchesTest.testWithShardedKeyInGlobalWindow',
+
+ // These tests use static state and don't work with remote execution.
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle',
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful',
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElement',
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElementStateful',
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetup',
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetupStateful',
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundle',
+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful',
+ ]
+]
+
task validatesRunnerStreaming {
group = "Verification"
description "Validates Dataflow runner forcing streaming mode"
- dependsOn(createLegacyWorkerValidatesRunnerTest(
+
dependsOn(createLegacyWorkerValidatesRunnerTest(validatesRunnerStreamingConfig
+ [
name: 'validatesRunnerLegacyWorkerTestStreaming',
- pipelineOptions: legacyPipelineOptions + ['--streaming'],
- excludedCategories: [
- 'org.apache.beam.sdk.testing.UsesCommittedMetrics',
- 'org.apache.beam.sdk.testing.UsesRequiresTimeSortedInput',
- ],
- excludedTests: [
- // TODO(https://github.com/apache/beam/issues/21472)
-
'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerUsingState',
- // GroupIntoBatches.withShardedKey not supported on streaming runner v1
- // https://github.com/apache/beam/issues/22592
-
'org.apache.beam.sdk.transforms.GroupIntoBatchesTest.testWithShardedKeyInGlobalWindow',
+ ]))
+}
- // These tests use static state and don't work with remote execution.
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle',
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful',
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElement',
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElementStateful',
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetup',
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetupStateful',
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundle',
-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful',
-]
- ))
+task validatesRunnerStreamingTagEncodingV2 {
+ group = "Verification"
+ description "Validates Dataflow runner (legacy) with Tag Encoding V2
experiments"
+
dependsOn(createLegacyWorkerValidatesRunnerTest(validatesRunnerStreamingConfig
+ [
+ name: 'validatesRunnerLegacyWorkerTestStreamingTagEncodingV2',
+ pipelineOptions: validatesRunnerStreamingConfig.pipelineOptions + [
+ '--experiments=enable_streaming_engine',
+ '--experiments=enable_streaming_engine_state_tag_encoding_v2',
+ '--experiments=streaming_engine_state_tag_encoding_v2_supported'
+ ],
+ ]))
}
def setupXVR = tasks.register("setupXVR") {
