This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a change to branch 20260609_spotlessVersion
in repository https://gitbox.apache.org/repos/asf/beam.git
discard ab8316a800c revert spotless reformatting and upgrade checks
discard c07140591f7 exclude generated code from checkstyle and declare task
execution order
discard f152d8b9283 try earlier dependency resolution
discard 03bb903543c revert prior changes
discard 1191cf86af0 hopefully fix timeout and run spotlessApply again
discard 92e7dc67100 try a different idea with narrower scope
discard f958bd99532 try some more ideas on spotless timing out
discard 4238d6c87a5 exclude build and gradle files to help with timeouts
discard 67762b31428 remove getRatchetBranch and ratchetFrom from
BeamModulePlugin
discard 710b5a3c2be fix settings
discard 2efee311221 add another mavenCentral block
discard 2593e80a4d5 add repositories
discard af3793aac1b try minimizing checks
discard 9f9e1b3f0c0 centralize resolution and exclude build folders
discard 1470124c411 increase heap memory and change resolution strategy
discard a5926619799 try to limit memory
discard c1e50a99ac2 set java version
discard d6b1dca14e7 exclude hadoop and cassandra and run spotless again
discard 2b73d664e84 Exclude spotless configurations from forced Guava 25.1-jre
dependency in hadoop-file-system
discard 39141e10932 Add mavenCentral repository to runners flink/spark
spotless build files for dependency resolution
discard 04c7b092511 Downgrade com.diffplug.spotless to 7.2.1 for JRE 11 CI
compatibility
discard 500a588055c update spotless to 8.4 and reformat
add 5c34e1c1c85 Exclude tests for batch-dataset (#39014)
add 9e465a7cfd7 fix parsing error for typescript (#38898)
add c81059fb246 Drop Flink 1.17,1.18 support (#39006)
add 466989d50a4 cancel user code with interruption after timeout (#39018)
add 09c962c4aef Fix Milvus ML PreCommit xdist, container startup (#39007)
add b4e023afece Bump nodemailer from 8.0.5 to 9.0.1 in
/scripts/ci/issue-report (#39019)
add 17bed431e05 remove shardedKey (#39024)
add 63bcd3dd2b5 Validate Project of staging bucket (#39008)
add 8ba04741b47 Sanitize logging in FanOutStreamingEngineWorkerHarness
(#39029)
add 974ad17b60d Increase CallTest sleep durations to keep test from being
flaky due to timeout not being observed. (#39016)
add caba399f96e [Dataflow Java] Fix incorrect warning log when specifying
logger override. (#39034)
add 1df57236fe3 Bump github.com/moby/moby/api from 1.54.2 to 1.55.0 in
/sdks (#39031)
add fc370c9fdc8 [FnApi Java] Add support for separate named data streams
to provide bundle isolation (#38863)
add 630a34c3bfa Bump github.com/moby/moby/client from 0.4.1 to 0.5.0 in
/sdks (#39032)
add a21ca397b42 [Iceberg AddFiles] Add NameMapping when creating table
(#39022)
add 5334dc1d92a Add Flink 2.2.1 runner support (#38980)
add 06aa37bdffd use proper table naming in iceberg sql (#39035)
add e948ef62041 Roll forward io expansion service to Java17 (#38974)
add eaba570c569 Bump github.com/golang-cz/devslog from 0.0.16 to 0.0.17 in
/sdks (#39054)
add 54231e89266 try to fix forward tests for java 17 (#39058)
add 9aed894f5d4 Update pandas version upper bound to support python 3.14
(#39056)
add 9868954ce8c Fix playground precommit test failure (#39062)
add c2474960db7 [yaml] - expand jinja method (#38547)
add d031bddfdc9 Bump actions/checkout from 6 to 7 (#39033)
add 117bb893ca8 Bump google.golang.org/api from 0.285.0 to 0.286.0 in
/sdks (#39068)
add 9a9f48b9542 Fix Flink 2.1+ dependency resolution (#39061)
add c77b2d40626 Update Python Dependencies (#39070)
add a467fdc437f Enable precommit yaml xlang test on code push. (#39071)
add f61d0b5eeff Release Note: Rename Dataflow Runner to Dataflow Portable
Runner (#38747)
add f29d9af139a init (#39025)
add 447899e46e9 [mqtt] Add Python Xlang Messaging PostCommit with MQTT
integration tests (#38966)
add cedfd0bb3e8 [ReduceFnRunner] Fix Prefetches (#39082)
add c3d9a731b42 [Dataflow Streaming] Increase stuck commit invalidation
timeout to 1 hour (#39083)
add 24e9ed4e0e9 Bump actions/cache from 4 to 6 (#39081)
add f7f0e1655d5 Bump actions/checkout from 6 to 7 (#39080)
add 0165344ab13 Add UnboundedCountingSource::to for bounded reads from an
UnboundedCountingSource (#39084)
add f33feafb2b2 [Dataflow Java Streaming] Add a timeout to how long
commits will retry to the service. (#39085)
add 857dfc9a875 [mqtt] Fix streaming xlang IT failing the Messaging
PostCommit (#39088)
add 0802263e48e Add a github workflow to publish vllm image. (#39089)
add 8e4ea736213 [Python] Optimize BigQuery copy jobs in file loads using
multi-source copy (#38983)
add 5afb93c8c09 [OpenTelemetry] Turn off gcp otel auth extension by
default. (#38940)
add 4e9e483a51a Fix Flink XVR for Flink 2
add 49fd8745223 Fix Go Flink VR conf path
add 16acfa8a2c6 Update CHANGES.md and website
add f13ccf7afbe Merge pull request #39077 from Abacn/fix-flink-xvr
add 9e28c9d2150 Disable flaky SDF finalization tests
add 6095fd0de62 Merge pull request #39092 from
arunpandianp/disablesdffinalization
add 70b72adffee Fix Install xmllint
add 0aa808dba09 Merge pull request #39094 from apache/cut-release-branch
add cad9f1ebe76 Moving to 2.76.0-SNAPSHOT on master branch.
add beeb715739f Update beam-master (#39095)
add c531095fc40 [GSOC 2026] Adding logic to validate and generate an error
if there are key… (#38992)
add 8d252c4659b update changes log for yaml improvements etc (#39093)
add 3fba9950feb Update beam-master container
add e56ed7718a5 Trigger failed postcommit tests.
add 7454aa6a0bf Merge pull request #39100 from
shunping/update-python-beam-master
add d71c4aca4f4 Fix race condition in DirectRunner executor shutdown
add 0af28db1142 Apply suggested fix
add 23521069470 Merge pull request #39096 from shunping/fix-flaky-rrio-test
add 96733599e6c Fix JsonToRow swallowing downstream errors when runners
fuse transforms. (#39098)
add 8092349ca28 Enable playground tests triggered on cron schedule (#39069)
add 8ecdab8da94 [Iceberg CDC] Import BaseIncrementalChangelogScan (#38823)
add 8c311e45672 [Iceberg CDC] Add utils base, delete reader, and
serializable files (#38821)
add c1898ff68b8 Fix pip ResolutionTooDeep (#39017)
add 9d3694165e1 [Python] Add UnboundedSource SDF wrapper (#19137) (#38724)
add 5eefc931b5d remove infra change (#39104)
add eef26ca8b71 Revert "[Python] Optimize BigQuery copy jobs in file loads
using multi-source copy" (#39106)
add 0c51b0d288b Write append tables can be asynchronous (#39120)
add b642c5cbd0e Fix typo (#39123)
add 56b5a52294d Bump google-cloud-aiplatform (#39113)
add d77d12a47b2 Update cython requirement from <4,>=3.2.5 to >=3.2.6,<4 in
/sdks/python (#39099)
add 5cb2f09298c [Website] Update Python version compatibility table
(#38969)
add 698420027d5 Docs: update Python version support in Python quickstart
(#39122)
add 8ccd1d70c04 including GitHub Actions to schedule daily audits and Add
documentati… (#39116)
add 5a2e98bb712 Fix a flaky test in ApproximateQuantilesTest (#39132)
add e3b4d8c5d0e Fix flaky ML RunInference tests by disabling reshuffle on
beam.Create (#39118)
add 3cca8fa3dac [Dataflow Streaming] Access state/timerinternals via
StepContext (#38991)
add 2e8f75aaf4b Bump actions/checkout from 6 to 7 (#39147)
add b63ea65d6a3 Bump github.com/aws/smithy-go from 1.27.2 to 1.27.3 in
/sdks (#39145)
add 31cd92f5f55 Bump cloud.google.com/go/storage from 1.62.3 to 1.63.0 in
/sdks (#39144)
add edd3bfca48e Fix flaky GcsIOIntegrationTest and handle GCS bucket
creation race conditions (#39130)
add af005ef679c fix post commit yaml test (#39142)
add 31175f9ab82 fix race condition in statesampler with lock for reset
(#39129)
add ccac91ad9b8 add zizmor to existing precommit GHA workflow (#39108)
add f17b370e74a Fix flaky PubSub integration test by deduplicating
messages (#39134)
add 402b41c726d Fix ThreadSanitizer data race and failed RPC calls (#39076)
add 961ed7c31ad [IcebergIO] Fix stale openWriters count in
RecordWriterManager (#39157)
add f909e8a7af9 Use Flink 2.2 container for load tests
add 984df275bf4 debug only change -- preserve cluster to check error
add 397d9c61a27 Set configs for Flink 2
add a0fd9502818 Refactor Flink Load Tests to use native Dataproc
components and Auto Zone
add 3e76ff518fa Remove unnecessary gcloud volume mount from Job Server
add 5d28c414f62 Change memory allocation, use resize feature
add 30a5a00d584 Merge pull request #39154 from apache/flink-load-test-ver
add f9171da59bc Bump github.com/nats-io/nats-server/v2 from 2.14.2 to
2.14.3 in /sdks (#39168)
add aa6942a9853 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#39169)
add fcd105de015 Bump js-yaml from 3.14.2 to 3.15.0 in /website/www (#39172)
add 24a2ccc830a Bump com.gradle.common-custom-user-data-gradle-plugin
(#39166)
add 1c4b2047016 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#39170)
add aec5b5da813 Add embedded NVIDIA Dynamo support to vLLM ModelHandler
(#38701)
add 8f60e69177b Fixed DirectRunner PubSub subscriber client lifecycle
(#39079)
add ed0a5832eb1 Add open telemetry dependency to dataflow worker (#38941)
add a86e6115270 Retry starting SubprocessServer on failure (#39159)
add e34cc96df4d [yaml] - mongodb read normalization (#38772)
add 78fdc959cda [Dataflow Streaming] [Multi Key] Prepare
StreamingModeExecutionContext and StreamingWorkScheduler for multi-key
execution. (#38814)
add aec0392a343 Revert previous workarounds and upgrade TFX stack to
1.21.0 (#39164)
add e69933fd09c update spotless to 8.4 and reformat
add 99778c1d416 Downgrade com.diffplug.spotless to 7.2.1 for JRE 11 CI
compatibility
add d407cd380d0 Add mavenCentral repository to runners flink/spark
spotless build files for dependency resolution
add f75fb13bb09 Exclude spotless configurations from forced Guava 25.1-jre
dependency in hadoop-file-system
add 160b2f6271d exclude hadoop and cassandra and run spotless again
add 51994f26941 set java version
add 6b4297587b2 try to limit memory
add 07b09bfd424 increase heap memory and change resolution strategy
add 042bdfc771e centralize resolution and exclude build folders
add 6a7e0b8e57d try minimizing checks
add 0d689eaa2ef add repositories
add e5f57f6768b add another mavenCentral block
add 3699461ac7a fix settings
add 01c02d2ac9b remove getRatchetBranch and ratchetFrom from
BeamModulePlugin
add 51256dd8354 exclude build and gradle files to help with timeouts
add 330a120adf2 try some more ideas on spotless timing out
add 168fee6a1a9 try a different idea with narrower scope
add b5927374835 hopefully fix timeout and run spotlessApply again
add 4643d218da0 revert prior changes
add 099157b26e2 try earlier dependency resolution
add b12aadb213d exclude generated code from checkstyle and declare task
execution order
add 51126536524 revert spotless reformatting and upgrade checks
add c5ae8bf76a3 update spotless workflow
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ab8316a800c)
\
N -- N -- N refs/heads/20260609_spotlessVersion (c5ae8bf76a3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
.../test-properties.json | 2 +-
.../actions/setup-environment-action/action.yml | 2 +-
.../trigger_files/beam_PostCommit_Go_VR_Flink.json | 2 +-
.../beam_PostCommit_Java_DataflowV2.json | 2 +-
.../beam_PostCommit_Java_Nexmark_Flink.json | 2 +-
....json => beam_PostCommit_Java_Tpcds_Flink.json} | 0
...it_Java_ValidatesRunner_Dataflow_Streaming.json | 2 +-
..._ValidatesRunner_Dataflow_Streaming_Engine.json | 2 +-
...ostCommit_Java_ValidatesRunner_Dataflow_V2.json | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Spark.json | 18 +-
.github/trigger_files/beam_PostCommit_Python.json | 6 +-
.../beam_PostCommit_Python_Examples_Dataflow.json | 4 +-
..._PostCommit_Python_Xlang_Messaging_Direct.json} | 2 +-
.../trigger_files/beam_PostCommit_XVR_Flink.json | 2 +-
.../trigger_files/beam_PostCommit_XVR_Samza.json | 1 -
.../beam_PostCommit_Yaml_Xlang_Direct.json | 2 +-
.../trigger_files/beam_PreCommit_Python_ML.json | 2 +-
.github/workflows/IO_Iceberg_Integration_Tests.yml | 2 +-
.../IO_Iceberg_Integration_Tests_Dataflow.yml | 2 +-
..._Iceberg_Managed_Integration_Tests_Dataflow.yml | 2 +-
.github/workflows/IO_Iceberg_Performance_Tests.yml | 2 +-
.github/workflows/IO_Iceberg_Unit_Tests.yml | 2 +-
.github/workflows/README.md | 3 +
.github/workflows/assign_milestone.yml | 2 +-
.github/workflows/beam_CancelStaleDataflowJobs.yml | 2 +-
.../workflows/beam_CleanUpDataprocResources.yml | 2 +-
.github/workflows/beam_CleanUpGCPResources.yml | 2 +-
.../workflows/beam_CleanUpPrebuiltSDKImages.yml | 2 +-
.../workflows/beam_CloudML_Benchmarks_Dataflow.yml | 2 +-
.../beam_IODatastoresCredentialsRotation.yml | 2 +-
.../beam_Inference_Python_Benchmarks_Dataflow.yml | 2 +-
.../beam_Infrastructure_AuditUnmanagedKeys.yml | 69 +
.../beam_Infrastructure_PolicyEnforcer.yml | 2 +-
.../beam_Infrastructure_SecurityLogging.yml | 2 +-
.../beam_Infrastructure_ServiceAccountKeys.yml | 2 +-
.../beam_Infrastructure_UsersPermissions.yml | 2 +-
.github/workflows/beam_Java_JMH.yml | 2 +-
.../beam_Java_LoadTests_Combine_Smoke.yml | 2 +-
.../beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_CoGBK_Flink_batch.yml | 10 +-
.../beam_LoadTests_Go_Combine_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_Combine_Flink_Batch.yml | 16 +-
.../beam_LoadTests_Go_GBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_GBK_Flink_Batch.yml | 16 +-
.../beam_LoadTests_Go_ParDo_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_ParDo_Flink_Batch.yml | 8 +-
.../beam_LoadTests_Go_SideInput_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Go_SideInput_Flink_Batch.yml | 10 +-
.../beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml | 2 +-
...s_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml | 2 +-
...va_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml | 2 +-
...s_Java_CoGBK_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_Combine_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Java_Combine_Dataflow_Streaming.yml | 2 +-
...Java_Combine_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_Batch.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml | 2 +-
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 2 +-
...am_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml | 2 +-
...Tests_Java_GBK_Dataflow_V2_Streaming_Java17.yml | 2 +-
.../workflows/beam_LoadTests_Java_GBK_Smoke.yml | 2 +-
...sts_Java_GBK_SparkStructuredStreaming_Batch.yml | 2 +-
.../beam_LoadTests_Java_ParDo_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Java_ParDo_Dataflow_Streaming.yml | 2 +-
...s_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml | 2 +-
...va_ParDo_Dataflow_V2_Streaming_JavaVersions.yml | 2 +-
...s_Java_ParDo_SparkStructuredStreaming_Batch.yml | 2 +-
.github/workflows/beam_LoadTests_Java_PubsubIO.yml | 2 +-
.../beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Python_CoGBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 8 +-
...eam_LoadTests_Python_Combine_Dataflow_Batch.yml | 2 +-
...LoadTests_Python_Combine_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_Combine_Flink_Batch.yml | 18 +-
...am_LoadTests_Python_Combine_Flink_Streaming.yml | 8 +-
...LoadTests_Python_FnApiRunner_Microbenchmark.yml | 2 +-
.../beam_LoadTests_Python_GBK_Dataflow_Batch.yml | 2 +-
...eam_LoadTests_Python_GBK_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_GBK_Flink_Batch.yml | 18 +-
...adTests_Python_GBK_reiterate_Dataflow_Batch.yml | 2 +-
...sts_Python_GBK_reiterate_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_ParDo_Dataflow_Batch.yml | 2 +-
...m_LoadTests_Python_ParDo_Dataflow_Streaming.yml | 2 +-
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 10 +-
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 10 +-
...m_LoadTests_Python_SideInput_Dataflow_Batch.yml | 2 +-
.github/workflows/beam_LoadTests_Python_Smoke.yml | 2 +-
.../workflows/beam_MetricsCredentialsRotation.yml | 2 +-
.github/workflows/beam_Metrics_Report.yml | 2 +-
.../workflows/beam_PerformanceTests_AvroIOIT.yml | 2 +-
.../beam_PerformanceTests_AvroIOIT_HDFS.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Avro.yml | 2 +-
...PerformanceTests_BigQueryIO_Batch_Java_Json.yml | 2 +-
..._PerformanceTests_BigQueryIO_Streaming_Java.yml | 2 +-
...eam_PerformanceTests_BiqQueryIO_Read_Python.yml | 2 +-
...formanceTests_BiqQueryIO_Write_Python_Batch.yml | 2 +-
.github/workflows/beam_PerformanceTests_Cdap.yml | 2 +-
.../beam_PerformanceTests_Compressed_TextIOIT.yml | 2 +-
...m_PerformanceTests_Compressed_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_HadoopFormat.yml | 2 +-
.github/workflows/beam_PerformanceTests_JDBC.yml | 2 +-
.../workflows/beam_PerformanceTests_Kafka_IO.yml | 2 +-
.../beam_PerformanceTests_ManyFiles_TextIOIT.yml | 2 +-
...am_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_MongoDBIO_IT.yml | 2 +-
.../beam_PerformanceTests_ParquetIOIT.yml | 2 +-
.../beam_PerformanceTests_ParquetIOIT_HDFS.yml | 2 +-
...erformanceTests_PubsubIOIT_Python_Streaming.yml | 2 +-
...m_PerformanceTests_SQLBigQueryIO_Batch_Java.yml | 2 +-
.../beam_PerformanceTests_SingleStoreIO.yml | 2 +-
..._PerformanceTests_SpannerIO_Read_2GB_Python.yml | 2 +-
...manceTests_SpannerIO_Write_2GB_Python_Batch.yml | 2 +-
.../beam_PerformanceTests_SparkReceiver_IO.yml | 2 +-
.../beam_PerformanceTests_TFRecordIOIT.yml | 2 +-
.../beam_PerformanceTests_TFRecordIOIT_HDFS.yml | 2 +-
.../workflows/beam_PerformanceTests_TextIOIT.yml | 2 +-
.../beam_PerformanceTests_TextIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_TextIOIT_Python.yml | 2 +-
...PerformanceTests_WordCountIT_PythonVersions.yml | 2 +-
.../workflows/beam_PerformanceTests_XmlIOIT.yml | 2 +-
.../beam_PerformanceTests_XmlIOIT_HDFS.yml | 2 +-
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 7 +-
.github/workflows/beam_Playground_CI_Nightly.yml | 2 +-
.github/workflows/beam_Playground_Precommit.yml | 5 +-
.github/workflows/beam_PostCommit_Go.yml | 2 +-
.../workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +-
.github/workflows/beam_PostCommit_Go_VR_Flink.yml | 2 +-
.github/workflows/beam_PostCommit_Go_VR_Spark.yml | 2 +-
.github/workflows/beam_PostCommit_Java.yml | 2 +-
.../beam_PostCommit_Java_Avro_Versions.yml | 2 +-
.../beam_PostCommit_Java_BigQueryEarlyRollout.yml | 2 +-
.../workflows/beam_PostCommit_Java_DataflowV1.yml | 2 +-
.../workflows/beam_PostCommit_Java_DataflowV2.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 2 +-
...beam_PostCommit_Java_Examples_Dataflow_Java.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_V2.yml | 2 +-
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 2 +-
.../beam_PostCommit_Java_Examples_Direct.yml | 2 +-
.../beam_PostCommit_Java_Examples_Flink.yml | 4 +-
.../beam_PostCommit_Java_Examples_Spark.yml | 2 +-
.../beam_PostCommit_Java_Hadoop_Versions.yml | 2 +-
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 4 +-
.../beam_PostCommit_Java_InfluxDbIO_IT.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Dataflow.yml | 2 +-
...beam_PostCommit_Java_Jpms_Dataflow_Versions.yml | 2 +-
.../workflows/beam_PostCommit_Java_Jpms_Direct.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Versions.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Flink_Java11.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Spark_Java11.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Dataflow.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Dataflow_V2.yml | 2 +-
...am_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Direct.yml | 2 +-
.../beam_PostCommit_Java_Nexmark_Flink.yml | 7 +-
.../beam_PostCommit_Java_Nexmark_Spark.yml | 2 +-
.../beam_PostCommit_Java_PVR_Flink_Batch.yml | 4 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.yml | 6 +-
.../beam_PostCommit_Java_PVR_Spark3_Streaming.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark4_Batch.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark4_Streaming.yml | 2 +-
.../beam_PostCommit_Java_PVR_Spark_Batch.yml | 2 +-
.../beam_PostCommit_Java_SingleStoreIO_IT.yml | 2 +-
.../beam_PostCommit_Java_Tpcds_Dataflow.yml | 2 +-
.../workflows/beam_PostCommit_Java_Tpcds_Flink.yml | 4 +-
.../workflows/beam_PostCommit_Java_Tpcds_Spark.yml | 2 +-
...am_PostCommit_Java_ValidatesRunner_Dataflow.yml | 2 +-
..._Java_ValidatesRunner_Dataflow_JavaVersions.yml | 2 +-
...mit_Java_ValidatesRunner_Dataflow_Streaming.yml | 2 +-
...a_ValidatesRunner_Dataflow_Streaming_Engine.yml | 2 +-
...atesRunner_Dataflow_Streaming_TagEncodingV2.yml | 2 +-
...PostCommit_Java_ValidatesRunner_Dataflow_V2.yml | 2 +-
..._Java_ValidatesRunner_Dataflow_V2_Streaming.yml | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Direct.yml | 2 +-
...it_Java_ValidatesRunner_Direct_JavaVersions.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_Flink.yml | 6 +-
.../beam_PostCommit_Java_ValidatesRunner_Spark.yml | 2 +-
...beam_PostCommit_Java_ValidatesRunner_Spark4.yml | 2 +-
...va_ValidatesRunner_SparkStructuredStreaming.yml | 2 +-
...am_PostCommit_Java_ValidatesRunner_Twister2.yml | 2 +-
.../beam_PostCommit_Java_ValidatesRunner_ULR.yml | 2 +-
.github/workflows/beam_PostCommit_Javadoc.yml | 2 +-
.../beam_PostCommit_PortableJar_Flink.yml | 2 +-
.../beam_PostCommit_PortableJar_Spark.yml | 2 +-
.github/workflows/beam_PostCommit_Python.yml | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 2 +-
.../beam_PostCommit_Python_Dependency.yml | 2 +-
.../beam_PostCommit_Python_Examples_Dataflow.yml | 2 +-
.../beam_PostCommit_Python_Examples_Direct.yml | 2 +-
.../beam_PostCommit_Python_Examples_Flink.yml | 2 +-
.../beam_PostCommit_Python_Examples_Spark.yml | 2 +-
.../beam_PostCommit_Python_MongoDBIO_IT.yml | 2 +-
.../beam_PostCommit_Python_Nexmark_Direct.yml | 2 +-
.../beam_PostCommit_Python_Portable_Flink.yml | 2 +-
...stCommit_Python_ValidatesContainer_Dataflow.yml | 2 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 2 +-
..._PostCommit_Python_ValidatesRunner_Dataflow.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Flink.yml | 2 +-
...eam_PostCommit_Python_ValidatesRunner_Spark.yml | 2 +-
.../workflows/beam_PostCommit_Python_Versions.yml | 2 +-
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 6 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.yml | 7 +-
.../beam_PostCommit_Python_Xlang_IO_Dataflow.yml | 6 +-
.../beam_PostCommit_Python_Xlang_IO_Direct.yml | 9 +-
...m_PostCommit_Python_Xlang_Messaging_Direct.yml} | 22 +-
.github/workflows/beam_PostCommit_SQL.yml | 2 +-
.../beam_PostCommit_TransformService_Direct.yml | 9 +-
.github/workflows/beam_PostCommit_Website_Test.yml | 2 +-
.github/workflows/beam_PostCommit_XVR_Direct.yml | 6 +-
.github/workflows/beam_PostCommit_XVR_Flink.yml | 8 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 6 +-
...eam_PostCommit_XVR_JavaUsingPython_Dataflow.yml | 6 +-
..._PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml | 6 +-
...eam_PostCommit_XVR_PythonUsingJava_Dataflow.yml | 6 +-
.github/workflows/beam_PostCommit_XVR_Spark3.yml | 6 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 8 +-
.../workflows/beam_PostRelease_NightlySnapshot.yml | 2 +-
.../workflows/beam_PreCommit_CommunityMetrics.yml | 2 +-
.../workflows/beam_PreCommit_Flink_Container.yml | 10 +-
.github/workflows/beam_PreCommit_GHA.yml | 8 +-
.github/workflows/beam_PreCommit_Go.yml | 2 +-
.github/workflows/beam_PreCommit_GoPortable.yml | 2 +-
.github/workflows/beam_PreCommit_GoPrism.yml | 2 +-
.github/workflows/beam_PreCommit_ItFramework.yml | 2 +-
.github/workflows/beam_PreCommit_Java.yml | 2 +-
...eCommit_Java_Amazon-Web-Services2_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Amqp_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Azure_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Cassandra_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Cdap_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Clickhouse_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Csv_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Datadog_IO_Direct.yml | 2 +-
.github/workflows/beam_PreCommit_Java_Dataflow.yml | 2 +-
.../beam_PreCommit_Java_Debezium_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Delta_IO_Direct.yml | 2 +-
...beam_PreCommit_Java_ElasticSearch_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Examples_Dataflow.yml | 2 +-
...eam_PreCommit_Java_Examples_Dataflow_Java21.yml | 2 +-
...Commit_Java_File-schema-transform_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Flink_Versions.yml | 2 +-
.../beam_PreCommit_Java_GCP_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Google-ads_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_HBase_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_HCatalog_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Hadoop_IO_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Java_IOs_Direct.yml | 2 +-
.../beam_PreCommit_Java_InfluxDb_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_JDBC_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Jms_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Kafka_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Kudu_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_MongoDb_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Mqtt_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Neo4j_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_PVR_Flink_Batch.yml | 4 +-
.../beam_PreCommit_Java_PVR_Flink_Docker.yml | 4 +-
.../beam_PreCommit_Java_PVR_Prism_Loopback.yml | 2 +-
.../beam_PreCommit_Java_Parquet_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Pulsar_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_RabbitMq_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Redis_IO_Direct.yml | 2 +-
...am_PreCommit_Java_RequestResponse_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_SingleStore_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Snowflake_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Solace_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Solr_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Spark_Versions.yml | 2 +-
.../beam_PreCommit_Java_Splunk_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Thrift_IO_Direct.yml | 2 +-
.../beam_PreCommit_Java_Tika_IO_Direct.yml | 2 +-
.../workflows/beam_PreCommit_Kotlin_Examples.yml | 2 +-
.../workflows/beam_PreCommit_Portable_Python.yml | 2 +-
.github/workflows/beam_PreCommit_Prism_Python.yml | 2 +-
.github/workflows/beam_PreCommit_Python.yml | 2 +-
.github/workflows/beam_PreCommit_PythonDocker.yml | 2 +-
.github/workflows/beam_PreCommit_PythonDocs.yml | 2 +-
.../workflows/beam_PreCommit_PythonFormatter.yml | 2 +-
.github/workflows/beam_PreCommit_PythonLint.yml | 2 +-
.../workflows/beam_PreCommit_Python_Coverage.yml | 2 +-
.../workflows/beam_PreCommit_Python_Dataframes.yml | 2 +-
.github/workflows/beam_PreCommit_Python_Dill.yml | 2 +-
.../workflows/beam_PreCommit_Python_Examples.yml | 2 +-
.../beam_PreCommit_Python_Integration.yml | 2 +-
.github/workflows/beam_PreCommit_Python_ML.yml | 2 +-
.../workflows/beam_PreCommit_Python_PVR_Flink.yml | 2 +-
.../workflows/beam_PreCommit_Python_Runners.yml | 2 +-
.../workflows/beam_PreCommit_Python_Transforms.yml | 2 +-
.github/workflows/beam_PreCommit_RAT.yml | 2 +-
.github/workflows/beam_PreCommit_SQL.yml | 2 +-
.github/workflows/beam_PreCommit_SQL_Java17.yml | 2 +-
.github/workflows/beam_PreCommit_Spotless.yml | 20 +-
.github/workflows/beam_PreCommit_Typescript.yml | 2 +-
.github/workflows/beam_PreCommit_Website.yml | 2 +-
.../workflows/beam_PreCommit_Website_Stage_GCS.yml | 2 +-
.github/workflows/beam_PreCommit_Whitespace.yml | 2 +-
.../beam_PreCommit_Xlang_Generated_Transforms.yml | 8 +-
.../workflows/beam_PreCommit_Yaml_Xlang_Direct.yml | 8 +-
.github/workflows/beam_Prober_CommunityMetrics.yml | 2 +-
.github/workflows/beam_Publish_BeamMetrics.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 7 +-
.../workflows/beam_Publish_Docker_Snapshots.yml | 6 +-
...Jobs.yml => beam_Publish_Python_VLLM_Image.yml} | 71 +-
.github/workflows/beam_Publish_Website.yml | 4 +-
.../beam_Python_CostBenchmarks_Dataflow.yml | 2 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 2 +-
.github/workflows/beam_Release_NightlySnapshot.yml | 2 +-
.../beam_Release_Python_NightlySnapshot.yml | 2 +-
.../workflows/beam_StressTests_Java_BigQueryIO.yml | 2 +-
.../workflows/beam_StressTests_Java_BigTableIO.yml | 2 +-
.../workflows/beam_StressTests_Java_KafkaIO.yml | 2 +-
.../workflows/beam_StressTests_Java_PubSubIO.yml | 2 +-
.../workflows/beam_StressTests_Java_SpannerIO.yml | 2 +-
.github/workflows/build_release_candidate.yml | 24 +-
.github/workflows/build_runner_image.yml | 2 +-
.github/workflows/build_wheels.yml | 6 +-
.github/workflows/cancel.yml | 2 +-
.github/workflows/code_completion_plugin_tests.yml | 4 +-
.github/workflows/codeql.yml | 2 +-
.github/workflows/cut_release_branch.yml | 9 +-
.github/workflows/dask_runner_tests.yml | 4 +-
.../workflows/deploy_release_candidate_pypi.yaml | 2 +-
.github/workflows/finalize_release.yml | 6 +-
.github/workflows/flaky_test_detection.yml | 2 +-
.github/workflows/git_tag_released_version.yml | 2 +-
.github/workflows/go_tests.yml | 2 +-
.github/workflows/issue-tagger.yml | 2 +-
.github/workflows/java_tests.yml | 4 +-
.github/workflows/local_env_tests.yml | 4 +-
.github/workflows/playground_frontend_test.yml | 4 +-
.github/workflows/pr-bot-new-prs.yml | 2 +-
.github/workflows/pr-bot-pr-updates.yml | 2 +-
.github/workflows/pr-bot-prs-needing-attention.yml | 2 +-
.github/workflows/publish_github_release_notes.yml | 4 +-
.github/workflows/python_dependency_tests.yml | 2 +-
.github/workflows/python_tests.yml | 8 +-
.github/workflows/refresh_looker_metrics.yml | 2 +-
.github/workflows/reportGenerator.yml | 2 +-
.../republish_released_docker_containers.yml | 2 +-
.github/workflows/run_perf_alert_tool.yml | 2 +-
.../workflows/run_rc_validation_go_wordcount.yml | 2 +-
.../run_rc_validation_java_mobile_gaming.yml | 2 +-
.../run_rc_validation_java_quickstart.yml | 4 +-
.../run_rc_validation_python_mobile_gaming.yml | 2 +-
.../workflows/run_rc_validation_python_yaml.yml | 2 +-
.github/workflows/tour_of_beam_backend.yml | 2 +-
.../workflows/tour_of_beam_backend_integration.yml | 2 +-
.github/workflows/tour_of_beam_frontend_test.yml | 4 +-
.github/workflows/typescript_tests.yml | 18 +-
.github/workflows/update_python_dependencies.yml | 4 +-
.test-infra/dataproc/flink_cluster.sh | 78 +-
.test-infra/dataproc/init-actions/docker.sh | 103 --
.test-infra/dataproc/init-actions/flink.sh | 217 ---
CHANGES.md | 10 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 42 +-
examples/java/common.gradle | 12 +-
gradle.properties | 6 +-
infra/enforcement/README.md | 12 +-
infra/enforcement/account_keys.py | 103 +-
infra/enforcement/sending.py | 105 +-
.../beam/model/fn_execution/v1/beam_fn_api.proto | 38 +-
.../beam/model/pipeline/v1/beam_runner_api.proto | 4 +
.../life_cycle/life_cycle_setuper_test.go | 3 +-
release/build.gradle.kts | 2 +-
.../org/apache/beam/runners/core/DoFnRunners.java | 4 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 17 +-
.../apache/beam/runners/core/ReduceFnRunner.java | 3 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 4 +-
.../apache/beam/runners/core/ReduceFnTester.java | 20 +-
.../direct/ExecutorServiceParallelExecutor.java | 28 +-
runners/flink/1.17/build.gradle | 25 -
runners/flink/1.18/build.gradle | 25 -
.../flink/1.18/job-server-container/build.gradle | 26 -
runners/flink/1.18/job-server/build.gradle | 31 -
.../flink/streaming/MemoryStateBackendWrapper.java | 80 --
.../runners/flink/streaming/StreamSources.java | 61 -
runners/flink/2.1/build.gradle | 10 +-
runners/flink/2.1/job-server/build.gradle | 10 +-
runners/flink/{2.1 => 2.2}/build.gradle | 31 +-
.../job-server-container/build.gradle | 0
.../flink/{1.17 => 2.2}/job-server/build.gradle | 9 +-
.../runners/flink/FlinkExecutionEnvironments.java | 4 +-
.../wrappers/streaming/DoFnOperator.java | 2 +-
.../runners/flink/FlinkPipelineOptionsTest.java | 16 +-
runners/flink/job-server/flink_job_server.gradle | 22 +-
.../wrappers/streaming/WindowDoFnOperator.java | 2 +-
.../flink/streaming/MemoryStateBackendWrapper.java | 28 +-
.../runners/flink/streaming/StreamSources.java | 7 +
runners/google-cloud-dataflow-java/build.gradle | 10 +-
.../options/DataflowStreamingPipelineOptions.java | 19 +-
.../dataflow/worker/GroupAlsoByWindowsParDoFn.java | 2 +-
.../dataflow/worker/StreamingDataflowWorker.java | 8 +
.../worker/StreamingModeExecutionContext.java | 390 +++++-
.../worker/WindmillReaderIteratorBase.java | 29 +-
.../dataflow/worker/WindmillTimerInternals.java | 23 +
.../dataflow/worker/WindowingWindmillReader.java | 119 +-
.../logging/DataflowWorkerLoggingInitializer.java | 12 +-
.../worker/streaming/ComputationWorkExecutor.java | 43 +-
.../runners/dataflow/worker/streaming/Work.java | 29 +-
.../FanOutStreamingEngineWorkerHarness.java | 7 +-
.../windmill/client/grpc/GrpcCommitWorkStream.java | 94 +-
.../client/grpc/GrpcWindmillStreamFactory.java | 10 +
.../windmill/state/WindmillStateInternals.java | 17 +
.../processing/ComputationWorkExecutorFactory.java | 33 +-
.../work/processing/StreamingWorkScheduler.java | 312 +++--
.../worker/StreamingDataflowWorkerTest.java | 36 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 3 +-
.../worker/StreamingModeExecutionContextTest.java | 158 ++-
.../worker/WindmillReaderIteratorBaseTest.java | 92 ++
.../worker/WindowingWindmillReaderTest.java | 275 ++++
.../dataflow/worker/WorkerCustomSourcesTest.java | 51 +-
.../DataflowWorkerLoggingInitializerTest.java | 44 +-
.../dataflow/worker/streaming/WorkTest.java | 132 ++
.../client/grpc/GrpcCommitWorkStreamTest.java | 82 ++
.../fnexecution/control/SdkHarnessClient.java | 2 +-
.../runners/fnexecution/data/FnDataService.java | 3 +-
.../runners/fnexecution/data/GrpcDataService.java | 13 +-
.../fnexecution/data/GrpcDataServiceTest.java | 2 +-
scripts/beam-sql.sh | 2 +-
scripts/ci/issue-report/package-lock.json | 14 +-
scripts/ci/issue-report/package.json | 2 +-
sdks/go.mod | 36 +-
sdks/go.sum | 76 +-
sdks/go/examples/wasm/README.md | 6 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/test/build.gradle | 2 +-
.../sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 2 +-
.../sdk/fn/data/BeamFnDataOutboundAggregator.java | 131 +-
.../org/apache/beam/sdk/io/CountingSource.java | 40 +-
.../apache/beam/sdk/transforms/AsyncWrapper.java | 10 +-
.../org/apache/beam/sdk/transforms/JsonToRow.java | 8 +-
.../beam/sdk/util/construction/Environments.java | 1 +
.../fn/data/BeamFnDataOutboundAggregatorTest.java | 135 +-
.../org/apache/beam/sdk/io/CountingSourceTest.java | 36 +
.../beam/sdk/transforms/AsyncWrapperTest.java | 66 +-
.../apache/beam/sdk/transforms/JsonToRowTest.java | 26 +
sdks/java/expansion-service/container/Dockerfile | 2 +-
.../meta/provider/iceberg/IcebergMetastore.java | 8 +-
.../provider/iceberg/IcebergMetastoreTest.java | 13 +
.../java/org/apache/beam/fn/harness/FnHarness.java | 55 +-
.../fn/harness/control/ProcessBundleHandler.java | 50 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 28 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 91 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 44 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 2 +-
.../PTransformRunnerFactoryTestContext.java | 12 +-
.../harness/control/ProcessBundleHandlerTest.java | 47 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 41 +-
sdks/java/io/expansion-service/build.gradle | 4 +-
.../org/apache/beam/sdk/io/iceberg/AddFiles.java | 39 +-
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 2 +-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 78 +-
.../apache/beam/sdk/io/iceberg/PartitionUtils.java | 100 +-
.../apache/beam/sdk/io/iceberg/ReadFromTasks.java | 4 +-
.../org/apache/beam/sdk/io/iceberg/ReadUtils.java | 137 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 5 +-
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 4 +-
.../beam/sdk/io/iceberg/SerializableDataFile.java | 100 +-
.../sdk/io/iceberg/SerializableDeleteFile.java | 335 +++++
.../beam/sdk/io/iceberg/cdc/CdcOutputUtils.java | 178 +++
.../beam/sdk/io/iceberg/cdc/CdcReadUtils.java | 698 ++++++++++
.../beam/sdk/io/iceberg/cdc/DeleteReader.java | 309 +++++
.../io/iceberg/cdc/IcebergCdcMetadataColumns.java | 93 ++
.../io/iceberg/cdc/SerializableChangelogTask.java | 282 ++++
.../beam/sdk/io/iceberg/cdc}/package-info.java | 4 +-
.../iceberg/BaseIncrementalChangelogScan.java | 1014 ++++++++++++++
.../java/org/apache/iceberg}/package-info.java | 4 +-
.../IcebergCdcReadSchemaTransformProviderTest.java | 12 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 83 +-
.../IcebergSchemaTransformTranslationTest.java | 33 +-
.../beam/sdk/io/iceberg/PartitionUtilsTest.java | 68 +
.../apache/beam/sdk/io/iceberg/ReadUtilsTest.java | 59 +-
.../sdk/io/iceberg/SerializableDataFileTest.java | 3 +
.../sdk/io/iceberg/SerializableDeleteFileTest.java | 222 ++++
.../beam/sdk/io/iceberg/TestDataWarehouse.java | 2 +-
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 7 +-
.../beam/sdk/io/iceberg/cdc/CdcReadUtilsTest.java | 381 ++++++
.../beam/sdk/io/iceberg/cdc/DeleteReaderTest.java | 419 ++++++
.../iceberg/cdc/SerializableChangelogTaskTest.java | 256 ++++
sdks/java/io/mongodb/build.gradle | 2 +
.../beam/sdk/io/mongodb/AggregationQuery.java | 9 +-
...> MongoDbReadSchemaTransformConfiguration.java} | 34 +-
.../MongoDbReadSchemaTransformProvider.java | 151 +++
.../apache/beam/sdk/io/mongodb/MongoDbUtils.java | 131 ++
.../MongoDbReadSchemaTransformProviderTest.java | 221 ++++
.../beam/sdk/io/mongodb/MongoDbUtilsTest.java | 123 ++
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 47 +
.../org/apache/beam/io/requestresponse/Call.java | 1 +
.../apache/beam/io/requestresponse/CallTest.java | 53 +-
sdks/java/testing/nexmark/build.gradle | 15 +
sdks/java/testing/tpcds/build.gradle | 15 +
.../controller-container/Dockerfile | 2 +-
sdks/python/apache_beam/dataframe/frames.py | 7 +-
sdks/python/apache_beam/dataframe/frames_test.py | 4 +-
.../examples/inference/vllm_text_completion.py | 24 +-
.../mltransform_one_hot_encoding_test.py | 2 -
.../io/external/xlang_mqttio_it_test.py | 268 ++++
.../apache_beam/io/gcp/bigquery_file_loads.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 61 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 21 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 132 ++
.../apache_beam/io/gcp/pubsub_integration_test.py | 22 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 79 +-
sdks/python/apache_beam/io/iobase.py | 14 +-
sdks/python/apache_beam/io/iobase_test.py | 49 +-
sdks/python/apache_beam/io/unbounded_source.py | 996 ++++++++++++++
.../python/apache_beam/io/unbounded_source_test.py | 1380 ++++++++++++++++++++
sdks/python/apache_beam/ml/inference/base_test.py | 2 +-
.../ml/inference/pytorch_inference_test.py | 5 +-
.../ml/inference/sklearn_inference_test.py | 4 +-
.../ml/inference/tensorflow_inference_test.py | 4 +-
.../ml/inference/tensorrt_inference_test.py | 11 +-
.../inference/test_resources/vllm.dockerfile.old | 14 +-
.../apache_beam/ml/inference/vllm_inference.py | 331 ++++-
.../ml/inference/vllm_inference_test.py | 157 +++
.../ml/rag/enrichment/milvus_search_it_test.py | 1 +
.../ml/rag/ingestion/milvus_search_it_test.py | 1 +
sdks/python/apache_beam/ml/rag/test_utils.py | 1 +
.../python/apache_beam/options/pipeline_options.py | 2 +-
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../runners/direct/transform_evaluator.py | 113 +-
.../runners/worker/statesampler_fast.pyx | 11 +-
.../testing/benchmarks/cloudml/requirements.txt | 16 +-
sdks/python/apache_beam/transforms/async_dofn.py | 4 +-
.../apache_beam/transforms/async_dofn_test.py | 47 +
sdks/python/apache_beam/transforms/stats_test.py | 20 +-
sdks/python/apache_beam/transforms/util_test.py | 49 +
sdks/python/apache_beam/utils/subprocess_server.py | 89 +-
.../apache_beam/utils/subprocess_server_test.py | 22 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/apache_beam/yaml/examples/README.md | 4 +
.../yaml/extended_tests/databases/iceberg.yaml | 54 +-
sdks/python/apache_beam/yaml/integration_tests.py | 74 ++
sdks/python/apache_beam/yaml/main.py | 7 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 13 +-
sdks/python/apache_beam/yaml/tests/mongodb.yaml | 40 +-
sdks/python/apache_beam/yaml/yaml_io.py | 59 +-
sdks/python/apache_beam/yaml/yaml_provider.py | 11 +-
sdks/python/apache_beam/yaml/yaml_transform.py | 56 +-
.../python/apache_beam/yaml/yaml_transform_test.py | 119 ++
.../container/ml/py310/base_image_requirements.txt | 55 +-
.../container/ml/py310/gpu_image_requirements.txt | 73 +-
.../container/ml/py311/base_image_requirements.txt | 57 +-
.../container/ml/py311/gpu_image_requirements.txt | 75 +-
.../container/ml/py312/base_image_requirements.txt | 59 +-
.../container/ml/py312/gpu_image_requirements.txt | 77 +-
.../container/ml/py313/base_image_requirements.txt | 59 +-
.../container/py310/base_image_requirements.txt | 51 +-
.../container/py311/base_image_requirements.txt | 53 +-
.../container/py312/base_image_requirements.txt | 55 +-
.../container/py313/base_image_requirements.txt | 55 +-
.../container/py314/base_image_requirements.txt | 55 +-
sdks/python/pyproject.toml | 2 +-
sdks/python/pytest.ini | 1 +
sdks/python/setup.py | 7 +-
sdks/python/test-suites/dataflow/common.gradle | 48 +-
sdks/python/test-suites/direct/build.gradle | 6 +
sdks/python/test-suites/xlang/build.gradle | 8 +-
sdks/standard_external_transforms.yaml | 34 +
sdks/typescript/package.json | 2 +-
sdks/typescript/src/apache_beam/runners/flink.ts | 2 +-
settings.gradle.kts | 2 +-
website/www/site/assets/js/page-nav.js | 2 +-
.../site/content/en/documentation/runners/flink.md | 14 +-
.../site/content/en/documentation/sdks/python.md | 6 +-
.../www/site/content/en/documentation/sdks/yaml.md | 34 +
.../site/content/en/get-started/quickstart-py.md | 2 +-
website/www/yarn.lock | 6 +-
571 files changed, 13604 insertions(+), 2898 deletions(-)
copy .github/trigger_files/{beam_PostCommit_Java_Nexmark_Flink.json =>
beam_PostCommit_Java_Tpcds_Flink.json} (100%)
copy .github/trigger_files/{beam_PostCommit_SQL.json =>
beam_PostCommit_Python_Xlang_Messaging_Direct.json} (82%)
delete mode 100644 .github/trigger_files/beam_PostCommit_XVR_Samza.json
create mode 100644 .github/workflows/beam_Infrastructure_AuditUnmanagedKeys.yml
copy .github/workflows/{beam_PostCommit_Python_Xlang_IO_Direct.yml =>
beam_PostCommit_Python_Xlang_Messaging_Direct.yml} (84%)
copy .github/workflows/{beam_CancelStaleDataflowJobs.yml =>
beam_Publish_Python_VLLM_Image.yml} (55%)
delete mode 100644 .test-infra/dataproc/init-actions/docker.sh
delete mode 100644 .test-infra/dataproc/init-actions/flink.sh
delete mode 100644 runners/flink/1.17/build.gradle
delete mode 100644 runners/flink/1.18/build.gradle
delete mode 100644 runners/flink/1.18/job-server-container/build.gradle
delete mode 100644 runners/flink/1.18/job-server/build.gradle
delete mode 100644
runners/flink/1.19/src/test/java/org/apache/beam/runners/flink/streaming/MemoryStateBackendWrapper.java
delete mode 100644
runners/flink/1.19/src/test/java/org/apache/beam/runners/flink/streaming/StreamSources.java
copy runners/flink/{2.1 => 2.2}/build.gradle (61%)
rename runners/flink/{1.17 => 2.2}/job-server-container/build.gradle (100%)
rename runners/flink/{1.17 => 2.2}/job-server/build.gradle (80%)
copy runners/flink/{2.0 =>
2.2}/src/main/java/org/apache/beam/runners/flink/FlinkExecutionEnvironments.java
(99%)
copy runners/flink/{2.0 =>
2.2}/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/DoFnOperator.java
(99%)
copy runners/flink/{2.0 =>
2.2}/src/test/java/org/apache/beam/runners/flink/FlinkPipelineOptionsTest.java
(92%)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindowingWindmillReaderTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WorkTest.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFile.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/CdcOutputUtils.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/CdcReadUtils.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/DeleteReader.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/IcebergCdcMetadataColumns.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTask.java
copy sdks/java/{core/jmh/src/main/java/org/apache/beam/sdk/jmh/io =>
io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc}/package-info.java
(91%)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java
copy
{runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/state
=> sdks/java/io/iceberg/src/main/java/org/apache/iceberg}/package-info.java
(90%)
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFileTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/CdcReadUtilsTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/DeleteReaderTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTaskTest.java
copy
sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/{MongoDbWriteSchemaTransformConfiguration.java
=> MongoDbReadSchemaTransformConfiguration.java} (69%)
create mode 100644
sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformProvider.java
create mode 100644
sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformProviderTest.java
create mode 100644 sdks/python/apache_beam/io/external/xlang_mqttio_it_test.py
create mode 100644 sdks/python/apache_beam/io/unbounded_source.py
create mode 100644 sdks/python/apache_beam/io/unbounded_source_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/vllm_inference_test.py
