This is an automated email from the ASF dual-hosted git repository.
derrickaw pushed a change to branch 20260609_spotlessVersion
in repository https://gitbox.apache.org/repos/asf/beam.git
omit 1424569706c exclude generated code from checkstyle and declare task
execution order
omit 942d158fcd1 try earlier dependency resolution
omit c987d16bdc7 revert prior changes
omit 374e3949b61 hopefully fix timeout and run spotlessApply again
omit 58c206a8156 try a different idea with narrower scope
omit 61110b143cc try some more ideas on spotless timing out
omit ac4625bff98 exclude build and gradle files to help with timeouts
omit 84d09f8e4dd remove getRatchetBranch and ratchetFrom from
BeamModulePlugin
omit 6675cf55975 fix settings
omit cc0877f992c add another mavenCentral block
omit 51421019829 add repositories
omit c9afc3c1248 try minimizing checks
omit 0baee278c95 centralize resolution and exclude build folders
omit 3f3553bb8df increase heap memory and change resolution strategy
omit b0df55a008e try to limit memory
omit 8b6c9a7e06e set java version
omit 47ebe266106 exclude hadoop and cassandra and run spotless again
omit 4f4e76afb99 Exclude spotless configurations from forced Guava 25.1-jre
dependency in hadoop-file-system
omit 026b69eabda Add mavenCentral repository to runners flink/spark
spotless build files for dependency resolution
omit 6584d728df4 Downgrade com.diffplug.spotless to 7.2.1 for JRE 11 CI
compatibility
omit 69f25398f31 update spotless to 8.4 and reformat
add 135a5593441 Replace ClassLoadingStrategy with custom loading strategy
(#38906)
add a40685ccc62 Don't preempt snapshot runs (#38904)
add 8ea81a9f289 Bump @grpc/grpc-js from 1.14.3 to 1.14.4 in
/sdks/typescript (#38922)
add 6f30f147462 Adds a new agent SKILL for developing new I/O connectors
(#38910)
add 782953e3422 Add Delta Lake source to the Java Managed API (#38902)
add afbbc31369b Implement Asynchronous wrapper for DoFn in Java SDK
(#38609)
add 8d15c63d551 Updates references to the I/O connector skill (#38923)
add 6a30a3d68de Updates CHANGES.md to mention the Delta Lake source
(#38924)
add 03f4f797953 [SQL] Support positional parameters (#38880)
add 5307c7138af Support MAP type in RowJson and fix datetime parsing with
spaces (#38865)
add 56a41422765 SQL DDL documentation (#37539)
add a88686c1594 Add DoFnRunner::finishKey() method (#38454)
add 705db250d51 [Dataflow Streaming] Activate SourceState Finalizers
before submitting workitem to harness threads (#38921)
add e8249afb1f7 Fix wordcount_rust requirements.txt and documentation
(#38877)
add 64bb91099a9 Exclude testSideInputNotReadyTimer from Spark batch suites
(#38939)
add 7d642a024a4 Add Gemini RunInference example notebook (#38943)
add 712d981ea79 update agent skills table (#38818)
add 6ffee9feebb [Infra] Add beam_viewer and beam_writer roles for GSoC
2026 participant (#38933)
add ebde16e3713 Add instrumentation for memory profiling in Python SDK
(#38853)
add 22ac580f827 [#38059] Fix GCS glob matching to support ** and / in
object names (#38099)
add 6c083df5f8e SQL Database DDL & Usability Improvements (#38952)
add 001093e5e0b [mqtt] Add portable MqttIO Read/Write transforms for batch
and streaming (revives #32385) (#38493)
add b3a62ee97c7 Upgrade expansion jar to java17 (#38931)
add 6ed0635429a Spanner: Extend SpannerIO to support Spanner Omni mTLS
setup (#38928)
add 378e10de4ea ClickHouseIO: Add DateTime64 support for sub-second
timestamp precision (#38510)
add 98e29d99fcd Revert "Upgrade expansion jar to java17 (#38931)" (#38965)
add bde545f91e4 Add support for STDDEV_POP and STDDEV_SAMP in VarianceFn
(#38871)
add 77348121ee9 Update BEAM_DEV_SDK_CONTAINER_TAG version (#38963)
add a996a9fb610 Bump protobufjs from 8.4.0 to 8.6.0 in /sdks/typescript
(#38968)
add 2c4822959e1 Bump form-data from 2.5.5 to 2.5.6 in /sdks/typescript
(#38967)
add b7bc937f592 Use Flink 2.0 for load tests (#38279)
add c0cc8897f4a Bump cloud.google.com/go/spanner from 1.91.0 to 1.92.0 in
/sdks (#38976)
add 468ae40dc8e Fix AsyncWrapperTest timeout and flakiness issues. (#38970)
add e55a1f7399b fix failure in opentelemetry-gcp-auth-extension jar
(#38938)
add 33b34574b3c Bump jsonpickle upper bound. (#38769)
add 2c53494f735 [Dataflow Java] Clarify which portions of
DataflowWorkerLoggingOptions are deprecated (#38960)
add 0bc785253eb Fix WriteToPubSub to pass ordering_key to publish() method
(#37345)
add d032f0e3ef4 Add drain support for Dataflow and Flink (#38786)
add c608b4da8de Bump google.golang.org/api from 0.284.0 to 0.285.0 in
/sdks (#38997)
add 9765e29de68 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks
(#38998)
add 6ccd0c9c861 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#38999)
add b9342767cc5 fix sentence (#38989)
add 597f92530f1 address DOM text reinterpreted as HTML alert (#38949)
add 56d4ffdab31 Align gcpauth extension implementation with ability to
turn it off easily while still being added as dependency.
add 2802de39b2e Merge pull request #38846: [OpenTelemetry] Add turning off
to gcpauth extension
add ee92a553e5d Disable public IP for Dataflow Python and Go Validate
runner tests (#38990)
add 1c182507444 Add support for Apache Flink 2.1.3 (#38961)
add 1d703fe904f Make ModelManager import more robust (#38936)
add 849fcf89c81 [Beam SQL Shell] Add Iceberg deps when using icebergio
(#38994)
add a3f9e0e7d96 Add publish_time_field to ReadFromPubSub YAML transform
(#38985)
add 15d00559d75 Bump cloud.google.com/go/bigtable from 1.49.0 to 1.50.0 in
/sdks (#39012)
add 500a588055c update spotless to 8.4 and reformat
add 04c7b092511 Downgrade com.diffplug.spotless to 7.2.1 for JRE 11 CI
compatibility
add 39141e10932 Add mavenCentral repository to runners flink/spark
spotless build files for dependency resolution
add 2b73d664e84 Exclude spotless configurations from forced Guava 25.1-jre
dependency in hadoop-file-system
add d6b1dca14e7 exclude hadoop and cassandra and run spotless again
add c1e50a99ac2 set java version
add a5926619799 try to limit memory
add 1470124c411 increase heap memory and change resolution strategy
add 9f9e1b3f0c0 centralize resolution and exclude build folders
add af3793aac1b try minimizing checks
add 2593e80a4d5 add repositories
add 2efee311221 add another mavenCentral block
add 710b5a3c2be fix settings
add 67762b31428 remove getRatchetBranch and ratchetFrom from
BeamModulePlugin
add 4238d6c87a5 exclude build and gradle files to help with timeouts
add f958bd99532 try some more ideas on spotless timing out
add 92e7dc67100 try a different idea with narrower scope
add 1191cf86af0 hopefully fix timeout and run spotlessApply again
add 03bb903543c revert prior changes
add f152d8b9283 try earlier dependency resolution
add c07140591f7 exclude generated code from checkstyle and declare task
execution order
add 0a03fcc30d2 revert spotless reformatting and upgrade checks
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1424569706c)
\
N -- N -- N refs/heads/20260609_spotlessVersion (0a03fcc30d2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.agent/skills/README.md | 4 +
.../skills/developing-new-io-connectors/SKILL.md | 347 ++++++++
.agent/skills/io-connectors/SKILL.md | 3 +-
.../test-properties.json | 2 +-
.github/trigger_files/beam_PostCommit_Go.json | 2 +-
.github/trigger_files/beam_PostCommit_Python.json | 4 +-
...PostCommit_Python_ValidatesRunner_Dataflow.json | 2 +-
.../beam_LoadTests_Go_CoGBK_Flink_batch.yml | 4 +-
.../beam_LoadTests_Go_Combine_Flink_Batch.yml | 4 +-
.../beam_LoadTests_Go_GBK_Flink_Batch.yml | 4 +-
.../beam_LoadTests_Go_ParDo_Flink_Batch.yml | 4 +-
.../beam_LoadTests_Go_SideInput_Flink_Batch.yml | 4 +-
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 4 +-
.../beam_LoadTests_Python_Combine_Flink_Batch.yml | 4 +-
...am_LoadTests_Python_Combine_Flink_Streaming.yml | 4 +-
.../beam_LoadTests_Python_GBK_Flink_Batch.yml | 4 +-
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 4 +-
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 4 +-
.../workflows/beam_PreCommit_Flink_Container.yml | 4 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 5 -
.../workflows/beam_Publish_Docker_Snapshots.yml | 4 +-
.../run_rc_validation_java_quickstart.yml | 2 +-
.test-infra/dataproc/flink_cluster.sh | 6 +-
.../beam/testinfra/mockapis/echo/v1/Echo.java | 30 -
.test-infra/validate-runner/build.gradle | 13 +
CHANGES.md | 11 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 74 +-
examples/java/common.gradle | 10 +
.../apache/beam/examples/RateLimiterSimple.java | 1 -
.../apache/beam/examples/WindowedWordCount.java | 1 -
.../java/org/apache/beam/examples/WordCount.java | 3 -
.../beam/examples/complete/game/GameStats.java | 1 -
.../beam/examples/complete/game/LeaderBoard.java | 1 -
.../beam/examples/complete/game/UserScore.java | 1 -
.../options/KafkaToPubsubOptions.java | 3 +-
.../apache/beam/examples/snippets/Snippets.java | 4 -
.../subprocess/kernel/SubProcessIOFiles.java | 4 +-
.../examples/complete/game/LeaderBoardTest.java | 1 -
.../ReadFromTwitterDoFn.java | 1 -
.../notebooks/beam-ml/run_inference_gemini.ipynb | 609 ++++++++++++++
gradle.properties | 2 +-
infra/iam/users.yml | 8 +-
.../apache/beam/it/common/PipelineLauncher.java | 3 +-
.../it/gcp/dataflow/DefaultPipelineLauncher.java | 4 +-
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 3 +-
.../model/pipeline/v1/external_transforms.proto | 2 +
.../org/apache/beam/runners/core/DoFnRunner.java | 12 +
.../beam/runners/core/InMemoryTimerInternals.java | 8 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 6 +
...TimeBoundedSplittableProcessElementInvoker.java | 3 +-
.../org/apache/beam/runners/core/ReduceFn.java | 2 -
.../apache/beam/runners/core/ReduceFnRunner.java | 4 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 9 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 3 +-
.../org/apache/beam/runners/core/StateTags.java | 8 +-
.../beam/runners/core/StatefulDoFnRunner.java | 6 +
.../apache/beam/runners/core/TimerInternals.java | 14 +-
.../beam/runners/core/metrics/BoundedTrieData.java | 1 -
.../beam/runners/core/metrics/MetricUpdates.java | 1 -
.../runners/core/metrics/MetricsContainerImpl.java | 32 +-
.../core/metrics/MonitoringInfoConstants.java | 4 +-
.../core/metrics/MonitoringInfoMetricName.java | 4 +-
.../core/metrics/SimpleMonitoringInfoBuilder.java | 1 -
.../SimplePushbackSideInputDoFnRunnerTest.java | 4 +
.../core/metrics/MonitoringInfoTestUtil.java | 8 +-
.../core/triggers/AfterPaneStateMachineTest.java | 1 -
.../beam/runners/direct/CommittedBundle.java | 4 +-
.../beam/runners/direct/DirectTimerInternals.java | 8 +-
.../runners/direct/DirectTransformExecutor.java | 5 +-
.../beam/runners/direct/MultiStepCombine.java | 1 -
.../runners/direct/TransformEvaluatorFactory.java | 3 +-
.../beam/runners/direct/TransformResult.java | 3 +-
.../beam/runners/direct/UncommittedBundle.java | 3 +-
.../beam/runners/direct/WatermarkManager.java | 1 -
.../beam/runners/direct/DirectRunnerTest.java | 3 +-
.../wrappers/streaming/DoFnOperator.java | 18 +-
.../FlinkStreamingPortablePipelineTranslator.java | 4 +-
.../wrappers/streaming/DoFnOperator.java | 18 +-
.../streaming/io/UnboundedSourceWrapper.java | 2 -
runners/flink/{2.0 => 2.1}/build.gradle | 17 +-
.../job-server-container/build.gradle | 0
.../flink/{1.19 => 2.1}/job-server/build.gradle | 15 +-
runners/flink/build.gradle | 15 +-
.../runners/flink/FlinkDetachedRunnerResult.java | 60 +-
.../beam/runners/flink/FlinkJobServerDriver.java | 3 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 4 +-
.../beam/runners/flink/FlinkRunnerResult.java | 6 +
.../FlinkStreamingPortablePipelineTranslator.java | 4 +-
.../flink/metrics/DoFnRunnerWithMetricsUpdate.java | 4 +
.../flink/translation/utils/CheckpointStats.java | 1 -
.../wrappers/streaming/DoFnOperator.java | 18 +-
.../streaming/ExecutableStageDoFnOperator.java | 8 +-
.../streaming/io/UnboundedSourceWrapper.java | 2 -
.../streaming/stableinput/BufferingDoFnRunner.java | 13 +-
.../beam/runners/flink/FlinkRunnerResultTest.java | 92 +++
.../wrappers/streaming/DoFnOperatorTest.java | 3 +
.../streaming/ExecutableStageDoFnOperatorTest.java | 14 +-
.../beam/runners/dataflow/BatchViewOverrides.java | 4 -
.../beam/runners/dataflow/DataflowMetrics.java | 4 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 76 +-
.../beam/runners/dataflow/DataflowRunner.java | 10 +-
.../beam/runners/dataflow/TestDataflowRunner.java | 3 +-
.../options/DataflowPipelineWorkerPoolOptions.java | 24 +-
.../options/DataflowStreamingPipelineOptions.java | 8 +-
.../options/DataflowWorkerLoggingOptions.java | 56 +-
.../runners/dataflow/util/DataflowTemplateJob.java | 3 +-
...DefaultCoderCloudObjectTranslatorRegistrar.java | 2 -
.../beam/runners/dataflow/util/PackageUtil.java | 2 -
.../beam/runners/dataflow/util/PropertyNames.java | 1 -
.../runners/dataflow/DataflowPipelineJobTest.java | 46 ++
.../beam/runners/dataflow/DataflowRunnerTest.java | 7 +-
.../runners/dataflow/util/CloudObjectsTest.java | 2 +-
.../worker/AssignWindowsParDoFnFactory.java | 3 +
.../dataflow/worker/BatchDataflowWorker.java | 14 -
.../worker/BatchModeUngroupingParDoFn.java | 3 +
.../CreateIsmShardKeyAndSortKeyDoFnFactory.java | 3 +
.../dataflow/worker/DataflowMapTaskExecutor.java | 4 +-
.../worker/DataflowPortabilityPCollectionView.java | 3 +-
.../dataflow/worker/DataflowProcessFnRunner.java | 6 +
.../dataflow/worker/DoFnInstanceManagers.java | 1 -
.../runners/dataflow/worker/ForwardingParDoFn.java | 6 +
.../dataflow/worker/GroupAlsoByWindowFnRunner.java | 5 +-
.../dataflow/worker/GroupAlsoByWindowsParDoFn.java | 6 +
.../dataflow/worker/IsmSideInputReader.java | 2 -
.../runners/dataflow/worker/OperationalLimits.java | 2 -
.../worker/PairWithConstantKeyDoFnFactory.java | 3 +
.../dataflow/worker/PartialGroupByKeyParDoFns.java | 6 +
.../beam/runners/dataflow/worker/PubsubReader.java | 3 +-
.../ReifyTimestampAndWindowsParDoFnFactory.java | 3 +
.../worker/RemoveSafeDeltaCounterCell.java | 1 -
.../runners/dataflow/worker/SimpleParDoFn.java | 6 +-
.../dataflow/worker/StreamingDataflowWorker.java | 1 -
.../StreamingKeyedWorkItemSideInputDoFnRunner.java | 6 +
.../StreamingKeyedWorkItemSideInputParDoFn.java | 6 +-
.../worker/StreamingModeExecutionContext.java | 9 +-
.../StreamingPCollectionViewWriterParDoFn.java | 3 +
.../worker/StreamingSideInputDoFnRunner.java | 6 +
.../worker/StreamingStepMetricsContainer.java | 1 -
.../worker/ToIsmRecordForMultimapDoFnFactory.java | 3 +
.../dataflow/worker/UngroupedWindmillReader.java | 3 +-
.../runners/dataflow/worker/ValuesDoFnFactory.java | 3 +
.../dataflow/worker/WindowingWindmillReader.java | 3 +-
.../WorkerCustomSourceOperationExecutor.java | 2 +-
.../runners/dataflow/worker/counters/Counter.java | 45 +-
.../worker/graph/LengthPrefixUnknownCoders.java | 5 +-
.../logging/DataflowWorkerLoggingHandler.java | 1 -
.../logging/DataflowWorkerLoggingInitializer.java | 3 +-
.../streaming/harness/StreamingCounters.java | 2 -
.../dataflow/worker/util/MemoryMonitor.java | 14 +-
.../common/worker/BatchingShuffleEntryReader.java | 1 -
.../util/common/worker/FlattenOperation.java | 3 +-
.../worker/util/common/worker/GroupingTables.java | 1 -
.../worker/util/common/worker/MapTaskExecutor.java | 4 +-
.../worker/util/common/worker/Operation.java | 4 +-
.../worker/util/common/worker/ParDoFn.java | 4 +
.../worker/util/common/worker/ParDoOperation.java | 4 +-
.../worker/util/common/worker/ReadOperation.java | 2 +-
.../util/common/worker/ShuffleReadCounter.java | 1 -
.../worker/SimplePartialGroupByKeyParDoFn.java | 5 +
.../worker/util/common/worker/WorkExecutor.java | 2 +-
.../worker/util/common/worker/WriteOperation.java | 3 +-
.../worker/windmill/client/WindmillStreamPool.java | 4 +-
.../client/getdata/ApplianceGetDataClient.java | 3 +-
.../client/grpc/GrpcDirectGetWorkStream.java | 3 +-
.../client/grpc/GrpcWindmillStreamFactory.java | 4 +-
.../grpc/observers/DirectStreamObserver.java | 4 +-
.../client/grpc/stubs/FailoverChannel.java | 4 -
.../worker/windmill/state/PagingIterable.java | 8 +-
.../worker/windmill/state/ToIterableFunction.java | 9 +-
.../worker/windmill/state/WindmillBag.java | 1 -
.../windmill/state/WindmillStateInternals.java | 1 -
.../worker/windmill/state/WindmillStateReader.java | 19 +-
.../worker/windmill/state/WindmillValue.java | 2 -
.../windmill/state/WindmillWatermarkHold.java | 1 -
.../work/processing/StreamingWorkScheduler.java | 5 +-
.../dataflow/worker/AvroByteReaderFactoryTest.java | 10 +-
.../dataflow/worker/ConcatReaderFactoryTest.java | 2 +-
.../dataflow/worker/InMemoryReaderFactoryTest.java | 2 +-
.../IntrinsicMapTaskExecutorFactoryTest.java | 8 +-
.../worker/IntrinsicMapTaskExecutorTest.java | 11 +-
.../dataflow/worker/IsmSideInputReaderTest.java | 2 +-
.../runners/dataflow/worker/ReaderFactoryTest.java | 5 +-
.../dataflow/worker/ShuffleReaderFactoryTest.java | 6 +-
.../runners/dataflow/worker/SinkRegistryTest.java | 2 +-
.../worker/StreamingDataflowWorkerTest.java | 19 +-
.../worker/StreamingModeExecutionContextTest.java | 2 +-
...eamingPCollectionViewWriterDoFnFactoryTest.java | 2 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 4 +
.../dataflow/worker/WorkerCustomSourcesTest.java | 10 +-
.../graph/LengthPrefixUnknownCodersTest.java | 42 +-
.../logging/DataflowWorkerLoggingHandlerTest.java | 4 +-
.../harness/WindmillStreamSenderTest.java | 9 +-
.../worker/util/BoundedQueueExecutorTest.java | 4 +-
.../dataflow/worker/util/CloudSourceUtilsTest.java | 2 +-
.../worker/util/KeyGroupWorkQueueTest.java | 2 +-
.../dataflow/worker/util/TimerOrElementTest.java | 2 +-
.../util/common/worker/ExecutorTestUtils.java | 2 +-
.../util/common/worker/MapTaskExecutorTest.java | 11 +-
.../util/common/worker/ParDoOperationTest.java | 8 +-
.../common/worker/WorkProgressUpdaterTest.java | 3 +-
.../client/grpc/FakeWindmillGrpcService.java | 3 +-
.../client/grpc/GrpcCommitWorkStreamTest.java | 3 +-
.../processing/StreamingCommitFinalizerTest.java | 4 +-
.../failures/WorkFailureProcessorTest.java | 4 +-
.../work/refresh/ActiveWorkRefresherTest.java | 4 +-
.../artifact/ArtifactStagingService.java | 3 +-
.../fnexecution/control/BundleSplitHandler.java | 3 +-
.../control/DefaultJobBundleFactory.java | 1 -
.../control/ProcessBundleDescriptors.java | 12 +-
.../runners/fnexecution/data/GrpcDataService.java | 5 +-
.../status/BeamWorkerStatusGrpcService.java | 3 +-
.../runners/jobsubmission/InMemoryJobService.java | 6 +-
.../jobsubmission/PortablePipelineJarCreator.java | 4 +-
.../java/org/apache/beam/runners/local/Bundle.java | 3 +-
.../beam/runners/portability/PortableRunner.java | 2 -
.../apache/beam/runners/prism/PrismRegistrar.java | 1 -
runners/spark/build.gradle | 15 +-
runners/spark/job-server/spark_job_server.gradle | 2 +
runners/spark/spark_runner.gradle | 2 +
.../runners/spark/SparkNativePipelineVisitor.java | 4 +-
.../runners/spark/TestSparkPipelineOptions.java | 3 +-
.../translation/batch/Aggregators.java | 3 +-
.../translation/batch/DoFnRunnerFactory.java | 4 +
.../translation/batch/DoFnRunnerWithMetrics.java | 4 +
.../spark/translation/DoFnRunnerWithMetrics.java | 4 +
.../translation/SparkInputDataProcessorTest.java | 4 +
.../beam/runners/twister2/Twister2Runner.java | 1 -
scripts/beam-sql.sh | 73 +-
sdks/go.mod | 12 +-
sdks/go.sum | 24 +-
sdks/go/examples/wasm/README.md | 6 +-
sdks/go/pkg/beam/io/filesystem/gcs/gcs.go | 88 ++-
sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go | 146 ++++
sdks/go/test/build.gradle | 5 +-
.../main/java/org/apache/beam/sdk/Pipeline.java | 1 -
.../java/org/apache/beam/sdk/PipelineResult.java | 13 +
.../java/org/apache/beam/sdk/coders/Coder.java | 1 -
.../java/org/apache/beam/sdk/coders/ZstdCoder.java | 1 -
.../sdk/fn/data/BeamFnDataGrpcMultiplexer.java | 14 +-
.../sdk/fn/data/BeamFnDataInboundObserver.java | 5 +-
.../apache/beam/sdk/fn/server/GrpcFnServer.java | 4 +-
.../apache/beam/sdk/fn/server/ServerFactory.java | 1 -
.../sdk/io/BoundedReadFromUnboundedSource.java | 3 +-
.../org/apache/beam/sdk/io/CompressedSource.java | 4 +-
.../org/apache/beam/sdk/io/CountingSource.java | 4 -
.../apache/beam/sdk/io/DefaultFilenamePolicy.java | 1 -
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 13 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 2 -
.../java/org/apache/beam/sdk/io/FileSystems.java | 3 +-
.../org/apache/beam/sdk/io/OffsetBasedSource.java | 4 +-
.../main/java/org/apache/beam/sdk/io/Source.java | 4 +-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 12 +-
.../main/java/org/apache/beam/sdk/io/TextIO.java | 12 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 5 +-
.../java/org/apache/beam/sdk/io/fs/ResourceId.java | 3 +-
.../apache/beam/sdk/lineage/LineageOptions.java | 3 +-
.../java/org/apache/beam/sdk/metrics/Lineage.java | 4 +-
.../beam/sdk/metrics/MetricsEnvironment.java | 3 +-
.../java/org/apache/beam/sdk/options/Default.java | 2 -
.../beam/sdk/options/ExperimentalOptions.java | 3 +-
.../apache/beam/sdk/options/PipelineOptions.java | 3 +-
.../beam/sdk/options/PortablePipelineOptions.java | 9 +-
.../apache/beam/sdk/options/StreamingOptions.java | 3 +-
.../beam/sdk/schemas/FieldAccessDescriptor.java | 6 +-
.../beam/sdk/schemas/FieldTypeDescriptors.java | 2 -
.../apache/beam/sdk/schemas/FieldValueGetter.java | 3 +-
.../sdk/schemas/FieldValueTypeInformation.java | 7 +-
.../sdk/schemas/GetterBasedSchemaProvider.java | 9 +-
.../java/org/apache/beam/sdk/schemas/Schema.java | 39 +-
.../apache/beam/sdk/schemas/SchemaProvider.java | 9 +-
.../org/apache/beam/sdk/schemas/io/Failure.java | 1 -
.../sdk/schemas/logicaltypes/EnumerationType.java | 1 -
.../apache/beam/sdk/schemas/transforms/Cast.java | 3 +-
.../apache/beam/sdk/schemas/transforms/Join.java | 3 +-
.../beam/sdk/schemas/transforms/RenameFields.java | 1 -
.../apache/beam/sdk/schemas/transforms/Select.java | 1 -
.../beam/sdk/schemas/utils/ByteBuddyUtils.java | 3 +-
.../beam/sdk/schemas/utils/SelectHelpers.java | 1 -
.../java/org/apache/beam/sdk/state/Timers.java | 3 +-
.../java/org/apache/beam/sdk/state/ValueState.java | 3 +-
.../beam/sdk/testing/TestPipelineOptions.java | 3 +-
.../beam/sdk/transforms/ApproximateUnique.java | 16 +-
.../apache/beam/sdk/transforms/AsyncWrapper.java | 779 ++++++++++++++++++
.../apache/beam/sdk/transforms/BatchElements.java | 2 -
.../org/apache/beam/sdk/transforms/Create.java | 1 -
.../apache/beam/sdk/transforms/Deduplicate.java | 1 -
.../java/org/apache/beam/sdk/transforms/DoFn.java | 1 -
.../org/apache/beam/sdk/transforms/DoFnTester.java | 116 +--
.../beam/sdk/transforms/GroupIntoBatches.java | 3 +-
.../java/org/apache/beam/sdk/transforms/ParDo.java | 1 -
.../java/org/apache/beam/sdk/transforms/Top.java | 15 +-
.../java/org/apache/beam/sdk/transforms/View.java | 4 +-
.../java/org/apache/beam/sdk/transforms/Watch.java | 3 +-
.../sdk/transforms/errorhandling/ErrorHandler.java | 3 +-
.../beam/sdk/transforms/join/CoGbkResult.java | 1 -
.../reflect/ByteBuddyDoFnInvokerFactory.java | 10 +-
.../beam/sdk/transforms/reflect/DoFnInvoker.java | 15 +-
.../beam/sdk/transforms/reflect/DoFnSignature.java | 11 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 2 -
.../sdk/transforms/windowing/DefaultTrigger.java | 4 +-
.../org/apache/beam/sdk/util/HistogramData.java | 5 -
.../main/java/org/apache/beam/sdk/util/Holder.java | 3 +-
.../java/org/apache/beam/sdk/util/RowJson.java | 65 +-
.../beam/sdk/util/RowJsonValueExtractors.java | 9 +-
.../util/construction/DefaultArtifactResolver.java | 12 +-
.../beam/sdk/util/construction/Environments.java | 6 +-
.../beam/sdk/util/construction/External.java | 10 +-
.../sdk/util/construction/ExternalTranslation.java | 4 +-
.../util/construction/PTransformTranslation.java | 10 +-
.../sdk/util/construction/ParDoTranslation.java | 9 -
.../beam/sdk/util/construction/SdkComponents.java | 5 +-
.../sdk/util/construction/SplittableParDo.java | 1 -
.../sdk/util/construction/TransformUpgrader.java | 4 +-
.../UnboundedReadFromBoundedSource.java | 7 +-
.../util/construction/graph/ExecutableStage.java | 3 +-
.../graph/ImmutableExecutableStage.java | 3 +-
.../construction/graph/OutputDeduplicator.java | 4 +-
.../util/construction/graph/ProtoOverrides.java | 3 +-
.../construction/graph/SideInputReference.java | 2 -
.../graph/SplittableParDoExpander.java | 3 +-
.../util/construction/graph/TimerReference.java | 1 -
.../construction/graph/UserStateReference.java | 2 -
.../apache/beam/sdk/values/PCollectionList.java | 1 -
.../apache/beam/sdk/values/PCollectionView.java | 4 +-
.../apache/beam/sdk/values/PCollectionViews.java | 1 -
.../java/org/apache/beam/sdk/values/PValues.java | 3 +-
.../java/org/apache/beam/sdk/values/RowUtils.java | 6 +-
.../beam/sdk/values/ValueInSingleWindow.java | 3 +-
.../org/apache/beam/sdk/values/WindowedValue.java | 9 +-
.../org/apache/beam/sdk/values/WindowedValues.java | 4 +-
.../beam/sdk/schemas/transforms/FilterTest.java | 3 +-
.../beam/sdk/schemas/transforms/SelectTest.java | 3 +-
.../TypedSchemaTransformProviderTest.java | 3 +-
.../beam/sdk/transforms/AsyncWrapperTest.java | 877 +++++++++++++++++++++
.../org/apache/beam/sdk/transforms/CreateTest.java | 1 -
.../java/org/apache/beam/sdk/util/RowJsonTest.java | 26 +-
.../util/construction/ExternalTranslationTest.java | 4 +-
.../util/construction/ValidateRunnerXlangTest.java | 1 -
.../graph/GreedyPipelineFuserTest.java | 35 +-
.../construction/graph/GreedyStageFuserTest.java | 45 +-
.../sdk/expansion/service/ExpansionService.java | 4 +-
.../apache/beam/sdk/extensions/avro/io/AvroIO.java | 10 +-
.../beam/sdk/extensions/avro/io/AvroSource.java | 3 +-
.../extensions/avro/schemas/utils/AvroUtils.java | 3 +-
.../beam/sdk/extensions/avro/io/AvroIOTest.java | 4 +-
.../client/accumulators/AccumulatorProvider.java | 1 -
.../euphoria/core/client/type/TypeAware.java | 4 +-
.../euphoria/core/client/type/TypeAwareness.java | 4 +-
.../extensions/euphoria/core/client/util/Fold.java | 4 +-
.../core/client/util/PCollectionLists.java | 4 +-
.../core/translate/BeamAccumulatorProvider.java | 4 +-
.../euphoria/core/translate/FlatMapTranslator.java | 4 +-
.../core/translate/provider/CompositeProvider.java | 4 +-
.../sdk/extensions/euphoria/core/util/IOUtils.java | 4 +-
.../sdk/extensions/gcp/auth/CredentialFactory.java | 3 +-
.../extensions/gcp/auth/GcpCredentialFactory.java | 3 +-
.../sdk/extensions/gcp/options/GcpOptions.java | 17 +-
.../sdk/extensions/gcp/options/GcsOptions.java | 20 +-
.../beam/sdk/extensions/gcp/util/GcsUtil.java | 44 +-
.../beam/sdk/extensions/gcp/util/GcsUtilV1.java | 32 +-
.../beam/sdk/extensions/ml/DLPReidentifyText.java | 1 -
.../beam/sdk/extensions/ml/CloudVisionTest.java | 3 +-
.../opentelemetry-gcp-auth-extension/build.gradle | 1 +
.../opentelemetry/gcp/auth/ConfigurableOption.java | 1 +
...GcpAuthAutoConfigurationCustomizerProvider.java | 196 ++---
...nfigure.spi.AutoConfigurationCustomizerProvider | 16 -
...uthAutoConfigurationCustomizerProviderTest.java | 276 ++++---
.../ordered/ContiguousSequenceRange.java | 8 +-
.../beam/sdk/extensions/ordered/EventExaminer.java | 3 +-
.../ordered/combiner/SequenceRangeAccumulator.java | 8 +-
.../apache/beam/sdk/extensions/ordered/Event.java | 12 +-
.../extensions/protobuf/ProtoBeamConverter.java | 9 +-
.../extensions/protobuf/ProtoByteBuddyUtils.java | 49 +-
.../protobuf/ProtoDynamicMessageSchema.java | 4 +-
.../extensions/python/PythonExternalTransform.java | 4 +-
.../datacatalog/DataCatalogTableProvider.java | 10 +-
.../meta/provider/iceberg/IcebergMetastore.java | 13 +-
.../sql/meta/provider/iceberg/IcebergTable.java | 19 +-
.../meta/provider/iceberg/PubsubToIcebergIT.java | 4 +-
.../sql/src/main/codegen/includes/parserImpls.ftl | 4 +-
.../apache/beam/sdk/extensions/sql/BeamSqlCli.java | 1 -
.../beam/sdk/extensions/sql/SqlTransform.java | 1 -
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 1 -
.../extensions/sql/impl/CalciteQueryPlanner.java | 93 ++-
.../extensions/sql/impl/CatalogManagerSchema.java | 4 +-
.../sdk/extensions/sql/impl/CatalogSchema.java | 1 -
.../sql/impl/UdfImplReflectiveFunctionBase.java | 1 -
.../sdk/extensions/sql/impl/cep/CEPLiteral.java | 3 +-
.../extensions/sql/impl/cep/PatternCondition.java | 3 +-
.../sql/impl/parser/SqlCreateDatabase.java | 4 +-
.../extensions/sql/impl/parser/SqlDdlNodes.java | 7 +-
.../sdk/extensions/sql/impl/rel/BeamRelNode.java | 3 +-
.../sdk/extensions/sql/impl/rel/BeamUnnestRel.java | 1 -
.../impl/transform/BeamBuiltinAggregations.java | 5 +-
.../sql/impl/transform/agg/VarianceFn.java | 38 +-
.../sdk/extensions/sql/meta/catalog/Catalog.java | 3 +-
.../sql/meta/catalog/CatalogManager.java | 3 +-
.../sql/meta/catalog/InMemoryCatalog.java | 12 +-
.../sql/meta/provider/mongodb/MongoDbTable.java | 1 -
.../sql/meta/provider/test/TestBoundedTable.java | 1 -
.../sql/meta/provider/test/TestUnboundedTable.java | 1 -
.../sql/meta/store/InMemoryMetaStore.java | 4 +-
.../{BeamSqlAliasTest => BeamSqlAliasTest.java} | 12 +-
.../sdk/extensions/sql/BeamSqlCliDatabaseTest.java | 52 ++
.../sql/BeamSqlDslAggregationVarianceTest.java | 43 +-
.../extensions/sql/BeamSqlDslParametersTest.java | 78 ++
.../sql/impl/transform/agg/VarianceFnTest.java | 25 +-
.../sql/impl/utils/CalciteUtilsTest.java | 2 +-
.../provider/TestSchemaIOTableProviderWrapper.java | 1 -
.../sql/meta/provider/kafka/KafkaTestTable.java | 1 -
.../beam/sdk/extensions/yaml/YamlTransform.java | 1 -
.../zetasketch/ApproximateCountDistinctTest.java | 3 -
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 5 +-
.../org/apache/beam/fn/harness/HandlesSplits.java | 3 +-
.../fn/harness/control/ExecutionStateSampler.java | 1 -
.../apache/beam/fn/harness/control/Metrics.java | 2 -
.../fn/harness/control/ProcessBundleHandler.java | 10 +-
.../apache/beam/fn/harness/state/BagUserState.java | 9 +-
.../beam/fn/harness/state/MultimapUserState.java | 3 +-
.../fn/harness/state/StateFetchingIterators.java | 3 +-
.../beam/fn/harness/status/MemoryMonitor.java | 14 +-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 4 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 4 +-
.../apache/beam/fn/harness/CombineRunnersTest.java | 1 -
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 4 +-
...bleTruncateSizedRestrictionsDoFnRunnerTest.java | 4 +-
.../fn/harness/state/StateBackedIterableTest.java | 3 +-
.../beam/sdk/io/aws2/dynamodb/DynamoDBIO.java | 3 +-
.../sdk/io/aws2/kinesis/EFOShardSubscriber.java | 1 -
.../io/aws2/kinesis/EFOShardSubscribersPool.java | 3 +-
.../io/aws2/kinesis/WatermarkPolicyFactory.java | 3 +-
.../apache/beam/sdk/io/aws2/options/AwsModule.java | 3 +-
.../apache/beam/sdk/io/aws2/options/S3Options.java | 6 +-
.../org/apache/beam/sdk/io/aws2/sqs/SqsIO.java | 4 +-
.../beam/sdk/io/aws2/common/ObjectPoolTest.java | 3 +-
.../sdk/io/aws2/kinesis/testing/KinesisIOIT.java | 6 +-
.../beam/sdk/io/aws2/s3/S3FileSystemTest.java | 3 +-
.../apache/beam/sdk/io/aws2/s3/S3TestUtils.java | 3 +-
.../apache/beam/sdk/io/azure/cosmos/CosmosIO.java | 3 +-
.../beam/sdk/io/azure/cosmos/CosmosOptions.java | 6 +-
.../sdk/io/azure/blobstore/AzfsResourceId.java | 1 -
.../sdk/io/azure/options/BlobstoreOptions.java | 12 +-
.../org/apache/beam/sdk/io/cassandra/ReadFn.java | 9 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 1 -
.../beam/sdk/io/clickhouse/ClickHouseIO.java | 4 +-
.../beam/sdk/io/clickhouse/ClickHouseWriter.java | 40 +
.../apache/beam/sdk/io/clickhouse/TableSchema.java | 53 +-
.../clickhouse/src/main/javacc/ColumnTypeParser.jj | 27 +
.../beam/sdk/io/clickhouse/ClickHouseIOIT.java | 119 +++
.../sdk/io/clickhouse/ClickHouseWriterTest.java | 141 ++++
.../beam/sdk/io/clickhouse/TableSchemaTest.java | 102 +++
.../beam/sdk/io/common/IOTestPipelineOptions.java | 6 +-
.../io/contextualtextio/ContextualTextIOTest.java | 1 -
.../apache/beam/sdk/io/csv/CsvRowConversions.java | 3 +-
.../apache/beam/io/debezium/OffsetRetainer.java | 3 +-
.../io/debezium/DebeziumIOMySqlConnectorIT.java | 1 -
sdks/java/io/delta/build.gradle | 1 +
.../delta/DeltaReadSchemaTransformProvider.java} | 94 +--
.../org/apache/beam/sdk/io/delta/DeltaIOTest.java | 41 +
.../io/elasticsearch/ElasticsearchIOTestUtils.java | 1 -
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 6 +-
sdks/java/io/expansion-service/build.gradle | 1 -
.../io/common/FileBasedIOTestPipelineOptions.java | 3 +-
.../CsvWriteSchemaTransformFormatProviderTest.java | 3 +-
.../FileReadSchemaTransformFormatProviderTest.java | 3 +-
...FileWriteSchemaTransformFormatProviderTest.java | 9 +-
.../io/fileschematransform/XmlRowAdapterTest.java | 20 +-
.../beam/sdk/io/googleads/GoogleAdsOptions.java | 6 +-
.../beam/sdk/io/gcp/bigquery/AppendClientInfo.java | 3 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtils.java | 1 -
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 3 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 37 +-
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 16 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 22 +-
.../io/gcp/bigquery/BigQueryStorageAvroReader.java | 2 +-
.../gcp/bigquery/BigQueryStorageQuerySource.java | 4 +-
.../gcp/bigquery/BigQueryStorageTableSource.java | 6 +-
.../sdk/io/gcp/bigquery/DynamicDestinations.java | 3 +-
.../StorageApiDynamicDestinationsBeamRow.java | 9 +-
.../StorageApiDynamicDestinationsProto.java | 3 +-
.../StorageApiDynamicDestinationsTableRow.java | 6 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 13 +-
.../bigquery/StorageApiWritesShardedRecords.java | 17 +-
.../io/gcp/bigquery/StreamingInsertsMetrics.java | 1 -
.../sdk/io/gcp/bigquery/TableRowJsonCoder.java | 3 +-
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 21 +-
.../beam/sdk/io/gcp/bigquery/TableSchemaCache.java | 9 +-
.../io/gcp/bigquery/TableSchemaUpdateUtils.java | 3 +-
.../sdk/io/gcp/bigquery/UpgradeTableSchema.java | 3 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 1 -
.../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 21 +-
.../io/gcp/bigtable/BigtableConfigTranslator.java | 3 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 17 +-
.../changestreams/ChangeStreamMetrics.java | 1 -
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 -
.../sdk/io/gcp/firestore/FirestoreOptions.java | 6 +-
.../beam/sdk/io/gcp/firestore/FirestoreV1.java | 6 +-
.../beam/sdk/io/gcp/firestore/RpcQosImpl.java | 1 -
.../beam/sdk/io/gcp/firestore/RpcQosOptions.java | 1 -
.../apache/beam/sdk/io/gcp/healthcare/DicomIO.java | 1 -
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 7 -
.../io/gcp/healthcare/FhirIOPatientEverything.java | 2 -
.../sdk/io/gcp/healthcare/FhirSearchParameter.java | 2 -
.../apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java | 3 -
.../beam/sdk/io/gcp/healthcare/HL7v2Message.java | 1 -
.../sdk/io/gcp/healthcare/HealthcareApiClient.java | 1 -
.../beam/sdk/io/gcp/pubsub/ExternalWrite.java | 6 +
.../sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java | 5 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 11 +-
.../sdk/io/gcp/pubsub/PubsubSchemaIOProvider.java | 3 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 1 -
.../beam/sdk/io/gcp/spanner/ReadOperation.java | 4 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 13 +-
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 37 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 178 ++++-
.../io/gcp/spanner/SpannerTransformRegistrar.java | 30 +
.../changestreams/ChangeStreamsConstants.java | 1 -
.../MetadataSpannerConfigFactory.java | 6 +
.../action/DataChangeRecordAction.java | 4 +-
.../dao/PartitionMetadataAdminDao.java | 10 -
.../dofn/ReadChangeStreamPartitionDoFn.java | 1 -
.../mapper/ChangeStreamRecordMapper.java | 17 +-
.../beam/sdk/io/gcp/testing/TableContainer.java | 3 +-
.../gcp/bigquery/BigQueryIOStorageQueryTest.java | 88 +--
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 4 +-
.../StorageApiDataTriggeredSchemaUpdateIT.java | 3 +-
.../gcp/bigquery/TableRowToStorageApiProtoIT.java | 6 +-
.../gcp/bigquery/providers/BigQueryManagedIT.java | 3 +-
.../io/gcp/bigtable/BigtableSharedClientTest.java | 3 +-
.../io/gcp/datastore/AdaptiveThrottlerTest.java | 2 +-
.../gcp/firestore/BaseFirestoreV1WriteFnTest.java | 7 +-
...storeV1FnBatchWriteWithDeadLetterQueueTest.java | 3 +-
.../FirestoreV1FnBatchWriteWithSummaryTest.java | 3 +-
.../firestore/FirestoreV1FnPartitionQueryTest.java | 3 +-
.../gcp/firestore/FirestoreV1FnRunQueryTest.java | 10 +-
.../beam/sdk/io/gcp/firestore/QueryUtilsTest.java | 3 +-
.../beam/sdk/io/gcp/firestore/RpcQosTest.java | 3 +-
.../sdk/io/gcp/firestore/it/BaseFirestoreIT.java | 5 +-
.../sdk/io/gcp/healthcare/HL7v2IOTestUtil.java | 2 -
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 3 +-
.../beam/sdk/io/gcp/pubsub/PubsubWriteIT.java | 3 +-
.../sdk/io/gcp/spanner/SpannerAccessorTest.java | 33 +-
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 9 +-
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 36 +-
.../beam/sdk/io/gcp/spanner/SpannerTestHelper.java | 110 +++
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 113 +--
.../beam/sdk/io/gcp/spanner/StructUtilsTest.java | 3 +-
.../it/ChangeStreamTestPipelineOptions.java | 3 +-
.../changestreams/it/IntegrationTestEnv.java | 17 +-
.../changestreams/it/SpannerChangeStreamIT.java | 19 +-
...StreamOrderedByTimestampAndTransactionIdIT.java | 10 +-
...nnerChangeStreamOrderedWithinKeyGloballyIT.java | 10 +-
.../it/SpannerChangeStreamOrderedWithinKeyIT.java | 10 +-
.../it/SpannerChangeStreamPlacementTableIT.java | 27 +-
...pannerChangeStreamPlacementTablePostgresIT.java | 18 +-
.../it/SpannerChangeStreamPostgresIT.java | 18 +-
...SpannerChangeStreamTransactionBoundariesIT.java | 10 +-
.../it/SpannerChangeStreamsSchemaTransformIT.java | 4 +
.../beam/sdk/io/hadoop/format/HadoopFormatIO.java | 6 +-
.../hadoop/format/HadoopFormatIOCassandraIT.java | 1 -
.../io/hadoop/format/HadoopFormatIOElasticIT.java | 1 -
.../hadoop/format/HadoopFormatIOElasticTest.java | 1 -
.../io/hadoop/format/HadoopFormatIOReadTest.java | 1 -
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 3 +-
.../iceberg/AssignDestinationsAndPartitions.java | 4 +-
.../apache/beam/sdk/io/iceberg/FilterUtils.java | 1 -
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 10 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 1 -
.../apache/beam/sdk/io/iceberg/ReadUtilsTest.java | 7 +-
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 3 +-
.../apache/beam/sdk/io/influxdb/InfluxDbIO.java | 1 -
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 6 +-
.../beam/sdk/io/jdbc/JdbcSchemaIOProvider.java | 3 +-
.../org/apache/beam/sdk/io/jdbc/SchemaUtil.java | 4 +-
.../ReadFromPostgresSchemaTransformProvider.java | 3 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 10 +-
.../java/org/apache/beam/sdk/io/jms/CommonJms.java | 8 +-
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 4 +-
.../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 4 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 26 +-
.../KafkaIOReadImplementationCompatibility.java | 3 +-
.../org/apache/beam/sdk/io/kafka/KafkaMetrics.java | 3 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 3 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 1 -
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 3 +-
.../messaging-expansion-service}/build.gradle | 23 +-
.../apache/beam/sdk/io/mongodb/UpdateField.java | 3 +-
.../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 7 +-
.../java/org/apache/beam/sdk/io/mqtt/MqttIO.java | 7 +
.../io/mqtt/MqttReadSchemaTransformProvider.java | 150 ++++
.../io/mqtt/MqttWriteSchemaTransformProvider.java | 132 ++++
.../io/mqtt/MqttSchemaTransformProviderTest.java | 252 ++++++
.../java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java | 8 +-
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 4 +-
.../beam/sdk/io/rabbitmq/ExchangeTestPlan.java | 4 +-
.../beam/io/requestresponse/RequestResponseIO.java | 13 +-
.../RedisExternalResourcesRule.java | 3 +-
.../singlestore/SingleStoreDefaultRowMapper.java | 21 +-
.../solace/broker/BasicAuthSempClientFactory.java | 1 -
.../solace/broker/SempBasicAuthClientExecutor.java | 4 +-
.../org/apache/beam/sdk/io/solace/data/Solace.java | 1 -
.../beam/sdk/io/solace/read/WatermarkPolicy.java | 4 +-
.../it/BasicAuthMultipleSempClientFactory.java | 1 -
.../beam/sdk/io/sparkreceiver/HasOffset.java | 3 +-
.../beam/sdk/io/sparkreceiver/SparkConsumer.java | 3 +-
.../sdk/io/sparkreceiver/SparkReceiverIOIT.java | 5 +-
.../beam/sdk/io/splunk/HttpEventPublisher.java | 10 +-
.../sdk/io/splunk/CustomX509TrustManagerTest.java | 7 +-
.../beam/sdk/io/splunk/HttpEventPublisherTest.java | 20 +-
.../sdk/io/synthetic/SyntheticBoundedSource.java | 3 +-
.../beam/sdk/io/thrift/TestThriftStruct.java | 12 +-
.../apache/beam/sdk/io/thrift/TestThriftUnion.java | 12 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 8 +-
.../java/org/apache/beam/sdk/managed/Managed.java | 4 +
.../org/apache/beam/sdk/managed/ManagedTest.java | 6 +-
.../ml/inference/openai/OpenAIModelParameters.java | 1 -
.../sdk/ml/inference/remote/RemoteInference.java | 1 -
.../java/org/apache/beam/sdk/jpmstests/JpmsIT.java | 1 -
.../beam/sdk/loadtests/CoGroupByKeyLoadTest.java | 3 +-
.../apache/beam/sdk/loadtests/LoadTestOptions.java | 15 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 5 -
.../apache/beam/sdk/nexmark/NexmarkOptions.java | 159 ++--
.../sdk/nexmark/queries/AbstractSimulator.java | 3 +-
.../apache/beam/sdk/nexmark/queries/Query10.java | 7 +-
.../apache/beam/sdk/nexmark/queries/Query8.java | 3 +-
.../beam/sdk/nexmark/queries/WinningBids.java | 3 +-
.../testutils/publishing/InfluxDBPublisher.java | 8 +-
.../org/apache/beam/sdk/tpcds/TpcdsOptions.java | 18 +-
.../sdk/transformservice/ExpansionService.java | 6 +-
.../apache_beam/examples/wordcount_rust/README.md | 4 +-
.../examples/wordcount_rust/requirements.txt | 2 +-
sdks/python/apache_beam/io/external/gcp/pubsub.py | 11 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 48 +-
.../apache_beam/io/gcp/pubsub_integration_test.py | 90 +++
sdks/python/apache_beam/io/gcp/pubsub_test.py | 72 ++
sdks/python/apache_beam/ml/inference/base.py | 6 +
.../python/apache_beam/options/pipeline_options.py | 78 +-
.../apache_beam/options/pipeline_options_test.py | 37 +
.../options/pipeline_options_validator.py | 2 +
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
sdks/python/apache_beam/yaml/yaml_io.py | 21 +-
sdks/python/apache_beam/yaml/yaml_io_test.py | 103 +++
sdks/python/build.gradle | 1 +
sdks/python/container/Dockerfile | 3 +
.../container/base_image_requirements_manual.txt | 2 +
sdks/python/container/boot.go | 90 ++-
.../container/ml/py310/base_image_requirements.txt | 44 +-
.../container/ml/py310/gpu_image_requirements.txt | 60 +-
.../container/ml/py311/base_image_requirements.txt | 44 +-
.../container/ml/py311/gpu_image_requirements.txt | 60 +-
.../container/ml/py312/base_image_requirements.txt | 44 +-
.../container/ml/py312/gpu_image_requirements.txt | 60 +-
.../container/ml/py313/base_image_requirements.txt | 44 +-
sdks/python/container/profiler.go | 336 ++++++++
.../container/py310/base_image_requirements.txt | 41 +-
.../container/py311/base_image_requirements.txt | 41 +-
.../container/py312/base_image_requirements.txt | 41 +-
.../container/py313/base_image_requirements.txt | 41 +-
.../container/py314/base_image_requirements.txt | 41 +-
sdks/python/scripts/run_integration_test.sh | 11 +-
sdks/python/setup.py | 2 +-
sdks/python/test-suites/dataflow/common.gradle | 2 +
sdks/python/test-suites/xlang/build.gradle | 3 +-
sdks/standard_expansion_services.yaml | 15 +
sdks/standard_external_transforms.yaml | 57 +-
sdks/typescript/package-lock.json | 58 +-
sdks/typescript/package.json | 4 +-
sdks/typescript/src/apache_beam/runners/flink.ts | 2 +-
settings.gradle.kts | 1 +
website/www/site/assets/js/bootstrap.js | 46 +-
website/www/site/assets/js/bootstrap/alert.js | 7 +-
website/www/site/assets/js/bootstrap/carousel.js | 9 +-
website/www/site/assets/js/bootstrap/collapse.js | 6 +-
website/www/site/assets/js/bootstrap/dropdown.js | 7 +-
website/www/site/assets/js/bootstrap/modal.js | 8 +-
website/www/site/assets/js/bootstrap/tooltip.js | 9 +-
.../site/content/en/documentation/dsls/sql/ddl.md | 344 ++++++++
.../content/en/documentation/dsls/sql/overview.md | 2 +
.../en/documentation/io/built-in/iceberg.md | 2 +-
.../layouts/partials/section-menu/en/sdks.html | 1 +
website/www/site/layouts/shortcodes/tab.html | 22 +
681 files changed, 9111 insertions(+), 2619 deletions(-)
create mode 100644 .agent/skills/developing-new-io-connectors/SKILL.md
create mode 100644 examples/notebooks/beam-ml/run_inference_gemini.ipynb
copy runners/flink/{2.0 => 2.1}/build.gradle (79%)
copy runners/flink/{1.17 => 2.1}/job-server-container/build.gradle (100%)
copy runners/flink/{1.19 => 2.1}/job-server/build.gradle (71%)
create mode 100644
sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/AsyncWrapper.java
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/AsyncWrapperTest.java
delete mode 100644
sdks/java/extensions/opentelemetry-gcp-auth-extension/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider
rename
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/{BeamSqlAliasTest
=> BeamSqlAliasTest.java} (92%)
create mode 100644
sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlDslParametersTest.java
create mode 100644
sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriterTest.java
copy
sdks/java/io/{iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergReadSchemaTransformProvider.java
=>
delta/src/main/java/org/apache/beam/sdk/io/delta/DeltaReadSchemaTransformProvider.java}
(51%)
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTestHelper.java
copy sdks/java/{expansion-service/app =>
io/messaging-expansion-service}/build.gradle (69%)
create mode 100644
sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttReadSchemaTransformProvider.java
create mode 100644
sdks/java/io/mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttWriteSchemaTransformProvider.java
create mode 100644
sdks/java/io/mqtt/src/test/java/org/apache/beam/sdk/io/mqtt/MqttSchemaTransformProviderTest.java
create mode 100644 sdks/python/container/profiler.go
create mode 100644 website/www/site/content/en/documentation/dsls/sql/ddl.md
