This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 153875eaf24 Fix #37833: Use named logger instead of root logger in
transforms/util.py (#37857)
add 2209bd60122 Adds bundleFinalizer support to Dataflow non-portable
worker. (#37723)
add b3d2b06b2a8 [ErrorProne] Fix UnnecessaryMethodReference Check Findings
(#37785)
add c05733a3a4d Handle some cases during infer schema from dataclass
(#37855)
add d16f5d18d2a [ErrorProne] Fix EqualsGetClass warnings and enable Error
Prone check (#37758)
add 7b065555cd0 Fix flakiness in tests by increasing timeout and changing
windowing type. (#37887)
add c10028b2583 Fix UnusedVariable warnings across codebase (#37770)
add 48935a83720 Resolve Error Prone NonCanonicalType warnings across the
codebase (#37772)
add 4163dc752ac [Error Prone] Fix Slf4j formatting errors and enable
Slf4jFormatShouldBeConst check (#37771)
add 4221266ffab [ErrorProne] enable JavaUtilDate,
TimeUnitConversionChecker, InvalidThrows checks (#37791)
add 0afbdf684fb Update Go Version to 1.26.1 (#37897)
add 7620a935280 Fix #37736: Allow composite transforms to use implicit
input chaining (#37861)
add 181995db2c6 Upgrade Docker setup actions to version 4 (#37914)
add 5139ca67b64 Sync clickhouse resource manager client version with IO
(#37918)
add 0e7dd4f3cf7 Add version to beam-sdks-java-io-components
add 866d6f73ddb Merge pull request #37920 from apache/fix-maven-archetypes
add d9a6159d97f Pin docker actions to sha for now
add 0cd35ec0927 Merge pull request #37922 from apache/users/damccorm/shas
add 982c9bd8044 Pin version for setup-qemu-action (#37924)
add ca3421606ee Add Quick Start section to README (#37909)
add 65a896b9780 Fix #37894: Update yaml doc for composite transform
changes (#37901)
add 600b7d9050e Pin gcp actions (#37926)
No new revisions were added by this update.
Summary of changes:
.../test-properties.json | 4 +-
.../actions/setup-environment-action/action.yml | 2 +-
.../arc/images/Dockerfile | 2 +-
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 4 +-
...it_Java_ValidatesRunner_Dataflow_Streaming.json | 2 +-
.github/workflows/beam_CleanUpGCPResources.yml | 2 +-
.../beam_Infrastructure_PolicyEnforcer.yml | 2 +-
.../beam_Infrastructure_SecurityLogging.yml | 2 +-
.../beam_Infrastructure_ServiceAccountKeys.yml | 2 +-
.../beam_Infrastructure_UsersPermissions.yml | 2 +-
.github/workflows/beam_Playground_Precommit.yml | 2 +-
.github/workflows/beam_PostCommit_Go.yml | 2 +-
.../workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 6 +-
...stCommit_Python_ValidatesContainer_Dataflow.yml | 4 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 4 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 2 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 2 +-
.../workflows/beam_PreCommit_CommunityMetrics.yml | 2 +-
.github/workflows/beam_PreCommit_PythonDocker.yml | 2 +-
.../workflows/beam_PreCommit_Website_Stage_GCS.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 6 +-
.github/workflows/beam_Publish_Website.yml | 2 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 6 +-
.github/workflows/build_release_candidate.yml | 6 +-
.github/workflows/build_runner_image.yml | 2 +-
.github/workflows/build_wheels.yml | 2 +-
.github/workflows/finalize_release.yml | 2 +-
.github/workflows/refresh_looker_metrics.yml | 4 +-
.../republish_released_docker_containers.yml | 6 +-
.../run_rc_validation_python_mobile_gaming.yml | 2 +-
.../workflows/run_rc_validation_python_yaml.yml | 2 +-
CHANGES.md | 2 +
README.md | 19 ++-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 10 +-
dev-support/docker/Dockerfile | 2 +-
.../IcebergRestCatalogStreamingWriteExample.java | 2 +-
.../beam/examples/SchemaTransformExample.java | 2 +-
.../apache/beam/examples/SqlTransformExample.java | 2 +-
.../beam/examples/ApproximateQuantilesExample.java | 2 +-
.../beam/examples/CoCombineTransformExample.java | 2 +-
.../apache/beam/examples/CoGroupByKeyExample.java | 2 +-
.../org/apache/beam/examples/CombineExample.java | 2 +-
.../org/apache/beam/examples/CountExample.java | 2 +-
.../apache/beam/examples/CountPerKeyExample.java | 2 +-
.../org/apache/beam/examples/CreateExample.java | 2 +-
.../apache/beam/examples/DebuggingWordCount.java | 4 +-
.../org/apache/beam/examples/DistinctExample.java | 2 +-
.../beam/examples/FlatMapElementsExample.java | 2 +-
.../beam/examples/GroupIntoBatchesExample.java | 2 +-
.../java/org/apache/beam/examples/KeysExample.java | 2 +-
.../org/apache/beam/examples/KvSwapExample.java | 2 +-
.../org/apache/beam/examples/LatestExample.java | 2 +-
.../apache/beam/examples/MapElementsExample.java | 2 +-
.../java/org/apache/beam/examples/MaxExample.java | 2 +-
.../org/apache/beam/examples/MaxPerKeyExample.java | 2 +-
.../java/org/apache/beam/examples/MeanExample.java | 2 +-
.../apache/beam/examples/MeanPerKeyExample.java | 2 +-
.../java/org/apache/beam/examples/MinExample.java | 2 +-
.../org/apache/beam/examples/MinPerKeyExample.java | 2 +-
.../org/apache/beam/examples/PartitionExample.java | 2 +-
.../apache/beam/examples/RateLimiterSimple.java | 2 +-
.../org/apache/beam/examples/RegexExample.java | 2 +-
.../org/apache/beam/examples/SampleExample.java | 2 +-
.../java/org/apache/beam/examples/SumExample.java | 2 +-
.../org/apache/beam/examples/SumPerKeyExample.java | 2 +-
.../org/apache/beam/examples/ToStringExample.java | 2 +-
.../java/org/apache/beam/examples/TopExample.java | 2 +-
.../org/apache/beam/examples/ValuesExample.java | 2 +-
.../java/org/apache/beam/examples/ViewExample.java | 2 +-
.../org/apache/beam/examples/WindowExample.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../datatokenization/utils/CsvConverters.java | 2 +-
.../datatokenization/utils/SchemasUtils.java | 5 +-
.../beam/examples/complete/game/GameStats.java | 33 ++--
.../beam/examples/complete/game/LeaderBoard.java | 16 +-
.../examples/complete/game/StatefulTeamScore.java | 12 +-
.../beam/examples/complete/game/UserScore.java | 4 +-
.../complete/kafkatopubsub/KafkaToPubsub.java | 10 +-
.../cookbook/BigQueryStreamingTornadoes.java | 2 +-
.../beam/examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/MinimalBigQueryTornadoes.java | 2 +-
.../subprocess/kernel/SubProcessIOFiles.java | 9 +-
.../subprocess/kernel/SubProcessKernel.java | 10 +-
.../subprocess/utils/CallingSubProcessUtils.java | 2 +-
.../beam/examples/subprocess/utils/FileUtils.java | 25 ++-
.../apache/beam/examples/WindowedWordCountIT.java | 7 +-
.../java/org/apache/beam/examples/WordCountIT.java | 6 +-
.../org/apache/beam/examples/complete/TfIdfIT.java | 6 +-
.../examples/complete/TopWikipediaSessionsIT.java | 7 +-
.../beam/examples/cookbook/DistinctExampleIT.java | 6 +-
.../beam/examples/cookbook/JoinExamplesIT.java | 6 +-
.../ReadFromTwitterDoFn.java | 5 +-
.../twitterstreamgenerator/TwitterConfig.java | 2 +-
.../ReadFromTwitterDoFnTest.java | 1 +
it/clickhouse/build.gradle | 7 +-
.../beam/it/common/utils/IORedirectUtil.java | 2 +-
.../it/gcp/dataflow/DefaultPipelineLauncher.java | 2 +-
.../beam/it/gcp/dataflow/DirectRunnerClient.java | 4 +-
.../beam/it/jdbc/AbstractJDBCResourceManager.java | 2 +-
.../cloudbuild/playground_ci_examples.sh | 2 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 16 +-
.../construction/SerializablePipelineOptions.java | 2 +-
.../beam/runners/core/metrics/BoundedTrieData.java | 4 +-
.../beam/runners/core/metrics/MetricsLogger.java | 6 +-
.../runners/core/metrics/SimpleExecutionState.java | 2 +-
.../runners/core/MergingActiveWindowSetTest.java | 6 +-
...BoundedSplittableProcessElementInvokerTest.java | 8 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 3 +-
.../runners/core/SplittableParDoProcessFnTest.java | 17 +-
.../apache/beam/runners/direct/DirectMetrics.java | 4 +-
...LifecycleManagerRemovingTransformEvaluator.java | 2 +-
...cycleManagerRemovingTransformEvaluatorTest.java | 2 +-
.../runners/direct/DoFnLifecycleManagerTest.java | 2 +-
.../runners/direct/DoFnLifecycleManagersTest.java | 4 +-
.../runners/flink/FlinkExecutionEnvironments.java | 4 +-
.../FlinkStreamingPortablePipelineTranslator.java | 5 +-
.../translation/types/CoderTypeInformation.java | 2 +-
.../state/FlinkBroadcastStateInternals.java | 8 +-
.../streaming/state/FlinkStateInternals.java | 16 +-
.../flink/FlinkRequiresStableInputTest.java | 7 +-
.../runners/flink/FlinkExecutionEnvironments.java | 2 +-
.../flink/FlinkPortableClientEntryPoint.java | 3 +-
.../flink/FlinkStreamingPipelineTranslator.java | 2 +-
.../FlinkStreamingPortablePipelineTranslator.java | 5 +-
.../translation/types/CoderTypeInformation.java | 2 +-
.../translation/types/CoderTypeSerializer.java | 2 +-
.../streaming/ExecutableStageDoFnOperator.java | 4 +-
.../state/FlinkBroadcastStateInternals.java | 8 +-
.../streaming/state/FlinkStateInternals.java | 16 +-
.../flink/FlinkRequiresStableInputTest.java | 7 +-
.../beam/runners/flink/ReadSourcePortableTest.java | 2 +
.../flink/metrics/FlinkMetricContainerTest.java | 8 +-
runners/google-cloud-dataflow-java/build.gradle | 7 +-
.../beam/runners/dataflow/DataflowPipelineJob.java | 1 +
.../beam/runners/dataflow/DataflowRunner.java | 13 +-
.../runners/dataflow/internal/CustomSources.java | 2 +-
.../beam/runners/dataflow/util/MonitoringUtil.java | 1 +
.../beam/runners/dataflow/util/PackageUtil.java | 1 +
.../dataflow/DataflowPipelineTranslatorTest.java | 6 +-
.../dataflow/PrimitiveParDoSingleFactoryTest.java | 2 +-
.../runners/dataflow/util/CloudObjectsTest.java | 4 +-
.../beam/runners/dataflow/worker/ConcatReader.java | 2 +-
.../dataflow/worker/DoFnInstanceManagers.java | 10 +-
.../worker/SplittableProcessFnFactory.java | 5 +-
.../dataflow/worker/StreamingDataflowWorker.java | 2 +
.../worker/StreamingModeExecutionContext.java | 74 +++++++--
.../beam/runners/dataflow/worker/WindmillSink.java | 18 +--
.../dataflow/worker/WorkerCustomSources.java | 2 +-
.../worker/WorkerPipelineOptionsFactory.java | 4 +-
.../logging/DataflowWorkerLoggingHandler.java | 1 +
.../dataflow/worker/status/DebugCapture.java | 2 +
.../windmill/client/AbstractWindmillStream.java | 2 +-
.../client/grpc/GetWorkResponseChunkAssembler.java | 9 +-
.../grpc/observers/DirectStreamObserver.java | 2 +-
.../work/processing/StreamingCommitFinalizer.java | 178 +++++++++++++++++----
.../work/processing/StreamingWorkScheduler.java | 5 +-
.../dataflow/worker/DefaultParDoFnFactoryTest.java | 2 +-
.../dataflow/worker/DoFnInstanceManagersTest.java | 3 +-
.../runners/dataflow/worker/SimpleParDoFnTest.java | 111 ++++++++++++-
.../worker/StreamingDataflowWorkerTest.java | 26 ++-
.../dataflow/worker/WorkerCustomSourcesTest.java | 1 +
.../logging/DataflowWorkerLoggingHandlerTest.java | 1 +
.../worker/CachingShuffleBatchReaderTest.java | 2 +-
.../client/grpc/GrpcWindmillServerTest.java | 2 +-
.../processing/StreamingCommitFinalizerTest.java | 177 ++++++++++++++++++++
.../worker/windmill/src/main/proto/windmill.proto | 17 +-
.../artifact/ArtifactStagingService.java | 1 -
.../control/FnApiControlClientPoolService.java | 2 +-
...renceCountingExecutableStageContextFactory.java | 2 +-
.../fnexecution/logging/Slf4jLogWriter.java | 1 +
.../fnexecution/status/WorkerStatusClient.java | 6 +-
.../fnexecution/control/RemoteExecutionTest.java | 4 +-
.../runners/jobsubmission/InMemoryJobService.java | 26 +--
.../beam/runners/jobsubmission/JobInvocation.java | 3 +-
.../PortablePipelineJarCreatorTest.java | 4 +-
.../apache/beam/runners/jet/JetPipelineResult.java | 2 +-
.../org/apache/beam/runners/jet/JetRunner.java | 2 +-
.../java/org/apache/beam/runners/jet/Utils.java | 2 +-
.../beam/runners/samza/SamzaPipelineResult.java | 2 +-
.../beam/runners/samza/SamzaPipelineRunner.java | 2 +-
.../org/apache/beam/runners/samza/SamzaRunner.java | 2 +-
.../runners/samza/adapter/BoundedSourceSystem.java | 2 +-
.../samza/adapter/UnboundedSourceSystem.java | 2 +-
.../samza/container/ContainerCfgLoader.java | 4 +-
.../runners/samza/runtime/AsyncDoFnRunner.java | 2 +-
.../beam/runners/samza/runtime/KeyedTimerData.java | 2 +-
.../beam/runners/samza/runtime/OpMessage.java | 2 +-
.../samza/runtime/SamzaStoreStateInternals.java | 4 +-
.../runners/samza/translation/ConfigBuilder.java | 6 +-
.../translation/ParDoBoundMultiTranslator.java | 2 +-
.../samza/translation/TranslationContext.java | 7 +-
.../samza/runtime/ClassicBundleManagerTest.java | 2 +-
.../translation/helpers/EncoderHelpersTest.java | 2 +-
.../beam/runners/spark/SparkPipelineRunner.java | 2 +-
.../spark/stateful/SparkStateInternals.java | 2 +-
.../spark/translation/SparkContextFactory.java | 2 +-
.../apache/beam/runners/spark/util/ByteArray.java | 2 +-
.../runners/spark/util/SideInputBroadcast.java | 2 +-
.../beam/runners/spark/util/SideInputStorage.java | 2 +-
.../spark/metrics/SparkMetricsPusherTest.java | 2 +-
sdks/go.mod | 4 +-
.../core/runtime/contextreg/contextreg_test.go | 4 +-
sdks/go/pkg/beam/runners/dataflow/dataflow_test.go | 16 +-
sdks/go/run_with_go_version.sh | 2 +-
.../org/apache/beam/sdk/coders/AtomicCoder.java | 1 +
.../org/apache/beam/sdk/coders/DelegateCoder.java | 1 +
.../java/org/apache/beam/sdk/coders/RowCoder.java | 1 +
.../apache/beam/sdk/coders/SerializableCoder.java | 1 +
.../beam/sdk/coders/StringDelegateCoder.java | 1 +
.../apache/beam/sdk/coders/StructuredCoder.java | 1 +
.../java/org/apache/beam/sdk/coders/ZstdCoder.java | 2 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 2 +-
.../org/apache/beam/sdk/io/range/OffsetRange.java | 2 +-
.../beam/sdk/options/ProxyInvocationHandler.java | 1 +
.../apache/beam/sdk/schemas/CachingFactory.java | 2 +-
.../beam/sdk/schemas/FromRowUsingCreator.java | 2 +-
.../sdk/schemas/GetterBasedSchemaProvider.java | 3 +-
.../apache/beam/sdk/schemas/JavaBeanSchema.java | 6 +-
.../java/org/apache/beam/sdk/schemas/Schema.java | 6 +-
.../org/apache/beam/sdk/schemas/SchemaCoder.java | 2 +-
.../sdk/schemas/logicaltypes/EnumerationType.java | 2 +-
.../beam/sdk/schemas/logicaltypes/OneOfType.java | 2 +-
.../apache/beam/sdk/schemas/transforms/Select.java | 7 +-
.../providers/LoggingTransformProvider.java | 6 +-
.../beam/sdk/schemas/utils/ByteBuddyUtils.java | 2 +-
.../java/org/apache/beam/sdk/testing/PAssert.java | 2 +-
.../beam/sdk/transforms/ApproximateUnique.java | 2 +-
.../org/apache/beam/sdk/transforms/CombineFns.java | 2 +-
.../java/org/apache/beam/sdk/transforms/Count.java | 2 +-
.../beam/sdk/transforms/PeriodicSequence.java | 2 +-
.../java/org/apache/beam/sdk/transforms/Sum.java | 6 +-
.../beam/sdk/transforms/join/RawUnionValue.java | 2 +-
.../org/apache/beam/sdk/util/WindowTracing.java | 3 +-
.../sdk/util/construction/PTransformMatchers.java | 2 +-
.../util/construction/PTransformTranslation.java | 7 +-
.../sdk/util/construction/TransformUpgrader.java | 5 +-
.../org/apache/beam/sdk/RequiresStableInputIT.java | 7 +-
.../apache/beam/sdk/coders/CoderRegistryTest.java | 4 +-
.../beam/sdk/coders/SerializableCoderTest.java | 4 +-
.../coders/TimestampPrefixingWindowCoderTest.java | 2 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 6 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 5 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 8 +-
.../apache/beam/sdk/schemas/SchemaCoderTest.java | 4 +-
.../beam/sdk/schemas/transforms/CoGroupTest.java | 2 +-
.../beam/sdk/schemas/transforms/ConvertTest.java | 8 +-
.../beam/sdk/schemas/utils/JsonUtilsTest.java | 8 +-
.../beam/sdk/schemas/utils/TestJavaBeans.java | 36 ++---
.../apache/beam/sdk/schemas/utils/TestPOJOs.java | 40 ++---
.../apache/beam/sdk/testing/ExpectedLogsTest.java | 22 +--
.../apache/beam/sdk/testing/TestPipelineTest.java | 5 +-
.../apache/beam/sdk/transforms/CombineFnsTest.java | 2 +-
.../apache/beam/sdk/transforms/DoFnTesterTest.java | 2 +-
.../org/apache/beam/sdk/transforms/WaitTest.java | 2 +-
.../sdk/transforms/display/DisplayDataTest.java | 2 +-
.../beam/sdk/transforms/join/CoGbkResultTest.java | 2 +-
.../sdk/transforms/reflect/DefaultPackageTest.java | 2 +-
.../reflect/DoFnSignaturesSplittableDoFnTest.java | 12 +-
.../beam/sdk/transforms/windowing/WindowTest.java | 2 +-
.../util/construction/CombineTranslationTest.java | 2 +-
.../construction/PTransformReplacementsTest.java | 2 +-
.../construction/PTransformTranslationTest.java | 2 +-
.../util/construction/ParDoTranslationTest.java | 21 ++-
.../util/construction/PipelineTranslationTest.java | 2 +-
.../sdk/util/construction/ReadTranslationTest.java | 4 +-
.../sdk/util/construction/SplittableParDoTest.java | 12 +-
.../construction/WindowIntoTranslationTest.java | 2 +-
.../WindowingStrategyTranslationTest.java | 4 +-
.../construction/graph/FieldAccessVisitorTest.java | 7 +-
.../graph/ProjectionProducerVisitorTest.java | 2 +-
.../graph/ProjectionPushdownOptimizerTest.java | 9 +-
.../construction/graph/QueryablePipelineTest.java | 2 +-
.../sdk/expansion/service/ExpansionService.java | 12 +-
.../beam/sdk/extensions/avro/coders/AvroCoder.java | 4 +-
.../sdk/extensions/avro/io/AvroDatumFactory.java | 2 +-
.../extensions/avro/schemas/utils/AvroUtils.java | 4 +-
.../sdk/extensions/avro/coders/AvroCoderTest.java | 4 +-
.../avro/io/AvroGeneratedUserFactory.java | 2 +-
.../extensions/avro/schemas/SchemaCoderTest.java | 2 +-
.../extensions/avro/schemas/TestAvroFactory.java | 2 +-
.../avro/schemas/transforms/ConvertTest.java | 4 +-
.../sdk/extensions/gcp/options/GcpOptions.java | 19 +--
.../gcp/util/RetryHttpRequestInitializer.java | 8 +-
.../sdk/extensions/gcp/util/gcsfs/GcsPath.java | 2 +-
.../beam/sdk/extensions/gcp/util/GcsUtilIT.java | 9 +-
.../extensions/jackson/JacksonTransformsTest.java | 2 +-
.../apache/beam/sdk/extensions/kryo/KryoCoder.java | 2 +-
.../beam/sdk/extensions/kryo/KryoCoderTest.java | 4 +-
.../ordered/GlobalSequencesProcessorDoFn.java | 14 +-
.../ordered/OrderedProcessingStatus.java | 2 +-
.../beam/sdk/extensions/ordered/ProcessorDoFn.java | 11 +-
.../ordered/SequencePerKeyProcessorDoFn.java | 4 +-
.../ordered/combiner/DefaultSequenceCombiner.java | 2 +-
.../sdk/extensions/protobuf/DynamicProtoCoder.java | 1 +
.../beam/sdk/extensions/protobuf/ProtoCoder.java | 1 +
.../beam/sdk/extensions/python/PythonService.java | 6 +-
.../sdk/extensions/sql/jdbc/BeamSqlLineIT.java | 1 +
.../sdk/extensions/sql/example/model/Customer.java | 2 +-
.../sdk/extensions/sql/example/model/Order.java | 2 +-
.../sdk/extensions/sql/impl/CatalogSchema.java | 2 +-
.../beam/sdk/extensions/sql/impl/nfa/NFA.java | 31 +---
.../sdk/extensions/sql/impl/rel/BeamCalcRel.java | 2 +-
.../sql/impl/rel/BeamEnumerableConverter.java | 4 +-
.../sdk/extensions/sql/impl/rel/BeamWindowRel.java | 10 +-
.../provider/bigquery/BeamSqlUnparseContext.java | 2 +-
.../sql/meta/provider/bigquery/BigQueryTable.java | 2 +-
.../sql/meta/provider/kafka/BeamKafkaTable.java | 2 +-
.../sql/meta/provider/text/TextTable.java | 2 +-
.../extensions/sql/InferredJavaBeanSqlTest.java | 4 +-
.../sql/impl/rel/BeamEnumerableConverterTest.java | 2 +-
.../BigQueryHllSketchCompatibilityIT.java | 7 +-
.../org/apache/beam/fn/harness/CombineRunners.java | 4 +-
.../beam/fn/harness/ExternalWorkerService.java | 2 +-
.../fn/harness/control/ExecutionStateSampler.java | 37 ++---
.../fn/harness/control/ProcessBundleHandler.java | 2 +-
.../apache/beam/fn/harness/debug/DataSampler.java | 3 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 2 +-
.../sdk/io/aws2/common/AsyncBatchWriteHandler.java | 2 +-
.../apache/beam/sdk/io/aws2/common/ObjectPool.java | 2 +-
.../sdk/io/aws2/kinesis/EFOShardSubscriber.java | 2 +-
.../beam/sdk/io/aws2/kinesis/ShardCheckpoint.java | 2 +-
.../beam/sdk/io/aws2/kinesis/StartingPoint.java | 2 +-
.../sdk/io/aws2/schemas/AwsSchemaProvider.java | 2 +-
.../org/apache/beam/sdk/io/aws2/sns/SnsIO.java | 2 +-
.../beam/sdk/io/aws2/sqs/SqsCheckpointMark.java | 2 +-
.../sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java | 2 +-
.../aws2/kinesis/EFOShardSubscribersPoolTest.java | 2 +-
.../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 2 +-
.../azure/blobstore/AzureBlobStoreFileSystem.java | 1 +
.../beam/sdk/io/cassandra/CassandraIOTest.java | 8 +-
.../beam/sdk/io/common/DatabaseTestHelper.java | 11 +-
.../org/apache/beam/sdk/io/common/HashingFn.java | 2 +-
.../org/apache/beam/sdk/io/datadog/DatadogIO.java | 2 -
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 4 +-
.../beam/sdk/io/common/FileBasedIOITHelper.java | 3 +-
.../sdk/io/fileschematransform/XmlRowAdapter.java | 2 +-
.../sdk/io/fileschematransform/XmlRowValue.java | 2 +-
.../JsonReadSchemaTransformFormatProviderTest.java | 8 +-
.../sdk/io/googleads/DummyRateLimitPolicy.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 3 +-
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 5 +-
.../sdk/io/gcp/bigquery/BigQueryInsertError.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 14 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 2 +-
.../gcp/bigquery/DynamicDestinationsHelpers.java | 2 +-
.../io/gcp/bigquery/PassThroughThenCleanup.java | 3 +-
.../gcp/bigquery/StorageApiFinalizeWritesDoFn.java | 16 +-
.../bigquery/StorageApiFlushAndFinalizeDoFn.java | 13 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 2 +-
.../bigquery/StorageApiWritesShardedRecords.java | 15 +-
.../beam/sdk/io/gcp/bigquery/TableSchemaCache.java | 4 +-
.../io/gcp/bigquery/UpdateSchemaDestination.java | 11 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 2 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 4 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +-
.../io/gcp/bigtable/BigtableServiceFactory.java | 14 +-
.../action/DetectNewPartitionsAction.java | 8 +-
.../changestreams/dao/MetadataTableDao.java | 4 +-
.../changestreams/dofn/InitializeDoFn.java | 6 +-
.../beam/sdk/io/gcp/datastore/EntityToRow.java | 2 +-
.../beam/sdk/io/gcp/datastore/RowToEntity.java | 2 +-
.../sdk/io/gcp/firestore/FirestoreV1WriteFn.java | 2 +-
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 38 ++---
.../io/gcp/healthcare/FhirIOPatientEverything.java | 7 +-
.../sdk/io/gcp/healthcare/FhirSearchParameter.java | 2 +-
.../apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java | 34 ++--
.../io/gcp/healthcare/HttpHealthcareApiClient.java | 14 +-
.../sdk/io/gcp/pubsub/AddTimestampAttribute.java | 7 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 6 +-
.../pubsub/PubsubReadSchemaTransformProvider.java | 2 +-
.../beam/sdk/io/gcp/spanner/BatchSpannerRead.java | 2 +-
.../beam/sdk/io/gcp/spanner/NaiveSpannerRead.java | 2 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 20 +--
.../SpannerWriteSchemaTransformProvider.java | 4 +-
.../mapper/ChangeStreamRecordMapper.java | 1 -
.../beam/sdk/io/gcp/testing/BigqueryClient.java | 16 +-
.../sdk/io/gcp/testing/FakeBigQueryServices.java | 3 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 4 +-
.../cloud/spanner/FakeBatchTransactionId.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 2 +-
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 5 +-
.../beam/sdk/io/gcp/bigtable/BigtableReadIT.java | 6 +-
.../BigtableReadSchemaTransformProviderIT.java | 7 +-
.../io/gcp/bigtable/BigtableServiceImplTest.java | 2 -
...gtableSimpleWriteSchemaTransformProviderIT.java | 6 +-
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 5 +-
.../BigtableWriteSchemaTransformProviderIT.java | 6 +-
.../changestreams/dao/MetadataTableDaoTest.java | 2 +-
.../changestreams/it/BigtableChangeStreamIT.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 1 +
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 18 +--
.../dao/ChangeStreamResultSetTest.java | 1 -
.../changestreams/it/IntegrationTestEnv.java | 14 +-
...StreamOrderedByTimestampAndTransactionIdIT.java | 17 +-
...nnerChangeStreamOrderedWithinKeyGloballyIT.java | 16 +-
.../it/SpannerChangeStreamOrderedWithinKeyIT.java | 22 +--
...SpannerChangeStreamTransactionBoundariesIT.java | 18 +--
.../spanner/changestreams/util/TestJsonMapper.java | 2 +-
.../beam/sdk/io/gcp/storage/GcsKmsKeyIT.java | 7 +-
.../apache/beam/sdk/io/gcp/storage/GcsMatchIT.java | 7 +-
.../apache/beam/sdk/io/hadoop/format/Employee.java | 2 +-
.../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 10 +-
.../beam/sdk/io/hbase/HBaseRowMutationsCoder.java | 2 -
.../beam/sdk/io/hbase/HBaseSharedConnection.java | 2 +-
.../beam/sdk/io/iceberg/SerializableDataFile.java | 2 +-
.../sdk/io/iceberg/WriteDirectRowsToFiles.java | 2 +-
.../beam/sdk/io/influxdb/InfluxDbIOTest.java | 1 +
.../java/org/apache/beam/sdk/io/jdbc/JdbcUtil.java | 5 +-
.../io/jdbc/JdbcWriteSchemaTransformProvider.java | 2 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 3 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 3 +-
.../apache/beam/sdk/io/jdbc/JdbcTestHelper.java | 2 +-
.../apache/beam/sdk/io/jms/JmsCheckpointMark.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +-
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 7 +-
.../org/apache/beam/sdk/io/kafka/KafkaWriter.java | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 3 +-
.../KafkaUnboundedReaderIdlePartitionTest.java | 2 +-
.../java/org/apache/beam/sdk/io/kudu/KuduIO.java | 6 +-
.../org/apache/beam/sdk/io/kudu/KuduIOTest.java | 4 +
.../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +-
.../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 3 +-
.../java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java | 19 +--
.../apache/beam/sdk/io/pulsar/PulsarIOUtils.java | 3 -
.../apache/beam/sdk/io/pulsar/PulsarIOTest.java | 3 -
.../beam/sdk/io/rabbitmq/RabbitMqMessage.java | 3 +
.../org/apache/beam/sdk/io/redis/RedisCursor.java | 2 +-
.../apache/beam/io/requestresponse/CallTest.java | 4 +-
.../beam/sdk/io/singlestore/SingleStoreIO.java | 5 +-
.../apache/beam/sdk/io/snowflake/SnowflakeIO.java | 13 +-
.../beam/sdk/io/solace/broker/SessionService.java | 14 +-
...entialsBasicAuthJcsmpSessionServiceFactory.java | 2 +-
.../sdk/io/solace/it/SolaceContainerManager.java | 2 +-
.../java/org/apache/beam/sdk/io/solr/SolrIO.java | 1 +
.../sparkreceiver/RabbitMqReceiverWithOffset.java | 4 +-
.../sdk/io/sparkreceiver/SparkReceiverIOIT.java | 2 +-
.../org/apache/beam/sdk/io/thrift/ThriftIO.java | 6 +-
.../src/main/resources/archetype-resources/pom.xml | 1 +
.../src/main/resources/archetype-resources/pom.xml | 1 +
.../ml/inference/openai/OpenAIModelHandlerIT.java | 2 +-
.../ml/inference/remote/RemoteInferenceTest.java | 18 +--
.../java/org/apache/beam/sdk/jpmstests/JpmsIT.java | 6 +-
.../beam/sdk/testing/TestPipelineExtension.java | 1 -
.../beam/sdk/nexmark/NexmarkConfiguration.java | 5 +-
.../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Auction.java | 2 +-
.../apache/beam/sdk/nexmark/model/AuctionBid.java | 2 +-
.../beam/sdk/nexmark/model/AuctionCount.java | 2 +-
.../beam/sdk/nexmark/model/AuctionPrice.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Bid.java | 2 +-
.../beam/sdk/nexmark/model/BidsPerSession.java | 2 +-
.../beam/sdk/nexmark/model/CategoryPrice.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Done.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Event.java | 2 +-
.../beam/sdk/nexmark/model/IdNameReserve.java | 2 +-
.../beam/sdk/nexmark/model/NameCityStateId.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Person.java | 2 +-
.../apache/beam/sdk/nexmark/model/SellerPrice.java | 2 +-
.../apache/beam/sdk/nexmark/queries/Query10.java | 25 +--
.../apache/beam/sdk/nexmark/queries/Query5.java | 2 +-
.../beam/sdk/nexmark/queries/WinningBids.java | 4 +-
.../sdk/nexmark/sources/generator/Generator.java | 2 +-
.../sdk/testutils/metrics/MetricsReaderTest.java | 2 +-
.../apache/beam/sdk/tpcds/BeamSqlEnvRunner.java | 1 +
.../apache/beam/sdk/tpcds/SqlTransformRunner.java | 1 +
.../beam/sdk/tpcds/TpcdsParametersReader.java | 2 -
.../org/apache/beam/sdk/tpcds/TpcdsRunResult.java | 2 +
.../sdk/testing/watermarks/WatermarkLatency.java | 2 +-
.../launcher/TransformServiceLauncher.java | 12 +-
sdks/python/apache_beam/coders/coder_impl.py | 2 +-
.../typehints/native_type_compatibility.py | 46 +++++-
.../typehints/native_type_compatibility_test.py | 59 +++++++
sdks/python/apache_beam/typehints/row_type.py | 7 +-
sdks/python/apache_beam/typehints/row_type_test.py | 89 +++++++++++
sdks/python/apache_beam/typehints/schemas.py | 33 ++--
sdks/python/apache_beam/yaml/yaml_transform.py | 88 ++++++++--
.../python/apache_beam/yaml/yaml_transform_test.py | 20 +++
.../content/en/documentation/sdks/yaml-schema.md | 48 ++++++
482 files changed, 2015 insertions(+), 1199 deletions(-)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/processing/StreamingCommitFinalizerTest.java
