This is an automated email from the ASF dual-hosted git repository.
vterentev pushed a change to branch playground-interactive
in repository https://gitbox.apache.org/repos/asf/beam.git
omit 84cac177fbc Install setuptools for playground cd examples
omit 4e12d69eaf8 Add Apache Beam installation to Dockerfile
add 25518f07fe2 Upgrade Spanner to the latest version (#37552)
add c69e8a18601 remove list_prefix deprecated method (#37587)
add e2a2b291921 Fix spelling mistakes in Go comments (#37617)
add 9a60847ad6b Update CHANGES.md for PR 37331 (#37599)
add 8da478be0d7 switch from v19 to v23 (#37620)
add 5849f2e38ea [yaml] : fix validate compatible method (#37588)
add 5d6cb04ea05 [yaml] : switch js2py to pythonmonkey (#37560)
add f2aa7912401 Exercise Python PVR tests on Flink 2.0 (#37313)
add 2ac0fe431e6 [yaml] : fix create different type elements issue (#37585)
add 66134f7107f Fix infer_result_type for pcollection with tags. (#37608)
add 97e5ccf0c6f Upgrade ClickHouseIO to use ClickHouse Java Client V2
(#37611)
add 90253bf5857 Tune Ratelimiter config (#37598)
add 76a53610898 Rename Chunk to EmbeddableItem (#37623)
add a9c14d1698d Add Ubuntu 24.04 runner pools (#37624)
add 0337ecefc61 Fix iceberg nested filter (#37340)
add 1ee2c717f3e Don't prematurely close FileIO (#37626)
add bbaace8b6be update changes for 2.72 - yaml etc (#37627)
add 0699f3594e6 Add post submit test for dataflow streaming with windmill
tag encoding v2 (#37603)
add bab2374552d Add Pause and Paused states (#37581)
add 3729c75d027 Improve RateLimiter Config
add 11586e46c39 add flag
add 9ac321abca9 fix readme
add 46b27eee882 Merge pull request #37630: Improve RateLimiter Config in
terraform example
add b46a6352ba9 Add HTTPS detection for ClickHouse JDBC URL parsing
(#37635)
add 51eec3c8658 Update to latest go version (#37638)
add 30bdbafcc95 feat: Improve YamlUtils.dumpAsMap error handling (#37468)
add f2765cab203 Fix bom upgrader and upgrade to latest (#37636)
add 808d23877bd Fix DoFnInvoker cache collision for generic types (#37355)
add 8dc5da5f500 Skip test for dataflow (#37612)
add e87bb292545 Bigtable: add an option to pass in row filter text proto
for use in template (#37632)
add 74fc46c404d GCS client library migration in Java SDK - part 2b (#37592)
add 9d528015cb6 Moving to 2.73.0-SNAPSHOT on master branch.
add f73bd6a0919 Update CHANGES.md
add cecc2a6ae0c Add image embedding support to ml/rag (#37628)
add 9690853b873 Bump golang.org/x/net from 0.49.0 to 0.50.0 in /sdks
(#37558)
add 1164f941f9c Bump filippo.io/edwards25519 from 1.1.0 to 1.1.1 in /sdks
(#37644)
add 41779ad88d3 [Dataflow Streaming] Update windmill timer clasification
logic to work with windmill state tag encoding v2 (#37604)
add 120ceab62a7 Fix renaming of embeddable_to_dict_fn (#37646)
add e41b9b6ba69 runQuickstartJavaFlinkLocal on Flink 2.0 (#37649)
add 872c671b15b Update cibuildwheel dependency (#37504)
add b9d48fa1750 Fix type checking failure with Python 3.10+ union pipe
syntax (int | None) (#37645)
add dfb3f872192 [yaml] : add jinja inheritance example (#37601)
add 59922a390e0 Fix breaking changes from renaming Chunk to EmbeddableItem
(#37651)
add 87c50c67e37 [Python] Support large pipeline options via file (#37379)
add 1052216416f Migrate remaining checkNotNull to
checkArgumentNotNull/checkStateNotNull in options package #18719 (#37654)
add 9ff96dfd329 Update Golang - DebeziumIO 3.1.3 (#37667)
add caa18faf4ff Adds triggerring configuration to KafkaIO eos. (#37648)
add cb7caaad2ca [Dataflow Streaming] Remove dead code around redundant
experiments (#37684)
add d614058ca95 Bump go.mongodb.org/mongo-driver from 1.17.8 to 1.17.9 in
/sdks (#37657)
add 9bfb7ab8c42 Fix resource leak in KafkaIO GCS truststore file download
(#37681)
add 4fe61733f77 Update Python Dependencies (#37678)
add bc35bef5d8f fix some python deps (#37668)
add 3a41311a1f9 unpin grpc. (#37633)
add 26b18c6e6f1 Revert "[yaml] : switch js2py to pythonmonkey (#37560)"
(#37665)
add 7095c880f34 [Dataflow Streaming] Mark support for windmill state tag
encoding v2 (#37683)
add ace4a54a15a dataframes -> dataframe
add 71c46ef01e4 Merge pull request #37697 from apache/fix-dask
add 451be0e289d Bump github.com/aws/smithy-go from 1.24.0 to 1.24.1 in
/sdks (#37695)
add 1541d027c52 Remove test-only functions in GcsUtil and change related
unit tests to use V1. (#37597)
add 6a1618eca88 fix(python): Register all output pcollections of a
transform rather than only ones that happened to be accessed in DoOutputsTuple
(#37556)
add 015b6417f97 Improve PR Bot reviewer assignment / pinging
add 6cab6eb17a8 Merge pull request #37663: Improve PR Bot reviewer
assignment / pinging
add eac11fed1d4 Simply wheel validation logic. (#37687)
add d503869c450 Bump lodash from 4.17.21 to 4.17.23 in /website/www
add df59030c9ba Merge pull request #37403: Bump lodash from 4.17.21 to
4.17.23 in /website/www
add f47f81cf244 [Python] Update dev SDK container tag for Python SDK
(#37661)
add 2341d3f7b7d Remove relative change threshold condition (#37643)
add 80b49029c64 Fix Playground Frontend Test job (#37642)
add 9ee4fe64f2b [yaml] : add available yaml io connectors to website
(#37605)
add e755b53b5a5 Add an instruction for using Docker buildx when building
containers. (#37706)
add 27a5fca9cca Moves expectMessage to before creating BeamFnLoggingClient
since exception can be thrown inside createAndStart method.
add 6b647c7e3d6 Merge pull request #37692: Moves expectMessage to before
creating BeamFnLoggingClient
add 940a8ca5db9 Bigtable: clean up client side metrics flag
add 193db392a93 Merge pull request #37596: Bigtable: clean up client side
metrics flag
add caf661798f7 Fix PreCommit tests involving docker-in-docker after
Milvus dependency rearrangements (#37711)
add 06dd48e9313 Revert "Update cibuildwheel dependency (#37504)" (#37713)
add 72027e11d1f Fix DebeziumIO resuming from worker restart (#37689)
add c51e244bb95 revert 'Optimize data size estimation for proto coder.'
(#37709)
add 2403014afce Stabilize
test_check_standard_external_transforms_config_in_sync (#37717)
add 9eb3a18ae70 fix conflict (#37594)
add 32d6bce8c23 Enable pickling main by reference option within
CloudpickleConfig (#37554)
add 5c2c3530411 Add Pyrefly configuration for Beam Python (#37700)
add e9f4c99206a fix Python 3.12 PostCommit by installing setuptools
(#37719)
add 67c3183913a Revise title and header in YAML documentation (#37722)
add 9524b56c148 Add observability (#37716)
add 30518c840bf [Stateful] Implement length-aware keying to minimize
padding in BatchElements (Part 2/3) (#37565)
add 5bee123ce8e Add monitoring for model manager (#37703)
add 4b3dcf26142 Fix GroupBy snippet tests for issue #30778 (#37672)
add 342de16b2f2 Fix Python PostCommit Flink runner log spam switching to
simple logger (#37720)
add af81a0c1635 Fix ml_base dependency (#37721)
add 5a6f763da54 [Drain] OnTimer - propagate caused by drain bit up to
DoFnRunner (#37012)
add fcbb0340800 Bump go.opentelemetry.io/otel/sdk from 1.38.0 to 1.40.0 in
/sdks (#37730)
add e761581b30d Reduce Fn Runner changes
add 4739405cdbc Merge pull request #37727: [Drain] Reduce Fn Runner changes
add 1430241b6fe Pin aircompressor. (#37737)
add 3197d88ef5c Update java containers. (#37733)
add c9ab4b6ae8d Add default_pickle_library_override delegation to
InteractiveRunner (#37752)
add e5db1b3c63e [Dataflow Java Streaming] Reset state using finally blocks
instead of catching Exception, in cases where it may otherwise corrupt
datastructures if an OutOfMemoryError is thrown. (#37746)
add e1e0637d989 [ErrorProne] Fix FormatStringShouldUsePlaceholders string
concat when Preconditions is used (#37745)
add e28ed02b9b8 Add Apache Beam installation to Dockerfile
add 08af69c65f6 Install setuptools for playground cd examples
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (84cac177fbc)
\
N -- N -- N refs/heads/playground-interactive (08af69c65f6)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.agent/skills/gradle-build/SKILL.md | 26 +-
.agent/skills/python-development/SKILL.md | 12 +-
.asf.yaml | 1 +
.../arc/environments/beam.env | 70 ++
.../arc/images/Dockerfile | 2 +-
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../trigger_files/beam_PostCommit_Go_VR_Flink.json | 2 +-
.github/trigger_files/beam_PostCommit_Java.json | 4 +-
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 1 +
...esRunner_Dataflow_Streaming_TagEncodingV2.json} | 2 +-
...eam_PostCommit_Java_ValidatesRunner_Direct.json | 1 +
...beam_PostCommit_Java_ValidatesRunner_Flink.json | 1 +
...json => beam_PostCommit_PortableJar_Flink.json} | 0
.github/trigger_files/beam_PostCommit_Python.json | 4 +-
.../beam_PostCommit_Python_Examples_Flink.json | 3 +
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
...tesRunner_Dataflow_Streaming_TagEncodingV2.yml} | 18 +-
.../beam_PostCommit_Python_Portable_Flink.yml | 5 +-
...eam_PostCommit_Python_ValidatesRunner_Flink.yml | 2 +-
.../workflows/beam_PreCommit_Python_Coverage.yml | 35 +-
.github/workflows/beam_PreCommit_Python_ML.yml | 6 +-
.../workflows/beam_PreCommit_Python_PVR_Flink.yml | 1 +
.github/workflows/dask_runner_tests.yml | 2 +-
.../run_rc_validation_java_quickstart.yml | 2 +-
.gitignore | 7 +
CHANGES.md | 51 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 34 +-
.../bigquery_vector_ingestion_and_search.ipynb | 2 +-
examples/terraform/envoy-ratelimiter/README.md | 63 +-
examples/terraform/envoy-ratelimiter/deploy.sh | 66 ++
examples/terraform/envoy-ratelimiter/gke.tf | 4 +-
.../terraform/envoy-ratelimiter/prerequisites.tf | 1 +
examples/terraform/envoy-ratelimiter/ratelimit.tf | 156 ++--
examples/terraform/envoy-ratelimiter/variables.tf | 40 +-
gradle.properties | 4 +-
playground/frontend/build.gradle | 3 +-
.../integration_test/embedded_run_test.dart | 2 +-
.../frontend/playground_components/pubspec.yaml | 6 +-
.../lib/src/widget_tester.dart | 13 +-
.../playground_components_dev/pubspec.yaml | 4 +-
playground/frontend/pubspec.yaml | 4 +-
release/build.gradle.kts | 2 +-
.../org/apache/beam/runners/core/DoFnRunner.java | 4 +-
.../runners/core/LateDataDroppingDoFnRunner.java | 7 +-
...TimeBoundedSplittableProcessElementInvoker.java | 27 +-
.../apache/beam/runners/core/ProcessFnRunner.java | 4 +-
.../runners/core/PushbackSideInputDoFnRunner.java | 4 +-
.../org/apache/beam/runners/core/ReduceFn.java | 3 +
.../beam/runners/core/ReduceFnContextFactory.java | 38 +-
.../apache/beam/runners/core/ReduceFnRunner.java | 52 +-
.../apache/beam/runners/core/SimpleDoFnRunner.java | 25 +-
.../core/SimplePushbackSideInputDoFnRunner.java | 7 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 3 +-
.../beam/runners/core/StatefulDoFnRunner.java | 16 +-
.../apache/beam/runners/core/WindowMatchers.java | 52 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 17 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 20 +-
.../beam/runners/core/StatefulDoFnRunnerTest.java | 3 +-
.../apache/beam/runners/direct/ParDoEvaluator.java | 3 +-
.../wrappers/streaming/DoFnOperator.java | 3 +-
.../FlinkStreamingPortablePipelineTranslator.java | 21 +-
.../functions/FlinkStatefulDoFnFunction.java | 3 +-
.../wrappers/streaming/DoFnOperator.java | 3 +-
.../FlinkStreamingPortablePipelineTranslator.java | 21 +-
.../flink/adapter/BeamFlinkDataStreamAdapter.java | 3 +-
.../flink/metrics/DoFnRunnerWithMetricsUpdate.java | 14 +-
.../functions/FlinkSideInputReader.java | 2 +-
.../functions/FlinkStatefulDoFnFunction.java | 3 +-
.../wrappers/streaming/DoFnOperator.java | 12 +-
.../streaming/ExecutableStageDoFnOperator.java | 7 +-
.../streaming/stableinput/BufferedElements.java | 10 +-
.../streaming/stableinput/BufferingDoFnRunner.java | 4 +-
.../wrappers/streaming/DoFnOperatorTest.java | 4 +-
.../streaming/ExecutableStageDoFnOperatorTest.java | 13 +-
runners/google-cloud-dataflow-java/build.gradle | 67 +-
.../beam/runners/dataflow/DataflowRunner.java | 19 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 5 +-
.../dataflow/worker/DataflowProcessFnRunner.java | 4 +-
.../dataflow/worker/GroupAlsoByWindowFnRunner.java | 4 +-
.../runners/dataflow/worker/SimpleParDoFn.java | 3 +-
.../StreamingKeyedWorkItemSideInputDoFnRunner.java | 4 +-
.../worker/StreamingModeExecutionContext.java | 34 +-
.../worker/StreamingSideInputDoFnRunner.java | 4 +-
.../dataflow/worker/WindmillKeyedWorkItem.java | 16 +-
.../dataflow/worker/WindmillNamespacePrefix.java | 47 --
.../beam/runners/dataflow/worker/WindmillSink.java | 54 +-
.../dataflow/worker/WindmillTimerInternals.java | 28 +-
.../runners/dataflow/worker/WindmillTimerType.java | 10 +-
.../windmill/client/AbstractWindmillStream.java | 39 +-
.../windmill/client/grpc/GrpcGetDataStream.java | 14 +-
.../dataflow/worker/windmill/state/RangeCoder.java | 4 +-
.../worker/windmill/state/WindmillTagEncoding.java | 24 +-
.../windmill/state/WindmillTagEncodingV1.java | 88 ++-
.../windmill/state/WindmillTagEncodingV2.java | 44 +-
...ractWindmillMap.java => WindmillTimerData.java} | 17 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 97 ++-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 11 +-
.../windmill/state/WindmillTagEncodingV1Test.java | 23 +-
.../windmill/state/WindmillTagEncodingV2Test.java | 69 +-
...renceCountingExecutableStageContextFactory.java | 2 +-
.../fnexecution/control/TimerReceiverFactory.java | 3 +-
.../translation/PipelineTranslatorUtils.java | 3 +-
.../fnexecution/control/RemoteExecutionTest.java | 4 +-
.../runners/fnexecution/wire/CommonCoderTest.java | 4 +-
.../runners/jet/processors/StatefulParDoP.java | 3 +-
.../apache/beam/runners/prism/PrismLocator.java | 2 +-
.../samza/metrics/DoFnRunnerWithMetrics.java | 13 +-
.../runners/samza/runtime/AsyncDoFnRunner.java | 7 +-
.../apache/beam/runners/samza/runtime/DoFnOp.java | 3 +-
.../runtime/DoFnRunnerWithKeyedInternals.java | 13 +-
.../beam/runners/samza/runtime/PortableDoFnOp.java | 3 +-
.../runners/samza/runtime/SamzaDoFnRunners.java | 7 +-
.../translation/batch/DoFnRunnerFactory.java | 4 +-
.../translation/batch/DoFnRunnerWithMetrics.java | 14 +-
.../spark/translation/AbstractInOutIterator.java | 3 +-
.../spark/translation/DoFnRunnerWithMetrics.java | 14 +-
.../translation/AbstractInOutIteratorTest.java | 9 +-
.../translation/SparkInputDataProcessorTest.java | 4 +-
scripts/beam-sql.sh | 2 +-
scripts/ci/pr-bot/findPrsNeedingAttention.ts | 30 +-
scripts/ci/pr-bot/processNewPrs.ts | 41 +-
scripts/ci/pr-bot/processPrUpdate.ts | 52 +-
scripts/tools/bomupgrader.py | 1 -
sdks/go.mod | 30 +-
sdks/go.sum | 64 +-
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 2 +-
sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go | 8 +
.../runners/prism/internal/engine/teststream.go | 2 +-
sdks/go/test/build.gradle | 17 +-
.../test/integration/io/xlang/debezium/debezium.go | 3 +-
.../integration/io/xlang/debezium/debezium_test.go | 3 +-
sdks/go/test/run_validatesrunner_tests.sh | 6 +-
sdks/java/bom/gcp/build.gradle | 14 +-
.../container/license_scripts/dep_urls_java.yaml | 2 +-
sdks/java/core/build.gradle | 2 +-
.../apache/beam/sdk/coders/BigDecimalCoder.java | 4 +-
.../apache/beam/sdk/metrics/MetricNameFilter.java | 8 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 48 +-
.../beam/sdk/options/PipelineOptionsValidator.java | 6 +-
.../beam/sdk/options/ProxyInvocationHandler.java | 63 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 15 +-
.../org/apache/beam/sdk/options/ValueProvider.java | 15 +-
.../apache/beam/sdk/options/ValueProviders.java | 4 +-
.../apache/beam/sdk/schemas/AutoValueSchema.java | 10 +-
.../apache/beam/sdk/schemas/JavaBeanSchema.java | 10 +-
.../apache/beam/sdk/schemas/JavaFieldSchema.java | 10 +-
.../apache/beam/sdk/schemas/utils/YamlUtils.java | 33 +-
.../java/org/apache/beam/sdk/transforms/DoFn.java | 6 +
.../beam/sdk/transforms/DoFnOutputReceivers.java | 3 +-
.../org/apache/beam/sdk/transforms/DoFnTester.java | 26 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 141 +++-
.../org/apache/beam/sdk/util/InstanceBuilder.java | 24 +-
.../construction/SplittableParDoNaiveBounded.java | 6 +
.../apache/beam/sdk/util/construction/Timer.java | 29 +-
.../beam/sdk/values/ValueInSingleWindow.java | 20 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 1 -
.../sdk/transforms/reflect/DoFnInvokersTest.java | 53 +-
.../org/apache/beam/sdk/util/YamlUtilsTest.java | 30 +
.../beam/sdk/util/construction/TimerTest.java | 28 +-
.../beam/sdk/extensions/gcp/util/GcsUtil.java | 208 ++---
.../beam/sdk/extensions/gcp/util/GcsUtilV2.java | 161 +++-
.../gcp/util/GcsUtilParameterizedIT.java | 290 +++++++
.../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 207 +++--
.../sdk/extensions/gcp/util/gcsfs/GcsPathTest.java | 2 +
.../extensions/sql/expansion-service/build.gradle | 1 -
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 68 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 4 +-
...plittablePairWithRestrictionDoFnRunnerTest.java | 4 +-
.../harness/control/ProcessBundleHandlerTest.java | 13 +-
.../harness/logging/BeamFnLoggingClientTest.java | 2 +-
.../apache/beam/sdk/io/cassandra/CassandraIO.java | 23 +-
sdks/java/io/clickhouse/build.gradle | 7 +-
.../beam/sdk/io/clickhouse/ClickHouseIO.java | 393 +++++++--
.../sdk/io/clickhouse/ClickHouseJdbcUrlParser.java | 276 +++++++
.../apache/beam/sdk/io/clickhouse/TableSchema.java | 8 +-
.../beam/sdk/io/clickhouse/AtomicInsertTest.java | 28 +-
.../beam/sdk/io/clickhouse/BaseClickHouseTest.java | 140 +++-
.../ClickHouseIOJdbcBackwardCompatibilityTest.java | 98 +++
.../ClickHouseIOPropertyMergingTest.java | 211 +++++
.../beam/sdk/io/clickhouse/ClickHouseIOTest.java | 142 ++--
.../io/clickhouse/ClickHouseJdbcUrlParserTest.java | 429 ++++++++++
.../ratelimiter/EnvoyRateLimiterFactory.java | 5 +-
sdks/java/io/debezium/src/README.md | 7 +-
.../org/apache/beam/io/debezium/DebeziumIO.java | 28 +-
.../io/debezium/DebeziumTransformRegistrar.java | 9 +
.../beam/io/debezium/KafkaSourceConsumerFn.java | 151 ++--
.../io/debezium/KafkaSourceConsumerFnTest.java | 89 ++-
.../apache/beam/io/debezium/OffsetTrackerTest.java | 25 +-
sdks/java/io/google-ads/build.gradle | 4 +-
.../apache/beam/sdk/io/googleads/GoogleAdsIO.java | 6 +-
.../{GoogleAdsV19.java => GoogleAdsV23.java} | 56 +-
.../sdk/io/googleads/DummyRateLimitPolicy.java | 4 +-
.../beam/sdk/io/googleads/GoogleAdsIOTest.java | 40 +-
.../io/googleads/MockGoogleAdsClientFactory.java | 10 +-
sdks/java/io/google-cloud-platform/build.gradle | 16 +-
.../io/gcp/bigquery/BeamRowToStorageApiProto.java | 10 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 42 +-
.../sdk/io/gcp/bigtable/BigtableReadOptions.java | 12 +
.../io/gcp/bigtable/BigtableServiceFactory.java | 13 -
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 5 +
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 91 ++-
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 7 +-
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 7 +-
.../changestreams/it/IntegrationTestEnv.java | 7 +-
.../apache/beam/sdk/io/iceberg/FilterUtils.java | 30 +-
.../org/apache/beam/sdk/io/iceberg/ReadUtils.java | 10 +-
.../beam/sdk/io/iceberg/FilterUtilsTest.java | 16 +
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 19 +
.../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 28 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 38 +-
.../io/kafka/KafkaReadSchemaTransformProvider.java | 31 +-
.../beam/sdk/io/kafka/KafkaUnboundedSource.java | 8 +-
.../sdk/io/kafka/WatchForKafkaTopicPartitions.java | 6 +-
.../sdk/io/kafka/upgrade/KafkaIOTranslation.java | 7 +
.../io/kafka/upgrade/KafkaIOTranslationTest.java | 2 +
sdks/python/apache_beam/coders/coder_impl.py | 6 -
.../transforms/aggregation/groupby_attr.py | 13 +-
.../transforms/aggregation/groupby_attr_expr.py | 10 +-
.../transforms/aggregation/groupby_expr.py | 5 +-
.../aggregation/groupby_expr_aggregate.py | 10 +-
.../aggregation/groupby_global_aggregate.py | 6 +-
.../aggregation/groupby_simple_aggregate.py | 5 +-
.../transforms/aggregation/groupby_test.py | 20 +-
.../transforms/aggregation/groupby_two_exprs.py | 9 +-
.../internal/cloudpickle/cloudpickle.py | 23 +-
sdks/python/apache_beam/io/aws/s3io.py | 28 +-
sdks/python/apache_beam/io/aws/s3io_test.py | 36 +-
sdks/python/apache_beam/io/azure/blobstorageio.py | 26 +-
.../io/external/xlang_bigqueryio_it_test.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 26 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 5 +-
sdks/python/apache_beam/metrics/metric.py | 30 +-
sdks/python/apache_beam/ml/inference/base.py | 17 +
sdks/python/apache_beam/ml/inference/base_test.py | 39 +
.../apache_beam/ml/inference/gemini_inference.py | 5 +-
.../apache_beam/ml/inference/model_manager.py | 28 +
.../apache_beam/ml/inference/model_manager_test.py | 65 ++
sdks/python/apache_beam/ml/rag/embeddings/base.py | 66 +-
.../apache_beam/ml/rag/embeddings/base_test.py | 47 +-
.../apache_beam/ml/rag/embeddings/huggingface.py | 151 +++-
.../ml/rag/embeddings/huggingface_test.py | 135 ++++
.../apache_beam/ml/rag/embeddings/vertex_ai.py | 133 +++-
.../ml/rag/embeddings/vertex_ai_test.py | 128 +++
.../ml/rag/enrichment/bigquery_vector_search.py | 127 +--
.../apache_beam/ml/rag/enrichment/milvus_search.py | 71 +-
.../ml/rag/enrichment/milvus_search_test.py | 4 +-
.../python/apache_beam/ml/rag/ingestion/alloydb.py | 2 +-
sdks/python/apache_beam/ml/rag/ingestion/base.py | 34 +-
.../apache_beam/ml/rag/ingestion/bigquery.py | 121 +--
.../ml/rag/ingestion/bigquery_it_test.py | 2 +-
.../apache_beam/ml/rag/ingestion/cloudsql.py | 2 +-
.../apache_beam/ml/rag/ingestion/milvus_search.py | 32 +-
.../ml/rag/ingestion/milvus_search_it_test.py | 21 +-
sdks/python/apache_beam/ml/rag/ingestion/mysql.py | 15 +-
.../apache_beam/ml/rag/ingestion/mysql_common.py | 133 ++--
.../apache_beam/ml/rag/ingestion/postgres.py | 14 +-
.../ml/rag/ingestion/postgres_common.py | 155 ++--
.../python/apache_beam/ml/rag/ingestion/spanner.py | 92 +--
sdks/python/apache_beam/ml/rag/types.py | 140 +++-
sdks/python/apache_beam/ml/rag/utils.py | 4 +-
sdks/python/apache_beam/ml/transforms/base_test.py | 4 +-
.../ml/transforms/embeddings/huggingface_test.py | 2 +-
.../transforms/embeddings/tensorflow_hub_test.py | 2 +-
.../ml/transforms/embeddings/vertex_ai.py | 10 +-
sdks/python/apache_beam/pipeline.py | 11 +-
sdks/python/apache_beam/pipeline_test.py | 7 +-
.../runners/dataflow/dataflow_runner.py | 6 +-
.../runners/dataflow/dataflow_runner_test.py | 42 +
.../clients/dataflow/dataflow_v1b3_client.py | 515 ++++++------
.../clients/dataflow/dataflow_v1b3_messages.py | 879 +++++++++++++--------
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
.../runners/interactive/cache_manager.py | 2 +-
.../runners/interactive/interactive_runner.py | 6 +
.../runners/interactive/interactive_runner_test.py | 20 +
.../runners/portability/flink_runner_test.py | 29 +-
sdks/python/apache_beam/runners/runner.py | 2 +
.../apache_beam/runners/worker/sdk_worker_main.py | 27 +-
.../testing/analyzers/perf_analysis_utils.py | 6 -
.../external_transform_provider_it_test.py | 12 +-
sdks/python/apache_beam/transforms/util.py | 60 +-
sdks/python/apache_beam/transforms/util_test.py | 233 ++++++
.../typehints/tagged_output_typehints_test.py | 84 ++
sdks/python/apache_beam/typehints/typehints.py | 7 +-
.../python/apache_beam/typehints/typehints_test.py | 24 +
sdks/python/apache_beam/version.py | 2 +-
.../yaml/examples/testing/examples_test.py | 18 +-
.../yaml/examples/testing/input_data.py | 5 +
.../examples/transforms/io/bigquery_write.yaml | 59 ++
.../jinja/{include => inheritance}/README.md | 16 +-
.../base/base_pipeline.yaml} | 40 +-
.../wordCountInheritance.yaml} | 19 +-
sdks/python/apache_beam/yaml/json_utils.py | 30 +-
sdks/python/apache_beam/yaml/json_utils_test.py | 124 +++
sdks/python/apache_beam/yaml/tests/create.yaml | 18 +
sdks/python/apache_beam/yaml/yaml_provider.py | 32 +
.../apache_beam/yaml/yaml_provider_unit_test.py | 13 +
sdks/python/container/boot.go | 33 +-
.../container/ml/py310/base_image_requirements.txt | 91 +--
.../container/ml/py310/gpu_image_requirements.txt | 129 ++-
.../container/ml/py311/base_image_requirements.txt | 93 +--
.../container/ml/py311/gpu_image_requirements.txt | 129 ++-
.../container/ml/py312/base_image_requirements.txt | 90 +--
.../container/ml/py312/gpu_image_requirements.txt | 125 ++-
.../container/ml/py313/base_image_requirements.txt | 92 ++-
sdks/python/container/piputil.go | 39 +-
.../container/py310/base_image_requirements.txt | 74 +-
.../container/py311/base_image_requirements.txt | 76 +-
.../container/py312/base_image_requirements.txt | 73 +-
.../container/py313/base_image_requirements.txt | 75 +-
sdks/python/gen_xlang_wrappers.py | 6 +-
sdks/python/pyrefly.toml | 62 ++
sdks/python/setup.py | 35 +-
sdks/python/test-suites/portable/common.gradle | 101 ++-
sdks/python/tox.ini | 2 +-
sdks/typescript/package.json | 2 +-
start-build-env.sh | 4 +-
.../site/content/en/documentation/io/connectors.md | 140 +++-
.../www/site/content/en/documentation/sdks/yaml.md | 6 +-
website/www/yarn.lock | 6 +-
320 files changed, 8698 insertions(+), 3542 deletions(-)
copy .github/trigger_files/{beam_PostCommit_Java_ValidatesRunner_Dataflow.json
=> beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json}
(80%)
copy .github/trigger_files/{beam_PostCommit_Java_Examples_Dataflow_ARM.json =>
beam_PostCommit_PortableJar_Flink.json} (100%)
create mode 100644
.github/trigger_files/beam_PostCommit_Python_Examples_Flink.json
copy
.github/workflows/{beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml
=> beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.yml}
(86%)
create mode 100755 examples/terraform/envoy-ratelimiter/deploy.sh
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillNamespacePrefix.java
copy sdks/java/core/src/main/java/org/apache/beam/sdk/util/Weighted.java =>
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WindmillTimerType.java
(80%)
copy
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/state/{AbstractWindmillMap.java
=> WindmillTimerData.java} (63%)
create mode 100644
sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseJdbcUrlParser.java
create mode 100644
sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOJdbcBackwardCompatibilityTest.java
create mode 100644
sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOPropertyMergingTest.java
create mode 100644
sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseJdbcUrlParserTest.java
rename
sdks/java/io/google-ads/src/main/java/org/apache/beam/sdk/io/googleads/{GoogleAdsV19.java
=> GoogleAdsV23.java} (93%)
create mode 100644
sdks/python/apache_beam/yaml/examples/transforms/io/bigquery_write.yaml
copy sdks/python/apache_beam/yaml/examples/transforms/jinja/{include =>
inheritance}/README.md (77%)
copy
sdks/python/apache_beam/yaml/examples/transforms/jinja/{include/submodules/mapToFieldsSplitConfig.yaml
=> inheritance/base/base_pipeline.yaml} (53%)
copy
sdks/python/apache_beam/yaml/examples/transforms/jinja/{include/submodules/combineTransform.yaml
=> inheritance/wordCountInheritance.yaml} (69%)
create mode 100644 sdks/python/pyrefly.toml
