This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 961ed7c31ad [IcebergIO] Fix stale openWriters count in
RecordWriterManager (#39157)
add f909e8a7af9 Use Flink 2.2 container for load tests
add 984df275bf4 debug only change -- preserve cluster to check error
add 397d9c61a27 Set configs for Flink 2
add a0fd9502818 Refactor Flink Load Tests to use native Dataproc
components and Auto Zone
add 3e76ff518fa Remove unnecessary gcloud volume mount from Job Server
add 5d28c414f62 Change memory allocation, use resize feature
add 30a5a00d584 Merge pull request #39154 from apache/flink-load-test-ver
add f9171da59bc Bump github.com/nats-io/nats-server/v2 from 2.14.2 to
2.14.3 in /sdks (#39168)
add aa6942a9853 Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#39169)
add fcd105de015 Bump js-yaml from 3.14.2 to 3.15.0 in /website/www (#39172)
add 24a2ccc830a Bump com.gradle.common-custom-user-data-gradle-plugin
(#39166)
add 1c4b2047016 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in
/sdks (#39170)
add aec5b5da813 Add embedded NVIDIA Dynamo support to vLLM ModelHandler
(#38701)
add 8f60e69177b Fixed DirectRunner PubSub subscriber client lifecycle
(#39079)
add ed0a5832eb1 Add open telemetry dependency to dataflow worker (#38941)
add a86e6115270 Retry starting SubprocessServer on failure (#39159)
add e34cc96df4d [yaml] - mongodb read normalization (#38772)
add 78fdc959cda [Dataflow Streaming] [Multi Key] Prepare
StreamingModeExecutionContext and StreamingWorkScheduler for multi-key
execution. (#38814)
add aec0392a343 Revert previous workarounds and upgrade TFX stack to
1.21.0 (#39164)
No new revisions were added by this update.
Summary of changes:
.../beam_CloudML_Benchmarks_Dataflow.json | 2 +-
...it_Java_ValidatesRunner_Dataflow_Streaming.json | 2 +-
.github/trigger_files/beam_PostCommit_Python.json | 4 +-
.../beam_LoadTests_Go_CoGBK_Flink_batch.yml | 8 +-
.../beam_LoadTests_Go_Combine_Flink_Batch.yml | 14 +-
.../beam_LoadTests_Go_GBK_Flink_Batch.yml | 14 +-
.../beam_LoadTests_Go_ParDo_Flink_Batch.yml | 6 +-
.../beam_LoadTests_Go_SideInput_Flink_Batch.yml | 8 +-
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 6 +-
.../beam_LoadTests_Python_Combine_Flink_Batch.yml | 16 +-
...am_LoadTests_Python_Combine_Flink_Streaming.yml | 6 +-
.../beam_LoadTests_Python_GBK_Flink_Batch.yml | 16 +-
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 8 +-
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 8 +-
.../workflows/beam_PreCommit_Flink_Container.yml | 6 +-
.test-infra/dataproc/flink_cluster.sh | 78 +++--
.test-infra/dataproc/init-actions/docker.sh | 103 ------
.test-infra/dataproc/init-actions/flink.sh | 217 ------------
runners/google-cloud-dataflow-java/build.gradle | 3 +
.../worker/StreamingModeExecutionContext.java | 370 +++++++++++++++++----
.../worker/WindmillReaderIteratorBase.java | 29 +-
.../dataflow/worker/WindowingWindmillReader.java | 119 +++----
.../worker/streaming/ComputationWorkExecutor.java | 43 +--
.../runners/dataflow/worker/streaming/Work.java | 29 +-
.../processing/ComputationWorkExecutorFactory.java | 33 +-
.../work/processing/StreamingWorkScheduler.java | 312 +++++++++--------
.../worker/StreamingDataflowWorkerTest.java | 36 +-
.../worker/StreamingModeExecutionContextTest.java | 116 ++++---
.../worker/WindmillReaderIteratorBaseTest.java | 92 +++++
.../worker/WindowingWindmillReaderTest.java | 275 +++++++++++++++
.../dataflow/worker/WorkerCustomSourcesTest.java | 51 ++-
.../dataflow/worker/streaming/WorkTest.java | 132 ++++++++
sdks/go.mod | 20 +-
sdks/go.sum | 40 +--
sdks/java/io/mongodb/build.gradle | 2 +
.../beam/sdk/io/mongodb/AggregationQuery.java | 9 +-
...> MongoDbReadSchemaTransformConfiguration.java} | 34 +-
.../MongoDbReadSchemaTransformProvider.java | 151 +++++++++
.../apache/beam/sdk/io/mongodb/MongoDbUtils.java | 131 ++++++++
.../MongoDbReadSchemaTransformProviderTest.java | 221 ++++++++++++
.../beam/sdk/io/mongodb/MongoDbUtilsTest.java | 123 +++++++
.../examples/inference/vllm_text_completion.py | 24 +-
.../mltransform_one_hot_encoding_test.py | 2 -
sdks/python/apache_beam/io/gcp/pubsub_test.py | 79 ++++-
.../inference/test_resources/vllm.dockerfile.old | 14 +-
.../apache_beam/ml/inference/vllm_inference.py | 331 +++++++++++++++---
.../ml/inference/vllm_inference_test.py | 157 +++++++++
.../runners/direct/transform_evaluator.py | 113 +++++--
.../testing/benchmarks/cloudml/constraints.txt | 42 ---
.../testing/benchmarks/cloudml/requirements.txt | 21 +-
sdks/python/apache_beam/utils/subprocess_server.py | 89 ++---
.../apache_beam/utils/subprocess_server_test.py | 22 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 13 +-
sdks/python/apache_beam/yaml/tests/mongodb.yaml | 40 ++-
sdks/python/apache_beam/yaml/yaml_io.py | 59 +++-
sdks/python/test-suites/dataflow/common.gradle | 44 ++-
sdks/standard_external_transforms.yaml | 34 ++
settings.gradle.kts | 2 +-
website/www/yarn.lock | 6 +-
59 files changed, 2925 insertions(+), 1060 deletions(-)
delete mode 100644 .test-infra/dataproc/init-actions/docker.sh
delete mode 100644 .test-infra/dataproc/init-actions/flink.sh
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WindowingWindmillReaderTest.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WorkTest.java
copy
sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/{MongoDbWriteSchemaTransformConfiguration.java
=> MongoDbReadSchemaTransformConfiguration.java} (69%)
create mode 100644
sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformProvider.java
create mode 100644
sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDbReadSchemaTransformProviderTest.java
create mode 100644 sdks/python/apache_beam/ml/inference/vllm_inference_test.py
delete mode 100644
sdks/python/apache_beam/testing/benchmarks/cloudml/constraints.txt
