This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 4da60251729 Stop emitting upon truncate in Java PeriodicSequence
(#25716)
add 5a2b93ddaa7 [Spark Dataset runner] Avoid copying outputs for most
cases in ParDo translation (related to #24711) (#25624)
add 9b6b72f3379 [Spark Dataset runner] Prevent stackoverflow in ParDo load
tests when using RDDs to break linage for large number of consequtive ParDos
(#25732)
add fcf111d299a Update documentation by adding parameters for Dataflow
Runner (#25561)
add 08198142087 [Playground] Apache Kafka based Kafka emulator for
Playground examples (#25357)
add 6452dc79822 [Flink Runner] Add new Source classes that are based on
FLIP-27 Source API. (#25525)
add fd13e14dd67 Bump golang.org/x/sys from 0.5.0 to 0.6.0 in /sdks (#25727)
add b3a9aa795ce Bump github.com/tetratelabs/wazero in /sdks (#25689)
add 2011cfcc63a Fail when main session cannot be loaded. (#25617)
add 3b76f4ab6ed Run Playground precommit checks in CI, check gofmt,
reformat Go code (#25733)
add 889d4db025a Give an option to override the number of shards in BQ
streaming insert addresses (#25569)
add 0a4fdc70d44 Bump distlib from 0.3.1 to 0.3.6 in /sdks/python (#25730)
add 4bbd7b89944 Bump golang.org/x/net from 0.7.0 to 0.8.0 in /sdks (#25729)
add be4cbba4b1d add logging
add 237c088cf9a add logging
add e4ddb861756 Merge pull request #25744: Add more logging to Storage API
writes
add 0d804d102b2 Use credential for mongodb performance tests (#25631)
add 188f030065f Run the quickstart command has incorrect flag name
add 92459a2c29a Merge pull request #25703 Run the quickstart command has
incorrect flag name.
No new revisions were added by this update.
Summary of changes:
.github/workflows/playground_backend_precommit.yml | 68 ++++
.../job_PerformanceTests_MongoDBIO_IT.groovy | 4 +-
...ob_PostCommit_Python_MongoDBIO_Load_Test.groovy | 4 +-
.../kubernetes/mongodb/load-balancer/mongo.yml | 5 +
CHANGES.md | 14 +-
playground/backend/build.gradle.kts | 16 +
playground/backend/cmd/server/controller.go | 30 +-
playground/backend/cmd/server/http.go | 2 +-
playground/backend/cmd/server/wrapper.go | 2 +-
playground/backend/containers/go/Dockerfile | 7 +
playground/backend/containers/go/build.gradle | 8 +
playground/backend/containers/java/Dockerfile | 7 +-
playground/backend/containers/java/build.gradle | 9 +
playground/backend/containers/python/Dockerfile | 7 +
playground/backend/containers/python/build.gradle | 8 +
playground/backend/containers/scio/Dockerfile | 6 +
playground/backend/containers/scio/build.gradle | 8 +
playground/backend/env_setup.sh | 2 +-
playground/backend/internal/cache/cache.go | 12 +-
.../internal/code_processing/code_processing.go | 29 +-
.../code_processing/code_processing_test.go | 8 +-
.../internal/constants/datastore_constants.go | 2 +-
.../internal/constants/emulator_constants.go | 2 +-
.../backend/internal/db/datastore/datastore_db.go | 10 +-
.../internal/db/mapper/datastore_mapper_test.go | 2 +-
.../db/schema/migration/migrations_test.go | 2 +-
playground/backend/internal/emulators/emulator.go | 51 +--
playground/backend/internal/emulators/kafka.go | 109 +++++-
.../backend/internal/environment/application.go | 37 +-
.../internal/environment/environment_service.go | 91 ++---
.../environment/environment_service_test.go | 14 +-
.../backend/internal/environment/property.go | 2 +-
playground/backend/internal/executors/executor.go | 2 +-
.../backend/internal/executors/executor_builder.go | 48 +--
playground/backend/internal/fs_tool/fs.go | 24 ++
playground/backend/internal/logger/logger.go | 5 +-
.../backend/internal/preparers/go_preparers.go | 8 +-
.../backend/internal/preparers/java_preparers.go | 18 +-
playground/backend/internal/preparers/preparer.go | 6 +-
.../backend/internal/preparers/preparer_test.go | 2 +-
.../backend/internal/preparers/python_preparers.go | 8 +-
.../backend/internal/preparers/scio_preparers.go | 10 +-
.../setup_tools/life_cycle/life_cycle_setuper.go | 44 ++-
.../life_cycle/life_cycle_setuper_test.go | 3 +-
.../internal/streaming/run_output_writer.go | 7 +-
.../backend/internal/tests/test_data/test_data.go | 6 +-
playground/backend/internal/utils/common.go | 4 +-
.../backend/internal/utils/preparers_utils.go | 4 +-
playground/backend/internal/utils/system_utils.go | 4 +-
.../backend/internal/validators/java_validators.go | 4 +-
.../backend/internal/validators/scio_validators.go | 2 +-
playground/kafka-emulator/build.gradle | 53 +++
.../org/apache/beam/playground/KafkaEmulator.java | 72 ++++
.../org/apache/beam/playground/package-info.java | 22 ++
.../io/source/compat/FlinkSourceCompat.java | 31 ++
.../io/source/compat/SplitEnumeratorCompat.java | 27 ++
.../streaming/io/source/compat/package-info.java | 20 +
.../streaming/io/source/SourceTestCompat.java | 62 ++++
.../io/source/compat/SplitEnumeratorCompat.java | 27 ++
.../io/source/compat/FlinkSourceCompat.java | 28 ++
.../streaming/io/source/SourceTestCompat.java | 93 +++++
runners/flink/flink_runner.gradle | 2 +
.../flink/translation/utils/SerdeUtils.java | 85 +++++
.../wrappers/streaming/io/source/FlinkSource.java | 152 ++++++++
.../streaming/io/source/FlinkSourceReaderBase.java | 399 ++++++++++++++++++++
.../streaming/io/source/FlinkSourceSplit.java | 76 ++++
.../io/source/FlinkSourceSplitEnumerator.java | 181 +++++++++
.../io/source/bounded/FlinkBoundedSource.java | 67 ++++
.../source/bounded/FlinkBoundedSourceReader.java | 146 ++++++++
.../streaming/io/source/bounded/package-info.java | 21 ++
.../io/source/impulse/BeamImpulseSource.java | 107 ++++++
.../streaming/io/source/impulse/package-info.java | 21 ++
.../wrappers/streaming/io/source/package-info.java | 21 ++
.../io/source/unbounded/FlinkUnboundedSource.java | 64 ++++
.../unbounded/FlinkUnboundedSourceReader.java | 272 ++++++++++++++
.../io/source/unbounded/package-info.java | 21 ++
.../streaming/io/TestBoundedCountingSource.java | 150 ++++++++
.../wrappers/streaming/io/TestCountingSource.java | 63 +++-
.../io/source/FlinkSourceReaderTestBase.java | 403 +++++++++++++++++++++
.../io/source/FlinkSourceSplitEnumeratorTest.java | 151 ++++++++
.../wrappers/streaming/io/source/TestSource.java | 38 ++
.../bounded/FlinkBoundedSourceReaderTest.java | 146 ++++++++
.../unbounded/FlinkUnboundedSourceReaderTest.java | 318 ++++++++++++++++
.../translation/batch/ParDoTranslatorBatch.java | 42 ++-
sdks/go.mod | 10 +-
sdks/go.sum | 20 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 4 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 8 +-
.../bigquery/StorageApiWritesShardedRecords.java | 10 +-
.../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 34 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 13 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 12 +-
.../apache_beam/runners/worker/sdk_worker.py | 7 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 26 +-
sdks/python/build-requirements.txt | 2 +-
settings.gradle.kts | 1 +
.../en/documentation/ml/large-language-modeling.md | 27 +-
.../site/content/en/get-started/quickstart/go.md | 2 +-
98 files changed, 4039 insertions(+), 305 deletions(-)
create mode 100644 .github/workflows/playground_backend_precommit.yml
create mode 100644 playground/kafka-emulator/build.gradle
create mode 100644
playground/kafka-emulator/src/main/java/org/apache/beam/playground/KafkaEmulator.java
create mode 100644
playground/kafka-emulator/src/main/java/org/apache/beam/playground/package-info.java
create mode 100644
runners/flink/1.12/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/compat/FlinkSourceCompat.java
create mode 100644
runners/flink/1.12/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/compat/SplitEnumeratorCompat.java
create mode 100644
runners/flink/1.12/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/compat/package-info.java
create mode 100644
runners/flink/1.12/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/SourceTestCompat.java
create mode 100644
runners/flink/1.13/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/compat/SplitEnumeratorCompat.java
create mode 100644
runners/flink/1.14/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/compat/FlinkSourceCompat.java
create mode 100644
runners/flink/1.14/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/SourceTestCompat.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/utils/SerdeUtils.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSource.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderBase.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplit.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSource.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/package-info.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/impulse/BeamImpulseSource.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/impulse/package-info.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/package-info.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSource.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java
create mode 100644
runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/package-info.java
create mode 100644
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/TestBoundedCountingSource.java
create mode 100644
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderTestBase.java
create mode 100644
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumeratorTest.java
create mode 100644
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/TestSource.java
create mode 100644
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReaderTest.java
create mode 100644
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReaderTest.java
