This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 5b6aa703f0a Merge pull request #40683 from apache/fix-spanner-flatmap
add f5f60065ec9 rename TestRecord (#40728)
add 14846374005 drop transformers 428 (#40719)
add 9811046f33a [DebeziumIO] Add opt-in full metadata (op and Debezium
source fields) to readAsJson (#40702)
add e261c9f7689 [#36841] Spark 4 streaming: windowed GroupByKey and
Combine via transformWithState (#40703)
add cae975bc607 Update FlareDB runner doc (#40576)
add 248548d64f0 Merge pull request #40303 from
reuvenlax/merge_validates_runner_phase2
add 3d5375ac59d Expand Beam SQL TPC-DS query support and add Python TPC-DS
suite (#40695)
add 9f5f83e03d7 [Python] Keep BigQuery load jobs distinct for table
aliases (#40001)
add 6e5fdbf17fb Temporarily skip Iceberg cdc tests for managed service
(#40731)
add 94b84b6adcb [Java] Use GcsUtil V2 by default (#40713)
add 1fcf426e678 Upgrade allowed version of pillow (#40708)
add e333810ea3a Only feed total memory readings to the ModelManager
resource estimator (#40646)
add d593d389e0b [Python] Support custom BigQuery endpoints and universe
domains (#40675)
add d4ba44da40b [Java] Honor GOOGLE_CLOUD_UNIVERSE_DOMAIN for the Dataflow
endpoint (#40672)
add c8b5acf67ad Scope RunInference model_identifier annotation to
BeamML_RunInference (#40671)
add e4c2f8c71a3 Honor allowMissingRequiredFields when building Storage
Write API messages (#40204)
add 932dbe68f7f [Java][BigQueryIO] Fix BigQueryIO streaming insert
silently dropping transient network errors during response body read (#40420)
add 6848f8454da Handle null split initialization results in Flink lazy
source enumerator (#40417)
No new revisions were added by this update.
Summary of changes:
...Iceberg_Managed_Integration_Tests_Dataflow.json | 2 +-
.github/trigger_files/beam_PostCommit_Java.json | 2 +-
.../beam_PostCommit_Java_DataflowV1.json | 2 +-
.../beam_PostCommit_Java_DataflowV2.json | 2 +-
...=> beam_PostCommit_Java_Examples_Dataflow.json} | 0
.../beam_PostCommit_Java_Examples_Dataflow_V2.json | 2 +-
...n => beam_PostCommit_Java_Examples_Direct.json} | 0
.../beam_PostCommit_Java_Hadoop_Versions.json | 2 +-
.../beam_PostCommit_Java_IO_Performance_Tests.json | 2 +-
...on => beam_PostCommit_Java_Tpcds_Dataflow.json} | 0
.../beam_PostCommit_Java_Tpcds_Flink.json | 2 +-
.../beam_PostCommit_Java_Tpcds_Spark.json | 2 +-
...m_PostCommit_Java_ValidatesRunner_Dataflow.json | 2 +-
...a_ValidatesRunner_SparkStructuredStreaming.json | 2 +-
.github/trigger_files/beam_PostCommit_Python.json | 2 +-
.../beam_PostCommit_Python_Dependency.json | 2 +-
.../trigger_files/beam_PostCommit_XVR_Direct.json | 2 +-
.github/workflows/README.md | 1 +
.../beam_PostCommit_Java_Tpcds_Dataflow.yml | 8 +-
.../workflows/beam_PostCommit_Java_Tpcds_Flink.yml | 4 +-
.../workflows/beam_PostCommit_Java_Tpcds_Spark.yml | 4 +-
...am_PostCommit_Java_ValidatesRunner_Dataflow.yml | 2 +-
..._Java_ValidatesRunner_Dataflow_JavaVersions.yml | 2 +-
...PostCommit_Java_ValidatesRunner_Dataflow_V2.yml | 2 +-
....yml => beam_PostCommit_Python_Tpcds_Prism.yml} | 18 +-
CHANGES.md | 4 +
.../apache/beam/it/gcp/storage/ParquetIOLT.java | 40 +-
.../io/source/LazyFlinkSourceSplitEnumerator.java | 9 +-
.../io/source/FlinkSourceSplitEnumeratorTest.java | 28 +
runners/google-cloud-dataflow-java/build.gradle | 48 +-
.../beam/runners/dataflow/DataflowClient.java | 11 +
.../dataflow/DataflowTestBatchCoordinator.java | 1414 +++++++++++++++++++
.../beam/runners/dataflow/TestDataflowRunner.java | 282 +++-
.../options/DataflowPipelineDebugOptions.java | 17 +-
.../dataflow/options/DataflowPipelineOptions.java | 3 +-
.../runners/dataflow/util/DataflowTransport.java | 52 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 172 +--
.../DataflowRunnerValidatesRunnerTest.java | 218 +++
.../dataflow/DataflowTestBatchCoordinatorTest.java | 1477 ++++++++++++++++++++
.../dataflow/DataflowTestBatchFallbackTest.java | 132 ++
.../runners/dataflow/TestDataflowRunnerTest.java | 111 ++
.../dataflow/util/DataflowTransportTest.java | 109 ++
.../translation/PipelineTranslatorStreaming.java | 51 +-
...tor.java => GroupByKeyStreamingTranslator.java} | 42 +-
.../StatefulParDoStreamingTranslator.java | 6 +-
.../streaming/state/BeamStatefulProcessor.java | 173 +--
.../state/GroupByKeyStatefulProcessor.java | 141 ++
.../streaming/state/ParDoStatefulProcessor.java | 156 +++
.../PipelineTranslatorStreamingTest.java | 55 +-
.../streaming/StatefulParDoStreamingTest.java | 15 +-
.../translation/streaming/StreamingTestUtils.java | 13 +
.../streaming/WindowedGroupByKeyStreamingTest.java | 274 ++++
sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go | 17 +
.../xlang/debeziumio/debezium_test.go} | 26 +-
.../beam/sdk/testing/BeamParallelJunit4Runner.java | 136 +-
.../java/org/apache/beam/sdk/testing/PAssert.java | 8 +-
.../org/apache/beam/sdk/testing/TestPipeline.java | 338 ++++-
.../sdk/testing/BeamParallelJunit4RunnerTest.java | 154 ++
.../org/apache/beam/sdk/testing/PAssertTest.java | 39 +
.../apache/beam/sdk/testing/TestPipelineTest.java | 202 +++
.../util/construction/CoderTranslationTest.java | 23 +-
.../sdk/extensions/gcp/options/GcsOptions.java | 19 +-
.../beam/sdk/extensions/gcp/util/GcsUtil.java | 40 +-
.../beam/sdk/extensions/gcp/util/GcsUtilV1.java | 2 +-
.../beam/sdk/extensions/gcp/util/GcsUtilIT.java | 5 +-
.../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 54 +-
.../sdk/extensions/gcp/util/GcsUtilV1Test.java | 23 +-
.../extensions/sql/impl/CalciteQueryPlanner.java | 5 +
.../extensions/sql/impl/planner/BeamRuleSets.java | 3 +
.../sql/impl/planner/RelMdNodeStats.java | 6 +
.../extensions/sql/impl/rel/BeamCrossJoinRel.java | 173 +++
.../sql/impl/rel/BeamSetOperatorRelBase.java | 94 +-
.../sdk/extensions/sql/impl/rel/BeamSortRel.java | 11 +-
.../sdk/extensions/sql/impl/rel/BeamWindowRel.java | 104 +-
.../sql/impl/rule/BeamBasicAggregationRule.java | 4 +-
.../sql/impl/rule/BeamCoGBKJoinRule.java | 119 +-
...amCoGBKJoinRule.java => BeamCrossJoinRule.java} | 39 +-
.../extensions/sql/impl/rule/BeamWindowRule.java | 4 +-
.../extensions/sql/impl/schema/BeamTableUtils.java | 2 +
.../impl/transform/BeamBuiltinAggregations.java | 10 +-
.../transform/BeamBuiltinAnalyticFunctions.java | 246 +++-
.../sql/impl/transform/agg/VarianceFn.java | 22 +-
.../extensions/sql/BeamAnalyticFunctionsTest.java | 112 ++
.../sdk/extensions/sql/BeamSqlDslJoinTest.java | 13 +-
.../rel/BeamCoGBKJoinRelBoundedVsBoundedTest.java | 42 +-
sdks/java/io/debezium/src/README.md | 2 +
.../org/apache/beam/io/debezium/DebeziumIO.java | 42 +
.../io/debezium/DebeziumTransformRegistrar.java | 9 +
.../beam/io/debezium/KafkaSourceConsumerFn.java | 2 +-
.../apache/beam/io/debezium/SourceRecordJson.java | 119 +-
.../io/debezium/DebeziumIOMySqlConnectorIT.java | 74 +
.../apache/beam/io/debezium/DebeziumIOTest.java | 46 +
.../beam/io/debezium/SourceRecordJsonTest.java | 89 ++
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 50 +
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 20 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 141 ++
.../bigquery/TableRowToStorageApiProtoTest.java | 179 +++
sdks/java/io/iceberg/build.gradle | 11 +-
.../beam/sdk/testing/TestPipelineExtension.java | 3 +
.../org/apache/beam/sdk/tpcds/QueryReader.java | 4 +-
.../beam/sdk/tpcds/TableSchemaJSONLoader.java | 4 +-
.../org/apache/beam/sdk/tpcds/TpcdsOptions.java | 5 +-
.../beam/sdk/tpcds/TpcdsParametersReader.java | 20 +-
.../org/apache/beam/sdk/tpcds/TpcdsSchemas.java | 166 +--
.../beam/sdk/tpcds/TableSchemaJSONLoaderTest.java | 24 +-
.../beam/sdk/tpcds/TpcdsParametersReaderTest.java | 9 +
.../apache/beam/sdk/tpcds/TpcdsSchemasTest.java | 6 +-
sdks/python/apache_beam/io/debezium.py | 11 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 11 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 80 ++
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 97 +-
.../apache_beam/io/gcp/bigquery_tools_test.py | 128 ++
sdks/python/apache_beam/ml/inference/base.py | 86 +-
sdks/python/apache_beam/ml/inference/base_test.py | 19 +
.../apache_beam/ml/inference/model_manager.py | 9 +-
.../apache_beam/ml/inference/model_manager_test.py | 23 +
.../ml/inference/vertex_ai_model_monitoring_v2.py | 1 -
.../vertex_ai_model_monitoring_v2_test.py | 5 +-
.../python/apache_beam/options/pipeline_options.py | 33 +
.../apache_beam/options/pipeline_options_test.py | 67 +
.../apache_beam/testing/benchmarks/tpcds/README.md | 57 +
.../benchmarks/tpcds}/__init__.py | 2 +-
.../testing/benchmarks/tpcds/build.gradle | 84 ++
.../testing/benchmarks/tpcds/tpcds_launcher.py | 460 ++++++
.../testing/benchmarks/tpcds/tpcds_queries.py | 195 +++
.../testing/benchmarks/tpcds/tpcds_schemas.py | 141 ++
.../testing/benchmarks/tpcds/tpcds_test.py | 78 ++
sdks/python/setup.py | 2 +-
sdks/python/test-suites/tox/py311/build.gradle | 8 +-
sdks/python/tox.ini | 9 +-
settings.gradle.kts | 1 +
.../content/en/documentation/runners/flaredb.md | 18 +-
132 files changed, 9056 insertions(+), 811 deletions(-)
copy .github/trigger_files/{beam_PostCommit_Java_Nexmark_Spark.json =>
beam_PostCommit_Java_Examples_Dataflow.json} (100%)
copy .github/trigger_files/{beam_PostCommit_Java_Nexmark_Spark.json =>
beam_PostCommit_Java_Examples_Direct.json} (100%)
copy .github/trigger_files/{beam_PostCommit_PortableJar_Flink.json =>
beam_PostCommit_Java_Tpcds_Dataflow.json} (100%)
copy .github/workflows/{beam_PostCommit_Website_Test.yml =>
beam_PostCommit_Python_Tpcds_Prism.yml} (83%)
create mode 100644
runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowTestBatchCoordinator.java
create mode 100644
runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerValidatesRunnerTest.java
create mode 100644
runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowTestBatchCoordinatorTest.java
create mode 100644
runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowTestBatchFallbackTest.java
create mode 100644
runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/DataflowTransportTest.java
copy
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/{StatefulParDoStreamingTranslator.java
=> GroupByKeyStreamingTranslator.java} (57%)
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/state/GroupByKeyStatefulProcessor.java
create mode 100644
runners/spark/4/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/state/ParDoStatefulProcessor.java
create mode 100644
runners/spark/4/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/WindowedGroupByKeyStreamingTest.java
copy sdks/go/pkg/beam/{core/runtime/coderx/string_test.go =>
io/xlang/debeziumio/debezium_test.go} (66%)
create mode 100644
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/BeamCrossJoinRel.java
copy
sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/{BeamCoGBKJoinRule.java
=> BeamCrossJoinRule.java} (67%)
create mode 100644 sdks/python/apache_beam/testing/benchmarks/tpcds/README.md
copy sdks/python/apache_beam/{internal =>
testing/benchmarks/tpcds}/__init__.py (91%)
create mode 100644
sdks/python/apache_beam/testing/benchmarks/tpcds/build.gradle
create mode 100644
sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_launcher.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_queries.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_schemas.py
create mode 100644
sdks/python/apache_beam/testing/benchmarks/tpcds/tpcds_test.py
