This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 2be52540457 [Prism] Finalize bundles of DoFns fused after another
transform (#40640)
add 8ef717637e9 Bump minimum cryptography bounds beyond known severe CVEs
(#40591)
add 40df757ea39 Support attributing BigQuery quota and billing to a custom
GCP project (Python, Java, Go) (#39203)
add 707b951e5e0 Preserve HTTP errors when quota response is not JSON
(#40631)
add 5fc82a67af6 [Spanner Change Streams] Match single-RPC deadline to
total timeout in SpannerIO.ReadChangeStream (#40616)
add 10d3b8d316d Fix temp table cleanup for continuous read in BigQueryIO
(#37535)
add 49d41adc895 Bump google.golang.org/api from 0.302.0 to 0.304.0 in
/sdks (#40650)
add e4bc725b42d Bump zizmorcore/zizmor-action from 0.6.4 to 0.7.1 (#40652)
add 5fa8c2c878e Bump github/codeql-action from 4.38.3 to 4.39.0 (#40651)
add 8648b891523 Update users.yml (#40654)
add 590abac1145 [Python] Avoid ZeroDivisionError in ConcatRangeTracker for
zero weights (#40549)
add 14575dcd143 Update api key for tour of beam (#40655)
add 2c60b1996e7 Fix postgres database for grafana (#40656)
add bd102581f7f Run portable ValidatesRunner tests one at a time (#40632)
add 62357960fc3 Remove a race with another file write that is not
essential to test presence of duplicates. (#40596)
add a8ee2b84d37 [Spark 4] Group the streaming stateful ParDo by encoded
key bytes to support null keys (#40516)
add 59a8647c15a Update nlp_tensorflow_streaming.ipynb (#40641)
add 5dc12703dc9 [Prism] Keep a stateful key in progress while a split
primary holds it (#40624)
add 05c48794a86 allow dropping pk or watermark column (#40457)
add 3bef0c0cb08 [Python] Fail bundles on state write errors (#39992)
add 3ab18d061df [Go SDK] Add portable logical type registration and
standard date and micros_instant types (#40154)
add f8b1e8cdd75 [Python] Add typed decision questions and model adapters
(#40220)
add 75d7e2c467f Process deletes before adds when committing OrderedList
state updates. Otherwise we delete the newly added id from the idTracker.
(#40659)
add b4ecb26ada0 Modernize legacy DoFn in BatchElementsExample (#40557)
(#40560)
add 18db4896cf1 [Multikey] [Dataflow Streaming] Replace KeyGroupWorkQueue
with a sharded KeyGroupIndex (#40593)
add 68e762ecadb Default to parquet in AddFiles for extension less files.
(#40614)
add 0c370589685 Remove Beam SQL Data Catalog provider (#40663)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../beam_PostCommit_Java_PVR_Flink_Batch.json | 2 +-
.../beam_PostCommit_Java_PVR_Flink_Streaming.json | 2 +-
...eam_PostCommit_Java_ValidatesRunner_Spark4.json | 2 +-
...a_ValidatesRunner_SparkStructuredStreaming.json | 2 +-
.github/workflows/beam_PreCommit_GHA.yml | 2 +-
.github/workflows/codeql.yml | 4 +-
.../provisioning/datasources/beampostgresql.yaml | 2 +-
CHANGES.md | 7 +-
build.gradle.kts | 2 -
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 -
.../apache/beam/examples/BatchElementsExample.java | 7 +-
.../beam-ml/nlp_tensorflow_streaming.ipynb | 6 +-
infra/iam/users.yml | 7 +-
.../frontend/lib/firebase_options.dart | 2 +-
.../assets/symbols/java.g.yaml | 19 -
.../FlinkExecutableStageContextFactory.java | 51 +-
runners/flink/job-server/flink_job_server.gradle | 2 +
.../FlinkExecutableStageContextFactory.java | 51 +-
.../dataflow/worker/StreamingDataflowWorker.java | 7 +-
.../runners/dataflow/worker/streaming/Work.java | 3 +-
.../dataflow/worker/util/BoundedQueueExecutor.java | 224 +++++++--
.../dataflow/worker/util/KeyGroupWorkIndex.java | 190 ++++++++
.../dataflow/worker/util/KeyGroupWorkQueue.java | 472 -------------------
.../worker/windmill/state/WindmillOrderedList.java | 28 +-
.../worker/StreamingDataflowWorkerTest.java | 12 +-
.../worker/util/BoundedQueueExecutorTest.java | 428 +++++++++++------
.../worker/util/KeyGroupWorkIndexTest.java | 319 +++++++++++++
.../worker/util/KeyGroupWorkQueueTest.java | 523 ---------------------
.../windmill/state/WindmillStateInternalsTest.java | 105 +++++
.../processing/StreamingCommitFinalizerTest.java | 2 +-
.../failures/WorkFailureProcessorTest.java | 2 +-
.../work/refresh/ActiveWorkRefresherTest.java | 2 +-
.../io/streaming/BeamReaderCache.java | 2 +-
.../io/streaming/UnboundedSourceDataset.java | 2 +-
.../batch/GroupByKeyTranslatorBatch.java | 2 +-
.../translation/helpers/EncoderHelpers.java | 1 +
.../streaming/ReadUnboundedTranslator.java | 2 +-
.../StatefulParDoStreamingTranslator.java | 10 +-
.../streaming/state/BeamStatefulProcessor.java | 13 +-
.../io/streaming/BeamMicroBatchSourceTest.java | 12 +-
.../streaming/StatefulParDoStreamingTest.java | 36 ++
.../batch/GroupByKeyTranslatorBatch.java | 2 +-
.../batch/functions/SideInputValues.java | 2 +-
.../translation/helpers/CoderHelpers.java | 59 ---
.../translation/helpers/EncoderHelpers.java | 1 +
sdks/go.mod | 2 +-
sdks/go.sum | 4 +-
sdks/go/pkg/beam/core/graph/coder/row.go | 16 +
sdks/go/pkg/beam/core/graph/coder/row_decoder.go | 37 ++
sdks/go/pkg/beam/core/graph/coder/row_encoder.go | 21 +
sdks/go/pkg/beam/core/graph/coder/row_test.go | 105 +++++
.../core/runtime/graphx/schema/logicaltypes.go | 142 +++++-
.../beam/core/runtime/graphx/schema/schema_test.go | 30 ++
.../graphx/schema/standard_logical_types.go | 156 ++++++
.../graphx/schema/standard_logical_types_test.go | 421 +++++++++++++++++
.../pkg/beam/core/runtime/graphx/serialize_test.go | 16 +
sdks/go/pkg/beam/io/bigqueryio/bigquery.go | 54 ++-
sdks/go/pkg/beam/io/bigqueryio/bigquery_test.go | 125 +++++
.../prism/internal/engine/elementmanager.go | 15 +-
.../internal/engine/elementmanager_split_test.go | 194 ++++++++
.../go/test/regression/coders/fromyaml/fromyaml.go | 210 ++++++---
.../resources/beam/checkstyle/suppressions.xml | 1 -
sdks/java/extensions/sql/datacatalog/build.gradle | 107 -----
.../sql/example/BeamSqlDataCatalogExample.java | 107 -----
.../sdk/extensions/sql/example/package-info.java | 20 -
.../provider/datacatalog/BigQueryTableFactory.java | 72 ---
.../provider/datacatalog/ChainedTableFactory.java | 50 --
.../datacatalog/DataCatalogPipelineOptions.java | 43 --
.../DataCatalogPipelineOptionsRegistrar.java | 31 --
.../datacatalog/DataCatalogTableProvider.java | 235 ---------
.../meta/provider/datacatalog/GcsTableFactory.java | 61 ---
.../provider/datacatalog/PubsubTableFactory.java | 62 ---
.../sql/meta/provider/datacatalog/SchemaUtils.java | 190 --------
.../meta/provider/datacatalog/TableFactory.java | 37 --
.../meta/provider/datacatalog/ZetaSqlIdUtils.java | 83 ----
.../meta/provider/datacatalog/package-info.java | 20 -
.../datacatalog/DataCatalogBigQueryIT.java | 115 -----
.../provider/datacatalog/DataCatalogGCSIT.java | 83 ----
.../meta/provider/datacatalog/SchemaUtilsTest.java | 157 -------
.../provider/datacatalog/ZetaSqlIdUtilsTest.java | 64 ---
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 242 +++++++++-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 10 +
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 23 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 21 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 4 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 54 +++
.../io/gcp/bigquery/CleanupTempTableDoFnTest.java | 153 ++++++
.../gcp/spanner/SpannerIOReadChangeStreamTest.java | 52 ++
.../org/apache/beam/sdk/io/iceberg/AddFiles.java | 35 +-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 78 ++-
.../beam/sdk/io/iceberg/ReadFooterSchema.java | 61 ++-
.../io/iceberg/cdc/IncrementalChangelogSource.java | 34 +-
.../org/apache/beam/sdk/io/iceberg/AddFilesIT.java | 80 ++++
.../apache/beam/sdk/io/iceberg/AddFilesTest.java | 157 +++++++
.../IcebergCdcReadSchemaTransformProviderTest.java | 7 +-
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 18 -
.../beam/sdk/io/iceberg/IcebergScanConfigTest.java | 77 ++-
.../beam/sdk/io/iceberg/ReadFooterSchemaTest.java | 52 ++
.../cdc/IncrementalChangelogSourceTest.java | 87 +++-
.../apache_beam/examples/inference/README.md | 8 +-
.../examples/inference/decision_models/README.md | 304 ++++++++++++
.../inference/decision_models/__init__.py} | 3 +-
.../inference/decision_models/fraud_review.py | 126 +++++
.../inference/decision_models/fraud_review_test.py | 86 ++++
.../inference/decision_models/local_model.py | 78 +++
.../inference/decision_models/message_router.py | 199 ++++++++
.../inference/decision_models/message_router.svg | 98 ++++
.../decision_models/message_router_test.py | 75 +++
.../decision_models}/requirements.txt | 4 +-
.../apache_beam/examples/snippets/snippets_test.py | 8 +-
sdks/python/apache_beam/io/concat_source.py | 4 +-
sdks/python/apache_beam/io/concat_source_test.py | 9 +
sdks/python/apache_beam/io/fileio_test.py | 10 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 121 +++--
.../apache_beam/io/gcp/bigquery_change_history.py | 42 +-
.../io/gcp/bigquery_change_history_test.py | 75 +++
.../apache_beam/io/gcp/bigquery_file_loads.py | 46 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 47 ++
.../apache_beam/io/gcp/bigquery_read_internal.py | 11 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 355 ++++++++++++++
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 162 ++++++-
.../apache_beam/io/gcp/bigquery_tools_test.py | 232 ++++++++-
sdks/python/apache_beam/ml/inference/decision.py | 246 ++++++++++
.../apache_beam/ml/inference/decision_test.py | 254 ++++++++++
.../apache_beam/ml/inference/typesafe_inference.py | 164 +++++++
.../ml/inference/typesafe_inference_it_test.py | 93 ++++
.../ml/inference/typesafe_inference_test.py | 256 ++++++++++
.../ml/inference/typesafe_tests_requirements.txt} | 3 +-
.../python/apache_beam/options/pipeline_options.py | 11 +
.../apache_beam/runners/worker/bundle_processor.py | 35 +-
.../runners/worker/bundle_processor_test.py | 111 +++++
sdks/python/apache_beam/utils/retry.py | 2 +-
sdks/python/apache_beam/utils/retry_test.py | 18 +
sdks/python/setup.py | 3 +-
settings.gradle.kts | 1 -
136 files changed, 7056 insertions(+), 3210 deletions(-)
create mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkIndex.java
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkQueue.java
create mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkIndexTest.java
delete mode 100644
runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/util/KeyGroupWorkQueueTest.java
delete mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java
create mode 100644
sdks/go/pkg/beam/core/runtime/graphx/schema/standard_logical_types.go
create mode 100644
sdks/go/pkg/beam/core/runtime/graphx/schema/standard_logical_types_test.go
create mode 100644
sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager_split_test.go
delete mode 100644 sdks/java/extensions/sql/datacatalog/build.gradle
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlDataCatalogExample.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/package-info.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/BigQueryTableFactory.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/ChainedTableFactory.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogPipelineOptions.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogPipelineOptionsRegistrar.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogTableProvider.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/GcsTableFactory.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/PubsubTableFactory.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtils.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/TableFactory.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/ZetaSqlIdUtils.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/package-info.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogBigQueryIT.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogGCSIT.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtilsTest.java
delete mode 100644
sdks/java/extensions/sql/datacatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/ZetaSqlIdUtilsTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/CleanupTempTableDoFnTest.java
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/README.md
copy sdks/python/{.coveragerc =>
apache_beam/examples/inference/decision_models/__init__.py} (95%)
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/fraud_review.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/fraud_review_test.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/local_model.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/message_router.py
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/message_router.svg
create mode 100644
sdks/python/apache_beam/examples/inference/decision_models/message_router_test.py
copy sdks/python/apache_beam/examples/{wordcount_rust =>
inference/decision_models}/requirements.txt (86%)
create mode 100644 sdks/python/apache_beam/ml/inference/decision.py
create mode 100644 sdks/python/apache_beam/ml/inference/decision_test.py
create mode 100644 sdks/python/apache_beam/ml/inference/typesafe_inference.py
create mode 100644
sdks/python/apache_beam/ml/inference/typesafe_inference_it_test.py
create mode 100644
sdks/python/apache_beam/ml/inference/typesafe_inference_test.py
copy sdks/python/{.coveragerc =>
apache_beam/ml/inference/typesafe_tests_requirements.txt} (96%)
