This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from f4b13a08539 Bump pillow (#39396)
add f80294498cf Bump actions/checkout from 4 to 7 (#39399)
add 7735b396afb Bump pillow (#39395)
add ab36569659a fix GCP_PATH expansion (#39400)
add dca1253c7e3 Fix CdapIO dependency (#39393)
add 7bc2fb49f8d Fix a memory leak in the failed instruction id cache.
(#39405)
add 5b441f27082 Update CHANGES.md to document a memory growth issue.
(#39407)
add 202d339ad9f Fix OpenSearch env fallback for username/password in
examples
add c54595b4b9f Merge pull request #39111: Fix OpenSearch
username/password env fallback in examples
add 22b2f385a8a Refactor LoadTestBase
add 875cfd11ced Refactor DefaultPipelineLauncher
add 5312b03502c Add IcebergIOLT
add 7fb07784bf2 Add looker metrics for IcebergIO
add 9432acebb2b Refactoring
add ab142e93fb6 Fix conflict
add eccd9b78d50 Merge pull request #39270 from apache/icerbergio-benchmark
add 9baa19ad5ca Increase CI timeouts for Python 3.14 multiarch container
jobs (#39401)
add 0c151d8ddb2 Fix gRPC stream observer leak on ProcessBundleHandler
shutdown (#39390)
add 414b48ad644 add Apache third party allow list check and doc (#39392)
add 83fcb2f9cdf [Gemini] Add Java GeminiModelHandler class (#39245)
add 55e1ecbb138 Add metric for Kafka offset commit failures (#38889)
No new revisions were added by this update.
Summary of changes:
.github/actions/setup-k8s-access/action.yml | 2 +-
.github/workflows/README.md | 9 +
.../beam_Infrastructure_UsersPermissions.yml | 2 +-
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 2 +-
.github/workflows/beam_PostCommit_Python_Arm.yml | 3 +-
.github/workflows/beam_PreCommit_GHA.yml | 12 +-
...beam_Python_ValidatesContainer_Dataflow_ARM.yml | 2 +
.github/workflows/build_wheels.yml | 3 +-
.github/workflows/codeql.yml | 2 +-
.github/workflows/update_python_dependencies.yml | 1 +
.test-infra/tools/refresh_looker_metrics.py | 2 +
CHANGES.md | 26 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
.../beam-ml/rag_usecase/opensearch_connector.py | 8 +-
.../beam-ml/rag_usecase/opensearch_enrichment.py | 4 +-
it/build.gradle | 6 +-
it/clickhouse/build.gradle | 8 +-
it/common/build.gradle | 29 +-
.../apache/beam/it/common}/artifacts/Artifact.java | 2 +-
.../beam/it/common}/artifacts/ArtifactClient.java | 2 +-
.../beam/it/common}/artifacts/GcsArtifact.java | 2 +-
.../beam/it/common}/artifacts/package-info.java | 2 +-
.../it/common}/artifacts/utils/ArtifactUtils.java | 2 +-
.../it/common}/artifacts/utils/AvroTestUtil.java | 2 +-
.../it/common}/artifacts/utils/JsonTestUtil.java | 2 +-
.../common}/artifacts/utils/ParquetTestUtil.java | 2 +-
.../it/common}/artifacts/utils/package-info.java | 2 +-
.../common}/bigquery/BigQueryResourceManager.java | 2 +-
.../bigquery/BigQueryResourceManagerException.java | 2 +-
.../bigquery/BigQueryResourceManagerUtils.java | 2 +-
.../beam/it/common/bigquery}/package-info.java | 4 +-
.../common}/dataflow/AbstractPipelineLauncher.java | 2 +-
.../common}/dataflow/DefaultPipelineLauncher.java | 52 +--
.../beam/it/common}/dataflow/package-info.java | 2 +-
.../it/common}/monitoring/MonitoringClient.java | 2 +-
.../beam/it/common}/monitoring/package-info.java | 2 +-
.../it/common}/storage/GcsResourceManager.java | 10 +-
.../beam/it/common}/storage/package-info.java | 2 +-
.../src/main/resources/test-artifact.json | 0
.../beam/it/common}/artifacts/GcsArtifactTest.java | 4 +-
.../common}/artifacts/utils/ArtifactUtilsTest.java | 4 +-
.../bigquery/BigQueryResourceManagerTest.java | 2 +-
.../bigquery/BigQueryResourceManagerUtilsTest.java | 4 +-
.../dataflow/AbstractPipelineLauncherTest.java | 2 +-
.../dataflow/DefaultPipelineLauncherTest.java | 8 +-
.../beam/it/common/dataflow}/IOLoadTestBase.java | 21 +-
.../beam/it/common/dataflow}/IOStressTestBase.java | 2 +-
.../beam/it/common/dataflow}/LoadTestBase.java | 8 +-
.../it/common}/storage/GcsResourceManagerTest.java | 19 +-
it/google-cloud-platform/build.gradle | 21 +-
.../it/gcp/artifacts/matchers/package-info.java | 20 --
.../gcp/bigquery/conditions/BigQueryRowsCheck.java | 2 +-
.../it/gcp/dataflow/ClassicTemplateClient.java | 1 +
.../beam/it/gcp/dataflow/FlexTemplateClient.java | 1 +
.../beam/it/gcp/datagenerator/package-info.java | 20 --
.../it/gcp/spanner/matchers/SpannerAsserts.java | 2 +-
.../java/org/apache/beam/it/gcp/WordCountIT.java | 1 +
.../apache/beam/it/gcp/bigquery/BigQueryIOLT.java | 4 +-
.../apache/beam/it/gcp/bigquery/BigQueryIOST.java | 4 +-
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 2 +-
.../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 3 +-
.../apache/beam/it/gcp/bigtable/BigTableIOST.java | 3 +-
.../it/gcp/dataflow/ClassicTemplateClientTest.java | 9 +-
.../it/gcp/dataflow/FlexTemplateClientTest.java | 8 +-
.../beam/it/gcp/datagenerator/DataGenerator.java | 2 +-
.../org/apache/beam/it/gcp/pubsub/PubSubIOST.java | 3 +-
.../org/apache/beam/it/gcp/pubsub/PubsubIOLT.java | 3 +-
.../apache/beam/it/gcp/spanner/SpannerIOLT.java | 3 +-
.../apache/beam/it/gcp/spanner/SpannerIOST.java | 3 +-
.../apache/beam/it/gcp/storage/FileBasedIOLT.java | 4 +-
it/iceberg/build.gradle | 134 ++++++++
.../org/apache/beam/it/iceberg/IcebergIOLT.java | 363 +++++++++++++++++++++
it/kafka/build.gradle | 3 +-
.../java/org/apache/beam/it/kafka/KafkaIOLT.java | 3 +-
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 3 +-
it/truthmatchers/build.gradle | 6 +-
.../beam/it/truthmatchers}/ArtifactAsserts.java | 15 +-
.../beam/it/truthmatchers}/ArtifactsSubject.java | 21 +-
.../fn/harness/control/ProcessBundleHandler.java | 1 +
.../beam/fn/harness/data/BeamFnDataClient.java | 9 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 12 +
sdks/java/io/cdap/build.gradle | 5 +
.../beam/sdk/io/kafka/KafkaCommitOffset.java | 6 +
.../ml/inference/{openai => gemini}/build.gradle | 19 +-
.../ml/inference/gemini/GeminiImageResponse.java | 17 +-
.../inference/gemini/GeminiInferenceFunctions.java | 67 ++++
.../ml/inference/gemini/GeminiModelHandler.java | 86 +++++
.../ml/inference/gemini/GeminiModelParameters.java | 60 ++++
.../ml/inference/gemini/GeminiRequestFunction.java | 27 +-
.../sdk/ml/inference/gemini/GeminiStringInput.java | 17 +-
.../ml/inference/gemini/GeminiStringResponse.java | 17 +-
.../sdk/ml/inference/gemini}/package-info.java | 4 +-
.../ml/inference/gemini/GeminiModelHandlerIT.java | 163 +++++++++
.../inference/gemini/GeminiModelHandlerTest.java | 133 ++++++++
.../ml/inference/remote/PredictionResultCoder.java | 74 +++++
.../sdk/ml/inference/remote/RemoteInference.java | 34 +-
.../ml/inference/remote/RemoteInferenceTest.java | 4 +-
.../kfp/components/train/requirements.txt | 2 +-
.../apache_beam/runners/worker/sdk_worker.py | 12 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 38 +++
settings.gradle.kts | 2 +
website/www/site/content/en/performance/_index.md | 1 +
.../content/en/performance/icebergio/_index.md | 50 +++
website/www/site/data/performance.yaml | 31 ++
104 files changed, 1574 insertions(+), 281 deletions(-)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/Artifact.java (97%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/ArtifactClient.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/GcsArtifact.java (96%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/package-info.java
(95%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/ArtifactUtils.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/AvroTestUtil.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/JsonTestUtil.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/ParquetTestUtil.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/package-info.java
(94%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManager.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerException.java
(96%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerUtils.java
(98%)
copy
{examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/options
=>
it/common/src/main/java/org/apache/beam/it/common/bigquery}/package-info.java
(86%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/AbstractPipelineLauncher.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/DefaultPipelineLauncher.java
(92%)
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/package-info.java (95%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/monitoring/MonitoringClient.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/monitoring/package-info.java
(94%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/storage/GcsResourceManager.java
(97%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/storage/package-info.java (95%)
rename it/{google-cloud-platform =>
common}/src/main/resources/test-artifact.json (100%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/artifacts/GcsArtifactTest.java
(94%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/artifacts/utils/ArtifactUtilsTest.java
(94%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerTest.java
(99%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerUtilsTest.java
(93%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/dataflow/AbstractPipelineLauncherTest.java
(99%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/dataflow/DefaultPipelineLauncherTest.java
(91%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/IOLoadTestBase.java
(87%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/IOStressTestBase.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/LoadTestBase.java (98%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/storage/GcsResourceManagerTest.java
(95%)
delete mode 100644
it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/package-info.java
delete mode 100644
it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/package-info.java
rename it/google-cloud-platform/src/{main =>
test}/java/org/apache/beam/it/gcp/datagenerator/DataGenerator.java (99%)
create mode 100644 it/iceberg/build.gradle
create mode 100644
it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java
rename
it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers
=>
truthmatchers/src/main/java/org/apache/beam/it/truthmatchers}/ArtifactAsserts.java
(79%)
rename
it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers
=>
truthmatchers/src/main/java/org/apache/beam/it/truthmatchers}/ArtifactsSubject.java
(87%)
copy sdks/java/ml/inference/{openai => gemini}/build.gradle (68%)
copy
examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixConfiguration.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiImageResponse.java
(66%)
create mode 100644
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiInferenceFunctions.java
create mode 100644
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelHandler.java
create mode 100644
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelParameters.java
copy
runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/TestSource.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiRequestFunction.java
(62%)
copy
examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixConfiguration.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiStringInput.java
(70%)
copy
examples/multi-language/src/main/java/org/apache/beam/examples/multilanguage/JavaPrefixConfiguration.java
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini/GeminiStringResponse.java
(69%)
copy
{examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/options
=>
sdks/java/ml/inference/gemini/src/main/java/org/apache/beam/sdk/ml/inference/gemini}/package-info.java
(89%)
create mode 100644
sdks/java/ml/inference/gemini/src/test/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelHandlerIT.java
create mode 100644
sdks/java/ml/inference/gemini/src/test/java/org/apache/beam/sdk/ml/inference/gemini/GeminiModelHandlerTest.java
create mode 100644
sdks/java/ml/inference/remote/src/main/java/org/apache/beam/sdk/ml/inference/remote/PredictionResultCoder.java
create mode 100644 website/www/site/content/en/performance/icebergio/_index.md
