This is an automated email from the ASF dual-hosted git repository.
Amar3tto pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from c54595b4b9f Merge pull request #39111: Fix OpenSearch
username/password env fallback in examples
add 22b2f385a8a Refactor LoadTestBase
add 875cfd11ced Refactor DefaultPipelineLauncher
add 5312b03502c Add IcebergIOLT
add 7fb07784bf2 Add looker metrics for IcebergIO
add 9432acebb2b Refactoring
add ab142e93fb6 Fix conflict
new eccd9b78d50 Merge pull request #39270 from apache/icerbergio-benchmark
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../beam_PostCommit_Java_IO_Performance_Tests.yml | 2 +-
.test-infra/tools/refresh_looker_metrics.py | 2 +
it/build.gradle | 6 +-
it/clickhouse/build.gradle | 8 +-
it/common/build.gradle | 29 +-
.../apache/beam/it/common}/artifacts/Artifact.java | 2 +-
.../beam/it/common}/artifacts/ArtifactClient.java | 2 +-
.../beam/it/common}/artifacts/GcsArtifact.java | 2 +-
.../beam/it/common}/artifacts/package-info.java | 2 +-
.../it/common}/artifacts/utils/ArtifactUtils.java | 2 +-
.../it/common}/artifacts/utils/AvroTestUtil.java | 2 +-
.../it/common}/artifacts/utils/JsonTestUtil.java | 2 +-
.../common}/artifacts/utils/ParquetTestUtil.java | 2 +-
.../it/common}/artifacts/utils/package-info.java | 2 +-
.../common}/bigquery/BigQueryResourceManager.java | 2 +-
.../bigquery/BigQueryResourceManagerException.java | 2 +-
.../bigquery/BigQueryResourceManagerUtils.java | 2 +-
.../beam/it/common/bigquery}/package-info.java | 4 +-
.../common}/dataflow/AbstractPipelineLauncher.java | 2 +-
.../common}/dataflow/DefaultPipelineLauncher.java | 52 +--
.../beam/it/common}/dataflow/package-info.java | 2 +-
.../it/common}/monitoring/MonitoringClient.java | 2 +-
.../beam/it/common}/monitoring/package-info.java | 2 +-
.../it/common}/storage/GcsResourceManager.java | 10 +-
.../beam/it/common}/storage/package-info.java | 2 +-
.../src/main/resources/test-artifact.json | 0
.../beam/it/common}/artifacts/GcsArtifactTest.java | 4 +-
.../common}/artifacts/utils/ArtifactUtilsTest.java | 4 +-
.../bigquery/BigQueryResourceManagerTest.java | 2 +-
.../bigquery/BigQueryResourceManagerUtilsTest.java | 4 +-
.../dataflow/AbstractPipelineLauncherTest.java | 2 +-
.../dataflow/DefaultPipelineLauncherTest.java | 8 +-
.../beam/it/common/dataflow}/IOLoadTestBase.java | 21 +-
.../beam/it/common/dataflow}/IOStressTestBase.java | 2 +-
.../beam/it/common/dataflow}/LoadTestBase.java | 8 +-
.../it/common}/storage/GcsResourceManagerTest.java | 19 +-
it/google-cloud-platform/build.gradle | 21 +-
.../it/gcp/artifacts/matchers/package-info.java | 20 --
.../gcp/bigquery/conditions/BigQueryRowsCheck.java | 2 +-
.../it/gcp/dataflow/ClassicTemplateClient.java | 1 +
.../beam/it/gcp/dataflow/FlexTemplateClient.java | 1 +
.../beam/it/gcp/datagenerator/package-info.java | 20 --
.../it/gcp/spanner/matchers/SpannerAsserts.java | 2 +-
.../java/org/apache/beam/it/gcp/WordCountIT.java | 1 +
.../apache/beam/it/gcp/bigquery/BigQueryIOLT.java | 4 +-
.../apache/beam/it/gcp/bigquery/BigQueryIOST.java | 4 +-
.../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 2 +-
.../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 3 +-
.../apache/beam/it/gcp/bigtable/BigTableIOST.java | 3 +-
.../it/gcp/dataflow/ClassicTemplateClientTest.java | 9 +-
.../it/gcp/dataflow/FlexTemplateClientTest.java | 8 +-
.../beam/it/gcp/datagenerator/DataGenerator.java | 2 +-
.../org/apache/beam/it/gcp/pubsub/PubSubIOST.java | 3 +-
.../org/apache/beam/it/gcp/pubsub/PubsubIOLT.java | 3 +-
.../apache/beam/it/gcp/spanner/SpannerIOLT.java | 3 +-
.../apache/beam/it/gcp/spanner/SpannerIOST.java | 3 +-
.../apache/beam/it/gcp/storage/FileBasedIOLT.java | 4 +-
it/iceberg/build.gradle | 134 ++++++++
.../org/apache/beam/it/iceberg/IcebergIOLT.java | 363 +++++++++++++++++++++
it/kafka/build.gradle | 3 +-
.../java/org/apache/beam/it/kafka/KafkaIOLT.java | 3 +-
.../java/org/apache/beam/it/kafka/KafkaIOST.java | 3 +-
it/truthmatchers/build.gradle | 6 +-
.../beam/it/truthmatchers}/ArtifactAsserts.java | 15 +-
.../beam/it/truthmatchers}/ArtifactsSubject.java | 21 +-
settings.gradle.kts | 1 +
website/www/site/content/en/performance/_index.md | 1 +
.../content/en/performance/icebergio/_index.md | 50 +++
website/www/site/data/performance.yaml | 31 ++
69 files changed, 770 insertions(+), 201 deletions(-)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/Artifact.java (97%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/ArtifactClient.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/GcsArtifact.java (96%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/package-info.java
(95%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/ArtifactUtils.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/AvroTestUtil.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/JsonTestUtil.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/ParquetTestUtil.java
(98%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/artifacts/utils/package-info.java
(94%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManager.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerException.java
(96%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerUtils.java
(98%)
copy
{examples/java/cdap/hubspot/src/main/java/org/apache/beam/examples/complete/cdap/hubspot/options
=>
it/common/src/main/java/org/apache/beam/it/common/bigquery}/package-info.java
(86%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/AbstractPipelineLauncher.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/DefaultPipelineLauncher.java
(92%)
copy it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/dataflow/package-info.java (95%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/monitoring/MonitoringClient.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/monitoring/package-info.java
(94%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/storage/GcsResourceManager.java
(97%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/main/java/org/apache/beam/it/common}/storage/package-info.java (95%)
rename it/{google-cloud-platform =>
common}/src/main/resources/test-artifact.json (100%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/artifacts/GcsArtifactTest.java
(94%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/artifacts/utils/ArtifactUtilsTest.java
(94%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerTest.java
(99%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/bigquery/BigQueryResourceManagerUtilsTest.java
(93%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/dataflow/AbstractPipelineLauncherTest.java
(99%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/dataflow/DefaultPipelineLauncherTest.java
(91%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/IOLoadTestBase.java
(87%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/IOStressTestBase.java
(99%)
rename it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common/dataflow}/LoadTestBase.java (98%)
rename it/{google-cloud-platform/src/test/java/org/apache/beam/it/gcp =>
common/src/test/java/org/apache/beam/it/common}/storage/GcsResourceManagerTest.java
(95%)
delete mode 100644
it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/package-info.java
delete mode 100644
it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/datagenerator/package-info.java
rename it/google-cloud-platform/src/{main =>
test}/java/org/apache/beam/it/gcp/datagenerator/DataGenerator.java (99%)
create mode 100644 it/iceberg/build.gradle
create mode 100644
it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java
rename
it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers
=>
truthmatchers/src/main/java/org/apache/beam/it/truthmatchers}/ArtifactAsserts.java
(79%)
rename
it/{google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers
=>
truthmatchers/src/main/java/org/apache/beam/it/truthmatchers}/ArtifactsSubject.java
(87%)
create mode 100644 website/www/site/content/en/performance/icebergio/_index.md
