This is an automated email from the ASF dual-hosted git repository.
Amar3tto pushed a commit to branch icerbergio-benchmark
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/icerbergio-benchmark by this
push:
new c931d953c4b Refactoring
c931d953c4b is described below
commit c931d953c4b0929599c68f87169b2f274fbe0c25
Author: Vitaly Terentyev <[email protected]>
AuthorDate: Tue Jul 7 19:02:28 2026 +0400
Refactoring
---
it/clickhouse/build.gradle | 2 +-
it/common/build.gradle | 20 +++++++----
.../it/common/artifacts/matchers/package-info.java | 20 -----------
it/common/src/main/resources/test-artifact.json | 1 +
it/google-cloud-platform/build.gradle | 18 ----------
it/iceberg/build.gradle | 41 +---------------------
.../org/apache/beam/it/iceberg/IcebergIOLT.java | 4 +--
it/truthmatchers/build.gradle | 6 +++-
.../beam/it/truthmatchers}/ArtifactAsserts.java | 4 +--
.../beam/it/truthmatchers}/ArtifactsSubject.java | 10 +++---
10 files changed, 30 insertions(+), 96 deletions(-)
diff --git a/it/clickhouse/build.gradle b/it/clickhouse/build.gradle
index 858d0dea920..9fa1c8634a9 100644
--- a/it/clickhouse/build.gradle
+++ b/it/clickhouse/build.gradle
@@ -52,4 +52,4 @@ dependencies {
permitUnusedDeclared
"com.clickhouse:client-v2:$clickhouse_java_client_version:all"
-}
\ No newline at end of file
+}
diff --git a/it/common/build.gradle b/it/common/build.gradle
index dc789c08fd3..62dd45ddacf 100644
--- a/it/common/build.gradle
+++ b/it/common/build.gradle
@@ -29,32 +29,40 @@ dependencies {
implementation
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation project(path: ":runners:google-cloud-dataflow-java")
- implementation project(path: ":sdks:java:testing:test-utils")
implementation 'com.google.cloud:google-cloud-bigquery'
implementation 'com.google.cloud:google-cloud-monitoring'
implementation 'com.google.cloud:google-cloud-storage'
+ implementation 'org.apache.hadoop:hadoop-common:3.3.6'
+ implementation 'org.apache.avro:avro:1.11.1'
+ implementation 'org.apache.parquet:parquet-avro:1.15.2'
+ implementation 'org.apache.parquet:parquet-common:1.15.2'
+ implementation 'org.apache.parquet:parquet-hadoop:1.15.2'
provided 'com.google.api.grpc:proto-google-cloud-monitoring-v3'
+ implementation library.java.jackson_core
+ implementation library.java.jackson_databind
implementation library.java.gax
implementation library.java.google_api_services_dataflow
implementation library.java.google_auth_library_credentials
implementation library.java.google_auth_library_oauth2_http
- implementation library.java.google_api_services_bigquery
implementation library.java.vendored_guava_32_1_2_jre
implementation library.java.slf4j_api
implementation library.java.commons_lang3
implementation library.java.failsafe
implementation library.java.google_code_gson
+ implementation library.java.google_cloud_core
+ implementation library.java.google_api_client
implementation library.java.google_http_client
implementation library.java.guava
implementation library.java.protobuf_java_util
implementation library.java.protobuf_java
implementation library.java.junit
-
+ testImplementation library.java.mockito_inline
+ testRuntimeOnly library.java.slf4j_simple
// TODO: excluding Guava until Truth updates it to >32.1.x
testImplementation(library.java.truth) {
- exclude group: 'com.google.guava', module: 'guava'
+ exclude group: 'com.google.guava', module: 'guava'
}
- testImplementation library.java.mockito_inline
- testRuntimeOnly library.java.slf4j_simple
+ testImplementation project(path: ":runners:direct-java", configuration:
"shadow")
+ testImplementation project(path: ":sdks:java:testing:test-utils")
testImplementation project(path: ":sdks:java:extensions:protobuf",
configuration: "testRuntimeMigration")
}
diff --git
a/it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/package-info.java
b/it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/package-info.java
deleted file mode 100644
index 1167e1b6cc2..00000000000
---
a/it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Package for Artifact Truth matchers / subjects to have reusable
assertions. */
-package org.apache.beam.it.common.artifacts.matchers;
diff --git a/it/common/src/main/resources/test-artifact.json
b/it/common/src/main/resources/test-artifact.json
new file mode 100644
index 00000000000..551c80d14a6
--- /dev/null
+++ b/it/common/src/main/resources/test-artifact.json
@@ -0,0 +1 @@
+["This is a test artifact."]
\ No newline at end of file
diff --git a/it/google-cloud-platform/build.gradle
b/it/google-cloud-platform/build.gradle
index dd8376e3569..164a75c06ba 100644
--- a/it/google-cloud-platform/build.gradle
+++ b/it/google-cloud-platform/build.gradle
@@ -31,47 +31,29 @@ evaluationDependsOn(":it:common")
dependencies {
implementation
enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
- implementation project(path: ":sdks:java:core", configuration: "shadow")
- implementation project(path: ":runners:google-cloud-dataflow-java")
implementation project(path: ":it:common")
implementation project(path: ":it:conditions")
implementation project(path: ":it:truthmatchers")
- implementation project(path: ":sdks:java:testing:test-utils")
implementation library.java.failsafe
implementation library.java.google_api_services_dataflow
implementation library.java.junit
implementation library.java.slf4j_api
- implementation(library.java.truth) {
- exclude group: 'com.google.guava', module: 'guava'
- }
implementation library.java.vendored_guava_32_1_2_jre
implementation library.java.jackson_core
implementation library.java.jackson_databind
- implementation 'org.apache.hadoop:hadoop-common:3.3.5'
- implementation 'org.apache.avro:avro:1.11.1'
- implementation 'org.apache.parquet:parquet-avro:1.15.2'
- implementation 'org.apache.parquet:parquet-common:1.15.2'
- implementation 'org.apache.parquet:parquet-hadoop:1.15.2'
implementation 'org.apache.commons:commons-lang3:3.9'
implementation library.java.gax
implementation library.java.google_api_common
implementation library.java.google_api_client
- implementation library.java.google_api_services_bigquery
implementation library.java.google_auth_library_credentials
implementation library.java.google_auth_library_oauth2_http
- implementation library.java.google_http_client
implementation library.java.guava
- implementation library.java.protobuf_java_util
implementation library.java.protobuf_java
implementation library.java.threetenbp
implementation 'org.awaitility:awaitility:4.2.0'
- implementation 'joda-time:joda-time:2.10.10'
// Google Cloud Dependencies
implementation library.java.google_cloud_core
- implementation 'com.google.cloud:google-cloud-storage'
implementation 'com.google.cloud:google-cloud-bigquery'
- implementation 'com.google.cloud:google-cloud-monitoring'
- provided 'com.google.api.grpc:proto-google-cloud-monitoring-v3'
implementation 'com.google.cloud:google-cloud-bigtable'
implementation 'com.google.cloud:google-cloud-spanner'
implementation 'com.google.cloud:google-cloud-pubsub'
diff --git a/it/iceberg/build.gradle b/it/iceberg/build.gradle
index 0d4e648151b..f11374a1ca4 100644
--- a/it/iceberg/build.gradle
+++ b/it/iceberg/build.gradle
@@ -31,7 +31,6 @@ ext.summary = "Integration test utilities for Iceberg."
def iceberg_version = "1.10.0"
def parquet_version = "1.16.0"
-def orc_version = "1.9.6"
def hive_version = "3.1.3"
def hadoopVersions = [
@@ -43,27 +42,8 @@ hadoopVersions.each {kv ->
configurations.create("hadoopVersion$kv.key")}
evaluationDependsOn(":it:common")
dependencies {
- implementation library.java.vendored_guava_32_1_2_jre
- implementation project(path: ":sdks:java:core", configuration: "shadow")
- implementation project(path: ":model:pipeline", configuration: "shadow")
- implementation library.java.avro
- implementation library.java.slf4j_api
- implementation library.java.joda_time
- implementation "org.apache.parquet:parquet-column:$parquet_version"
- implementation "org.apache.parquet:parquet-hadoop:$parquet_version"
- implementation "org.apache.parquet:parquet-common:$parquet_version"
- implementation project(":sdks:java:io:parquet")
- implementation "org.apache.orc:orc-core:$orc_version"
- implementation "org.apache.iceberg:iceberg-core:$iceberg_version"
- implementation "org.apache.iceberg:iceberg-api:$iceberg_version"
- implementation "org.apache.iceberg:iceberg-parquet:$iceberg_version"
- implementation "org.apache.iceberg:iceberg-orc:$iceberg_version"
- implementation "org.apache.iceberg:iceberg-data:$iceberg_version"
- implementation "org.apache.hadoop:hadoop-common:3.3.6"
-
provided "org.immutables:value:2.8.8"
permitUnusedDeclared "org.immutables:value:2.8.8"
- implementation library.java.vendored_calcite_1_40_0
runtimeOnly "org.apache.iceberg:iceberg-gcp:$iceberg_version"
runtimeOnly "org.apache.iceberg:iceberg-aws:$iceberg_version"
runtimeOnly "org.apache.iceberg:iceberg-aws-bundle:$iceberg_version"
@@ -113,29 +93,10 @@ dependencies {
"hadoopVersion$kv.key"
"org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value"
}
- implementation project(path: ":runners:google-cloud-dataflow-java")
- implementation project(path: ":it:common")
- implementation project(path: ":it:conditions")
- implementation project(path: ":it:truthmatchers")
- implementation project(path: ":sdks:java:testing:test-utils")
- implementation library.java.failsafe
- implementation library.java.google_api_services_dataflow
- implementation(library.java.truth) {
- exclude group: 'com.google.guava', module: 'guava'
- }
- implementation library.java.jackson_core
- implementation library.java.jackson_databind
- implementation 'org.apache.commons:commons-lang3:3.9'
- implementation library.java.google_api_client
- implementation library.java.google_http_client
- implementation library.java.guava
- implementation library.java.protobuf_java_util
- implementation library.java.protobuf_java
- implementation library.java.threetenbp
-
testImplementation project(path: ":sdks:java:extensions:protobuf",
configuration: "testRuntimeMigration")
testImplementation project(path: ":sdks:java:io:synthetic")
testImplementation project(path: ":sdks:java:io:iceberg")
+ testImplementation project(path: ":it:truthmatchers")
testImplementation project(path: ":it:common", configuration:
"testRuntimeMigration")
testImplementation library.java.mockito_inline
testImplementation project(path:
":sdks:java:extensions:google-cloud-platform-core", configuration:
"testRuntimeMigration")
diff --git
a/it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java
b/it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java
index b4e477f8ec0..683ff4529b6 100644
--- a/it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java
+++ b/it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java
@@ -47,7 +47,6 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.catalog.TableIdentifier;
import org.junit.AfterClass;
import org.junit.Before;
@@ -133,8 +132,7 @@ public final class IcebergIOLT extends IOLoadTestBase {
tableIdentifier = TableIdentifier.of(NAMESPACE, "table_" +
uniqueSuffix.replace("-", "_"));
Map<String, String> catalogProperties =
- ImmutableMap.of(
- "type", CatalogUtil.ICEBERG_CATALOG_TYPE_HADOOP, "warehouse",
warehouseLocation);
+ ImmutableMap.of("type", "hadoop", "warehouse", warehouseLocation);
catalogConfig =
IcebergCatalogConfig.builder()
diff --git a/it/truthmatchers/build.gradle b/it/truthmatchers/build.gradle
index fd54ad591a0..2e21c1b5df0 100644
--- a/it/truthmatchers/build.gradle
+++ b/it/truthmatchers/build.gradle
@@ -27,11 +27,15 @@ ext.summary = "Truth matchers for integration tests."
dependencies {
implementation project(path: ":it:common")
+ implementation 'org.apache.avro:avro:1.11.1'
// TODO: excluding Guava until Truth updates it to >32.1.x
implementation(library.java.truth) {
exclude group: 'com.google.guava', module: 'guava'
}
+ implementation library.java.jackson_core
+ implementation library.java.jackson_databind
implementation library.java.guava
+ implementation library.java.vendored_guava_32_1_2_jre
permitUsedUndeclared library.java.guava
implementation library.java.google_code_gson
-}
\ No newline at end of file
+}
diff --git
a/it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/ArtifactAsserts.java
b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ArtifactAsserts.java
similarity index 95%
rename from
it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/ArtifactAsserts.java
rename to
it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ArtifactAsserts.java
index 48d04edadc7..92407e5a2fc 100644
---
a/it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/ArtifactAsserts.java
+++
b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ArtifactAsserts.java
@@ -15,12 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.it.common.artifacts.matchers;
+package org.apache.beam.it.truthmatchers;
+import com.google.common.truth.Truth;
import java.util.List;
import org.apache.avro.generic.GenericRecord;
import org.apache.beam.it.common.artifacts.Artifact;
-import org.apache.beam.it.truthmatchers.RecordsSubject;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
public class ArtifactAsserts {
diff --git
a/it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/ArtifactsSubject.java
b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ArtifactsSubject.java
similarity index 93%
rename from
it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/ArtifactsSubject.java
rename to
it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ArtifactsSubject.java
index 7a49bd22ff9..e2518280345 100644
---
a/it/common/src/main/java/org/apache/beam/it/common/artifacts/matchers/ArtifactsSubject.java
+++
b/it/truthmatchers/src/main/java/org/apache/beam/it/truthmatchers/ArtifactsSubject.java
@@ -15,10 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.it.common.artifacts.matchers;
+package org.apache.beam.it.truthmatchers;
-import static
org.apache.beam.it.common.artifacts.matchers.ArtifactAsserts.assertThatGenericRecords;
-import static
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing.sha256;
+import static
org.apache.beam.it.truthmatchers.ArtifactAsserts.assertThatGenericRecords;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -34,7 +33,7 @@ import org.apache.beam.it.common.artifacts.Artifact;
import org.apache.beam.it.common.artifacts.utils.AvroTestUtil;
import org.apache.beam.it.common.artifacts.utils.JsonTestUtil;
import org.apache.beam.it.common.artifacts.utils.ParquetTestUtil;
-import org.apache.beam.it.truthmatchers.RecordsSubject;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing;
/**
* Subject that has assertion operations for artifact lists (GCS files),
usually coming from the
@@ -97,7 +96,8 @@ public final class ArtifactsSubject extends Subject {
*/
public void hasHash(String hash) {
if (actual.stream()
- .noneMatch(artifact ->
sha256().hashBytes(artifact.contents()).toString().equals(hash))) {
+ .noneMatch(
+ artifact ->
Hashing.sha256().hashBytes(artifact.contents()).toString().equals(hash))) {
failWithActual("expected to contain hash", hash);
}
}
