>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21351?usp=email )
Change subject: [NO ISSUE][EXT]: allow custom iceberg catalog endpoint in tests ...................................................................... [NO ISSUE][EXT]: allow custom iceberg catalog endpoint in tests Ext-ref: MB-72418 Change-Id: I8667ce486c480757784046b77747a9b9f756026b Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21351 Integration-Tests: Hussain Towaileb <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Tested-by: Hussain Towaileb <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java 4 files changed, 42 insertions(+), 24 deletions(-) Approvals: Hussain Towaileb: Looks good to me, but someone else must approve; Verified; Verified Jenkins: Verified; Verified Michael Blow: Looks good to me, approved diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java index d745c55..8cbb5eb 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java @@ -115,9 +115,9 @@ public static final String SNAPSHOT_2_TIMESTAMP_DATETIME_PLACEHOLDER = "%SNAPSHOT_2_TIMESTAMP_DATETIME_PLACEHOLDER%"; public static final String SNAPSHOT_3_TIMESTAMP_DATE_PLACEHOLDER = "%SNAPSHOT_3_TIMESTAMP_DATE_PLACEHOLDER%"; - public static long SNAPSHOT_1_SNAPSHOT_ID_VALUE; - public static long SNAPSHOT_1_TIMESTAMP_LONG_VALUE; - public static String SNAPSHOT_2_TIMESTAMP_DATETIME_VALUE; - public static String SNAPSHOT_3_TIMESTAMP_DATE_VALUE; + public static long snapshot1SnapshotIdValue; + public static long snapshot1TimestampLongValue; + public static String snapshot2TimestampDatetimeValue; + public static String snapshot3TimestampDateValue; } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 48e1691..3a77ad2 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -41,14 +41,15 @@ import static org.apache.asterix.test.common.TestConstants.Azure.TENANT_ID_DEFAULT; import static org.apache.asterix.test.common.TestConstants.Azure.TENANT_ID_PLACEHOLDER; import static org.apache.asterix.test.common.TestConstants.Azure.sasToken; +import static org.apache.asterix.test.common.TestConstants.Iceberg.NESSIE_SERVER_URI_PLACEHOLDER; import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_1_SNAPSHOT_ID_PLACEHOLDER; -import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_1_SNAPSHOT_ID_VALUE; import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_1_TIMESTAMP_LONG_PLACEHOLDER; -import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_1_TIMESTAMP_LONG_VALUE; import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_2_TIMESTAMP_DATETIME_PLACEHOLDER; -import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_2_TIMESTAMP_DATETIME_VALUE; import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_3_TIMESTAMP_DATE_PLACEHOLDER; -import static org.apache.asterix.test.common.TestConstants.Iceberg.SNAPSHOT_3_TIMESTAMP_DATE_VALUE; +import static org.apache.asterix.test.common.TestConstants.Iceberg.snapshot1SnapshotIdValue; +import static org.apache.asterix.test.common.TestConstants.Iceberg.snapshot1TimestampLongValue; +import static org.apache.asterix.test.common.TestConstants.Iceberg.snapshot2TimestampDatetimeValue; +import static org.apache.asterix.test.common.TestConstants.Iceberg.snapshot3TimestampDateValue; import static org.apache.hyracks.util.NetworkUtil.toHostPort; import static org.apache.hyracks.util.file.FileUtil.canonicalize; @@ -121,7 +122,6 @@ import org.apache.asterix.common.utils.Servlets; import org.apache.asterix.lang.sqlpp.util.SqlppStatementUtil; import org.apache.asterix.runtime.evaluators.common.NumberUtils; -import org.apache.asterix.test.iceberg.IcebergTest; import org.apache.asterix.test.server.ITestServer; import org.apache.asterix.test.server.TestServerProvider; import org.apache.asterix.testframework.context.TestCaseContext; @@ -301,6 +301,9 @@ public String executorId = null; private String s3mockTemplate = null; + // iceberg parameters + private String nessieUri = null; + public TestExecutor() { this(Collections.singletonList( InetSocketAddress.createUnresolved(Inet4Address.getLoopbackAddress().getHostAddress(), 19002))); @@ -2528,13 +2531,14 @@ } protected String replaceIcebergPlaceholders(String str) { - str = str.replace(TestConstants.Iceberg.NESSIE_SERVER_URI_PLACEHOLDER, IcebergTest.NESSIE_URI); - str = str.replace(SNAPSHOT_1_SNAPSHOT_ID_PLACEHOLDER, String.valueOf(SNAPSHOT_1_SNAPSHOT_ID_VALUE)); - str = str.replace(SNAPSHOT_1_TIMESTAMP_LONG_PLACEHOLDER, String.valueOf(SNAPSHOT_1_TIMESTAMP_LONG_VALUE)); - str = SNAPSHOT_2_TIMESTAMP_DATETIME_VALUE != null - ? str.replace(SNAPSHOT_2_TIMESTAMP_DATETIME_PLACEHOLDER, SNAPSHOT_2_TIMESTAMP_DATETIME_VALUE) : str; - str = SNAPSHOT_3_TIMESTAMP_DATE_VALUE != null - ? str.replace(SNAPSHOT_3_TIMESTAMP_DATE_PLACEHOLDER, SNAPSHOT_3_TIMESTAMP_DATE_VALUE) : str; + str = getNessieEndpointDefault() != null + ? str.replace(NESSIE_SERVER_URI_PLACEHOLDER, getNessieEndpointDefault()) : str; + str = str.replace(SNAPSHOT_1_SNAPSHOT_ID_PLACEHOLDER, String.valueOf(snapshot1SnapshotIdValue)); + str = str.replace(SNAPSHOT_1_TIMESTAMP_LONG_PLACEHOLDER, String.valueOf(snapshot1TimestampLongValue)); + str = snapshot2TimestampDatetimeValue != null + ? str.replace(SNAPSHOT_2_TIMESTAMP_DATETIME_PLACEHOLDER, snapshot2TimestampDatetimeValue) : str; + str = snapshot3TimestampDateValue != null + ? str.replace(SNAPSHOT_3_TIMESTAMP_DATE_PLACEHOLDER, snapshot3TimestampDateValue) : str; return str; } @@ -2580,7 +2584,6 @@ str = str.replace(TestConstants.S3_API_CERTIFICATES_PLACEHOLDER, getS3ApiCertificatesDefault()); str = str.replace(TestConstants.S3_DDL_CERTIFICATES_PLACEHOLDER, getS3DdlCertificatesDefault()); str = str.replace(TestConstants.S3_HTTP_CERTIFICATES_PLACEHOLDER, getS3HttpCertificatesDefault()); - return str; } @@ -2596,7 +2599,7 @@ return null; } - protected String getS3ServiceEndpointDefault() { + public String getS3ServiceEndpointDefault() { return TestConstants.S3_SERVICE_ENDPOINT_DEFAULT; } @@ -2622,6 +2625,14 @@ : ", \\\"certificates\\\":" + escapeForHttpBodyString(writeAsJsonString(List.of(certificates))); } + public String getNessieEndpointDefault() { + return nessieUri; + } + + public void setNessieEndpointDefault(String nessieUri) { + this.nessieUri = nessieUri; + } + private static String escapeForHttpBodyString(String value) { return value.replace("\\", "\\\\").replace("\"", "\\\""); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java index c2b731b..cc340ce 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/iceberg/IcebergTest.java @@ -130,8 +130,8 @@ private static GenericContainer<?> nessie; private static final DockerImageName NESSIE_IMAGE = DockerImageName.parse("ghcr.io/projectnessie/nessie:0.107.5"); private static final int NESSIE_PORT = 19120; - public static final String NESSIE_URI = "http://localhost:"; + NESSIE_PORT + "/api/v2"; - public static final String NESSIE_WAREHOUSE = "s3://" + ICEBERG_CONTAINER + "/nessie/warehouse"; + private static final String NESSIE_URI = "http://localhost:"; + NESSIE_PORT + "/api/v2"; + private static final String NESSIE_WAREHOUSE = "s3://" + ICEBERG_CONTAINER + "/nessie/warehouse"; private static final Namespace NAMESPACE = Namespace.of("my_namespace"); private static final TableIdentifier TABLE_ID = TableIdentifier.of(NAMESPACE, "users"); private static final TableIdentifier ALL_TYPES_TABLE_ID = TableIdentifier.of(NAMESPACE, "allTypes"); @@ -150,6 +150,7 @@ nessie = new GenericContainer<>(NESSIE_IMAGE).withExposedPorts(NESSIE_PORT); nessie.setPortBindings(List.of(NESSIE_PORT + ":" + NESSIE_PORT)); nessie.start(); + testExecutor.setNessieEndpointDefault(NESSIE_URI); prepareIcebergContainer(); prepareIcebergData(); @@ -208,8 +209,8 @@ long snap1 = writeSnapshot(table, templateV1, 1, 800); Thread.sleep(TimeUnit.SECONDS.toMillis(3)); - TestConstants.Iceberg.SNAPSHOT_1_SNAPSHOT_ID_VALUE = snap1; - TestConstants.Iceberg.SNAPSHOT_1_TIMESTAMP_LONG_VALUE = System.currentTimeMillis(); + TestConstants.Iceberg.snapshot1SnapshotIdValue = snap1; + TestConstants.Iceberg.snapshot1TimestampLongValue = System.currentTimeMillis(); Thread.sleep(TimeUnit.SECONDS.toMillis(3)); evolveSchemaAddColumn(table); @@ -217,7 +218,7 @@ GenericRecord templateV2 = GenericRecord.create(table.schema()); long snap2 = writeSnapshot(table, templateV2, 801, 1000); Thread.sleep(TimeUnit.SECONDS.toMillis(3)); - TestConstants.Iceberg.SNAPSHOT_2_TIMESTAMP_DATETIME_VALUE = + TestConstants.Iceberg.snapshot2TimestampDatetimeValue = LocalDateTime.now(ZoneOffset.UTC).truncatedTo(ChronoUnit.SECONDS).toString(); Thread.sleep(TimeUnit.SECONDS.toMillis(3)); @@ -233,7 +234,7 @@ delta.commit(); long snap3 = table.currentSnapshot().snapshotId(); - TestConstants.Iceberg.SNAPSHOT_3_TIMESTAMP_DATE_VALUE = LocalDate.now(ZoneOffset.UTC).plusDays(1).toString(); + TestConstants.Iceberg.snapshot3TimestampDateValue = LocalDate.now(ZoneOffset.UTC).plusDays(1).toString(); LOGGER.info("[DELETE] Committed RowDelta snapshotId={} snapshotTimestamp={}", snap3, table.currentSnapshot().timestampMillis()); printSnapshotSummary(table); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java index 24c6df6..a468e5b 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergConstants.java @@ -31,6 +31,12 @@ throw new AssertionError("do not instantiate"); } + public static final String ICEBERG_CATALOG_FIELD_NAME = "catalog"; + public static final String ICEBERG_NAMESPACE_FIELD_NAME = "namespace"; + public static final String ICEBERG_TABLE_FIELD_NAME = "table"; + public static final String ICEBERG_SNAPSHOT_ID_FIELD_NAME = "id"; + public static final String ICEBERG_SNAPSHOT_TIMESTAMP_FIELD_NAME = "timestamp_ms"; + public static final String ICEBERG_URI_PROPERTY_KEY = "uri"; public static final String ICEBERG_TABLE_FORMAT = "iceberg"; public static final String ICEBERG_CATALOG_NAME = "catalogName"; -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21351?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I8667ce486c480757784046b77747a9b9f756026b Gerrit-Change-Number: 21351 Gerrit-PatchSet: 8 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]>
