>From Michael Blow <[email protected]>: Michael Blow has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20143 )
Change subject: [NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage ...................................................................... [NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage MinIO does not accept lists on a . directory. Fix the GLOBAL_TXN_DIR_NAME to use instead of . to avoid this incompatibility Ext-ref: MB-67814 Change-Id: I8e6ffb62ef1c412e263388b8dabb9f2d3bb66db5 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20143 Reviewed-by: Michael Blow <[email protected]> Tested-by: Michael Blow <[email protected]> Integration-Tests: Jenkins <[email protected]> --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java 4 files changed, 57 insertions(+), 28 deletions(-) Approvals: Michael Blow: Looks good to me, approved; Verified Jenkins: Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java index 9e67c92..cb37cd0 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java @@ -42,7 +42,7 @@ private static final Logger LOGGER = LogManager.getLogger(); private static final int MOCK_SERVER_PORT = 8001; - public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:"; + MOCK_SERVER_PORT; + public static final String MOCK_SERVER_ENDPOINT = "http://127.0.0.1:"; + MOCK_SERVER_PORT; public static final String CLOUD_STORAGE_BUCKET = "cloud-storage-container"; public static final String STORAGE_DUMMY_FILE = "storage/dummy.txt"; public static final String MOCK_SERVER_REGION = "us-west-2"; @@ -81,7 +81,7 @@ LOGGER.info("S3 mock server started successfully"); S3ClientBuilder builder = S3Client.builder(); - URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server + URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) .endpointOverride(endpoint); S3Client client = builder.build(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java index 1aefadf..ba7f856 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java @@ -18,7 +18,7 @@ */ package org.apache.asterix.api.common; -import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_HOSTNAME; +import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_ENDPOINT; import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_REGION; import java.net.URI; @@ -79,7 +79,7 @@ LOGGER.info("S3 mock server started successfully"); S3ClientBuilder builder = S3Client.builder(); - URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server + URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) .endpointOverride(endpoint); S3Client client = builder.build(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 890e188..1a50f44 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -2419,21 +2419,7 @@ // For adapter placeholder, it means we have a template to replace if (placeholder.getName().equals("adapter")) { str = str.replace("%adapter%", placeholder.getValue()); - - // Early terminate if there are no template place holders to replace - if (noTemplateRequired(str)) { - continue; - } - - if (placeholder.getValue().equalsIgnoreCase("S3")) { - str = applyS3Substitution(str, placeholders); - } else if (placeholder.getValue().equalsIgnoreCase("AzureBlob")) { - str = applyAzureSubstitution(str, placeholders); - } else if (placeholder.getValue().equalsIgnoreCase("GCS")) { - str = applyGCSSubstitution(str, placeholders); - } else if (placeholder.getValue().equalsIgnoreCase("HDFS")) { - str = applyHDFSSubstitution(str, placeholders); - } + str = applyAdapterSubstitution(str, placeholder.getValue(), placeholders); } else { // Any other place holders, just replace with the value str = str.replace("%" + placeholder.getName() + "%", placeholder.getValue()); @@ -2455,6 +2441,19 @@ return str; } + protected String applyAdapterSubstitution(String str, String adapter, List<Placeholder> placeholders) { + if (adapter.equalsIgnoreCase("S3")) { + str = applyS3Substitution(str, placeholders); + } else if (adapter.equalsIgnoreCase("AzureBlob")) { + str = applyAzureSubstitution(str, placeholders); + } else if (adapter.equalsIgnoreCase("GCS")) { + str = applyGCSSubstitution(str, placeholders); + } else if (adapter.equalsIgnoreCase("HDFS")) { + str = applyHDFSSubstitution(str, placeholders); + } + return str; + } + protected String replaceExternalEndpoint(String str) { return str.replace(BLOB_ENDPOINT_PLACEHOLDER, BLOB_ENDPOINT_DEFAULT); } @@ -2494,18 +2493,30 @@ str = setS3TemplateDefault(str); } - // Set to default if not replaced - if (isReplaced && !hasRegion) { - str = str.replace(TestConstants.S3_REGION_PLACEHOLDER, TestConstants.S3_REGION_DEFAULT); - } - - if (isReplaced && !hasServiceEndpoint) { - str = str.replace(TestConstants.S3_SERVICE_ENDPOINT_PLACEHOLDER, TestConstants.S3_SERVICE_ENDPOINT_DEFAULT); - } + str = str.replace(TestConstants.S3_REGION_PLACEHOLDER, getS3RegionDefault()); + str = str.replace(TestConstants.S3_SERVICE_ENDPOINT_PLACEHOLDER, getS3ServiceEndpointDefault()); + str = str.replace(TestConstants.S3_ACCESS_KEY_ID_PLACEHOLDER, getS3AccessKeyIdDefault()); + str = str.replace(TestConstants.S3_SECRET_ACCESS_KEY_PLACEHOLDER, getS3SecretAccessKeyDefault()); return str; } + protected String getS3AccessKeyIdDefault() { + return TestConstants.S3_ACCESS_KEY_ID_DEFAULT; + } + + protected String getS3SecretAccessKeyDefault() { + return TestConstants.S3_SECRET_ACCESS_KEY_DEFAULT; + } + + protected String getS3ServiceEndpointDefault() { + return TestConstants.S3_SERVICE_ENDPOINT_DEFAULT; + } + + protected String getS3RegionDefault() { + return TestConstants.S3_REGION_DEFAULT; + } + protected String setS3Template(String str) { return str.replace("%template%", TestConstants.S3_TEMPLATE); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java index 627f170..0f18536 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java @@ -33,7 +33,7 @@ public class StorageConstants { public static final String METADATA_TXN_NOWAL_DIR_NAME = "mtd-txn-logs"; - public static final String GLOBAL_TXN_DIR_NAME = "."; + public static final String GLOBAL_TXN_DIR_NAME = ""; public static final String STORAGE_ROOT_DIR_NAME = "storage"; public static final String APPLICATION_ROOT_DIR_NAME = APP_DIR_NAME; public static final String INGESTION_LOGS_DIR_NAME = "ingestion_logs"; -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20143 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: phoenix Gerrit-Change-Id: I8e6ffb62ef1c412e263388b8dabb9f2d3bb66db5 Gerrit-Change-Number: 20143 Gerrit-PatchSet: 5 Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-MessageType: merged
