>From Michael Blow <[email protected]>:
Michael Blow has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21047?usp=email )
Change subject: [NO ISSUE][*DB][STO] Refactoring to enable alternate cloud props
......................................................................
[NO ISSUE][*DB][STO] Refactoring to enable alternate cloud props
Ext-ref: MB-70796
Change-Id: Id21a22500a0f6d390e92435284fc3199992f8e6c
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSClientConfig.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
A
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ICloudProperties.java
6 files changed, 120 insertions(+), 11 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/47/21047/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index c675648..1a0fa78 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -24,7 +24,7 @@
import
org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageCloudClient;
import org.apache.asterix.cloud.clients.google.gcs.GCSClientConfig;
import org.apache.asterix.cloud.clients.google.gcs.GCSCloudClient;
-import org.apache.asterix.common.config.CloudProperties;
+import org.apache.asterix.common.config.ICloudProperties;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.cloud.util.CloudRetryableRequestUtil;
@@ -38,7 +38,7 @@
throw new AssertionError("do not instantiate");
}
- public static ICloudClient getClient(CloudProperties cloudProperties,
ICloudGuardian guardian)
+ public static ICloudClient getClient(ICloudProperties cloudProperties,
ICloudGuardian guardian)
throws HyracksDataException {
String storageScheme = cloudProperties.getStorageScheme();
ICloudClient cloudClient = switch (storageScheme.toLowerCase()) {
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index 4ff5ed0..b6c40b5 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -23,7 +23,7 @@
import java.util.Map;
import java.util.Objects;
-import org.apache.asterix.common.config.CloudProperties;
+import org.apache.asterix.common.config.ICloudProperties;
import org.apache.asterix.external.util.aws.AwsConstants;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
@@ -88,7 +88,13 @@
this.roundRobinDnsResolver = roundRobinDnsResolver;
}
- public static S3ClientConfig of(CloudProperties cloudProperties) {
+ public static S3ClientConfig forValidation(String region, String endpoint,
String prefix, boolean anonymousAuth,
+ Collection<String> certificates, boolean forcePathStyle, boolean
disableSslVerify) {
+ return new S3ClientConfig(region, endpoint, prefix, anonymousAuth,
certificates, 0, 8 * 1024 * 1024, 1, 0, 0, 0,
+ forcePathStyle, disableSslVerify, false, 0, 0, -1,
S3ParallelDownloaderClientType.ASYNC, false);
+ }
+
+ public static S3ClientConfig of(ICloudProperties cloudProperties) {
return new S3ClientConfig(cloudProperties.getStorageRegion(),
cloudProperties.getStorageEndpoint(),
cloudProperties.getStoragePrefix(),
cloudProperties.isStorageAnonymousAuth(),
cloudProperties.getStorageCertificates(),
cloudProperties.getProfilerLogInterval(),
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
index 725e3d7..1bf6128 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
@@ -18,10 +18,14 @@
*/
package org.apache.asterix.cloud.clients.azure.blobstorage;
+import static
org.apache.asterix.common.config.CloudProperties.HTTP_CONNECTION_ACQUIRE_TIMEOUT;
+import static
org.apache.asterix.common.config.CloudProperties.MAX_HTTP_CONNECTIONS;
+import static
org.apache.asterix.common.config.CloudProperties.MAX_PENDING_HTTP_CONNECTIONS;
+
import java.util.Map;
import java.util.Objects;
-import org.apache.asterix.common.config.CloudProperties;
+import org.apache.asterix.common.config.ICloudProperties;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.azure.AzureConstants;
@@ -54,8 +58,7 @@
public AzBlobStorageClientConfig(String region, String endpoint, String
prefix, boolean anonymousAuth,
long profilerLogInterval, String bucket, int writeBufferSize) {
this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
bucket, 1, 0, 0, writeBufferSize, false,
- null, CloudProperties.MAX_HTTP_CONNECTIONS,
CloudProperties.MAX_PENDING_HTTP_CONNECTIONS,
- CloudProperties.HTTP_CONNECTION_ACQUIRE_TIMEOUT);
+ null, MAX_HTTP_CONNECTIONS, MAX_PENDING_HTTP_CONNECTIONS,
HTTP_CONNECTION_ACQUIRE_TIMEOUT);
}
public AzBlobStorageClientConfig(String region, String endpoint, String
prefix, boolean anonymousAuth,
@@ -81,7 +84,14 @@
this.accessTier = accessTier;
}
- public static AzBlobStorageClientConfig of(CloudProperties
cloudProperties) {
+ public static AzBlobStorageClientConfig forValidation(String region,
String endpoint, String prefix,
+ boolean anonymousAuth, String bucket, boolean disableSslVerify) {
+ return new AzBlobStorageClientConfig(region, endpoint, prefix,
anonymousAuth, 0, bucket, 1, 0, 0,
+ 8 * 1024 * 1024, disableSslVerify, null, MAX_HTTP_CONNECTIONS,
MAX_PENDING_HTTP_CONNECTIONS,
+ HTTP_CONNECTION_ACQUIRE_TIMEOUT);
+ }
+
+ public static AzBlobStorageClientConfig of(ICloudProperties
cloudProperties) {
return new
AzBlobStorageClientConfig(cloudProperties.getStorageRegion(),
cloudProperties.getStorageEndpoint(),
cloudProperties.getStoragePrefix(),
cloudProperties.isStorageAnonymousAuth(),
cloudProperties.getProfilerLogInterval(),
cloudProperties.getStorageBucket(),
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSClientConfig.java
index 6af80a4..39d65b6 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSClientConfig.java
@@ -24,7 +24,7 @@
import java.io.IOException;
import java.util.Map;
-import org.apache.asterix.common.config.CloudProperties;
+import org.apache.asterix.common.config.ICloudProperties;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import com.google.auth.oauth2.GoogleCredentials;
@@ -64,7 +64,7 @@
this(region, endpoint, anonymousAuth, profilerLogInterval, 1, 0, 0,
writeBufferSize, prefix);
}
- public static GCSClientConfig of(CloudProperties cloudProperties) {
+ public static GCSClientConfig of(ICloudProperties cloudProperties) {
return new GCSClientConfig(cloudProperties.getStorageRegion(),
cloudProperties.getStorageEndpoint(),
cloudProperties.isStorageAnonymousAuth(),
cloudProperties.getProfilerLogInterval(),
cloudProperties.getTokenAcquireTimeout(),
cloudProperties.getWriteMaxRequestsPerSecond(),
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 9a7b4ed..b1bafae 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -42,7 +42,7 @@
import org.apache.hyracks.api.config.Section;
import org.apache.hyracks.util.StorageUtil;
-public class CloudProperties extends AbstractProperties {
+public class CloudProperties extends AbstractProperties implements
ICloudProperties {
public static final int MAX_HTTP_CONNECTIONS = 1000;
public static final int MAX_PENDING_HTTP_CONNECTIONS = 10000;
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ICloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ICloudProperties.java
new file mode 100644
index 0000000..0805bfa
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ICloudProperties.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.config;
+
+import static
org.apache.hyracks.util.annotations.AiProvenance.Agent.GPT_5_MINI;
+import static
org.apache.hyracks.util.annotations.AiProvenance.ContributionKind.GENERATED;
+import static
org.apache.hyracks.util.annotations.AiProvenance.Tool.GITHUB_COPILOT;
+
+import java.util.Collection;
+
+import org.apache.asterix.common.cloud.CloudCachePolicy;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Interface extracted from CloudProperties to allow consumers to depend on an
abstraction.
+ */
+@AiProvenance(agent = GPT_5_MINI, tool = GITHUB_COPILOT, contributionKind =
GENERATED)
+public interface ICloudProperties {
+
+ String getStorageScheme();
+
+ String getStorageBucket();
+
+ String getStoragePrefix();
+
+ String getStorageEndpoint();
+
+ String getStorageRegion();
+
+ boolean isStorageAnonymousAuth();
+
+ Collection<String> getStorageCertificates();
+
+ CloudCachePolicy getCloudCachePolicy();
+
+ double getStorageAllocationPercentage();
+
+ double getStorageSweepThresholdPercentage();
+
+ int getStorageDiskMonitorInterval();
+
+ long getStorageIndexInactiveDurationThreshold();
+
+ boolean isStorageDebugModeEnabled();
+
+ long getStorageDebugSweepThresholdSize();
+
+ long getProfilerLogInterval();
+
+ long getTokenAcquireTimeout();
+
+ int getWriteMaxRequestsPerSecond();
+
+ int getReadMaxRequestsPerSecond();
+
+ int getWriteBufferSize();
+
+ int getEvictionPlanReevaluationThreshold();
+
+ int getRequestsMaxHttpConnections();
+
+ int getRequestsMaxPendingHttpConnections();
+
+ int getRequestsHttpConnectionAcquireTimeout();
+
+ boolean isStorageForcePathStyle();
+
+ boolean isStorageDisableSSLVerify();
+
+ boolean isStorageListEventuallyConsistent();
+
+ String getS3ParallelDownloaderClientType();
+
+ int getS3ReadTimeoutInSeconds();
+
+ boolean useRoundRobinDnsResolver();
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21047?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: phoenix
Gerrit-Change-Id: Id21a22500a0f6d390e92435284fc3199992f8e6c
Gerrit-Change-Number: 21047
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <[email protected]>
