>From Michael Blow <[email protected]>:
Michael Blow has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21028?usp=email )
Change subject: [NO ISSUE][*DB][STO] Support explicit blob storage certificates
......................................................................
[NO ISSUE][*DB][STO] Support explicit blob storage certificates
Ext-ref: MB-68233
Change-Id: I0af4a59ff219de1a5213d90719a2995f8ebfe7b3
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
M
asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
5 files changed, 44 insertions(+), 13 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/28/21028/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index c9046a7..3ed608a 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -36,6 +36,7 @@
private final String endpoint;
private final String prefix;
private final boolean anonymousAuth;
+ private final String certificate;
private final long profilerLogInterval;
private final int writeBufferSize;
private final long tokenAcquireTimeout;
@@ -51,14 +52,14 @@
private final S3ParallelDownloaderClientType parallelDownloaderClientType;
private final boolean roundRobinDnsResolver;
- public S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth,
+ public S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth, String certificate,
long profilerLogInterval, int writeBufferSize,
S3ParallelDownloaderClientType parallelDownloaderClientType,
boolean roundRobinDnsResolver) {
- this(region, endpoint, prefix, anonymousAuth, profilerLogInterval,
writeBufferSize, 1, 0, 0, 0, false, false,
- false, 0, 0, -1, parallelDownloaderClientType,
roundRobinDnsResolver);
+ this(region, endpoint, prefix, anonymousAuth, certificate,
profilerLogInterval, writeBufferSize, 1, 0, 0, 0,
+ false, false, false, 0, 0, -1, parallelDownloaderClientType,
roundRobinDnsResolver);
}
- private S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth,
+ private S3ClientConfig(String region, String endpoint, String prefix,
boolean anonymousAuth, String certificate,
long profilerLogInterval, int writeBufferSize, long
tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
int readMaxRequestsPerSeconds, int requestsMaxHttpConnections,
boolean forcePathStyle,
boolean disableSslVerify, boolean storageListEventuallyConsistent,
int requestsMaxPendingHttpConnections,
@@ -68,6 +69,7 @@
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
this.anonymousAuth = anonymousAuth;
+ this.certificate = certificate;
this.profilerLogInterval = profilerLogInterval;
this.writeBufferSize = writeBufferSize;
this.tokenAcquireTimeout = tokenAcquireTimeout;
@@ -87,11 +89,11 @@
public static S3ClientConfig of(CloudProperties cloudProperties) {
return new S3ClientConfig(cloudProperties.getStorageRegion(),
cloudProperties.getStorageEndpoint(),
cloudProperties.getStoragePrefix(),
cloudProperties.isStorageAnonymousAuth(),
- cloudProperties.getProfilerLogInterval(),
cloudProperties.getWriteBufferSize(),
- cloudProperties.getTokenAcquireTimeout(),
cloudProperties.getWriteMaxRequestsPerSecond(),
- cloudProperties.getReadMaxRequestsPerSecond(),
cloudProperties.getRequestsMaxHttpConnections(),
- cloudProperties.isStorageForcePathStyle(),
cloudProperties.isStorageDisableSSLVerify(),
- cloudProperties.isStorageListEventuallyConsistent(),
+ cloudProperties.getStorageCertificate(),
cloudProperties.getProfilerLogInterval(),
+ cloudProperties.getWriteBufferSize(),
cloudProperties.getTokenAcquireTimeout(),
+ cloudProperties.getWriteMaxRequestsPerSecond(),
cloudProperties.getReadMaxRequestsPerSecond(),
+ cloudProperties.getRequestsMaxHttpConnections(),
cloudProperties.isStorageForcePathStyle(),
+ cloudProperties.isStorageDisableSSLVerify(),
cloudProperties.isStorageListEventuallyConsistent(),
cloudProperties.getRequestsMaxPendingHttpConnections(),
cloudProperties.getRequestsHttpConnectionAcquireTimeout(),
cloudProperties.getS3ReadTimeoutInSeconds(),
S3ParallelDownloaderClientType.valueOf(cloudProperties.getS3ParallelDownloaderClientType()),
@@ -125,10 +127,11 @@
// Dummy values;
String region = "";
String prefix = "";
+ String certificate = "";
boolean anonymousAuth = false;
- return new S3ClientConfig(region, endPoint, prefix, anonymousAuth,
profilerLogInterval, writeBufferSize,
- S3ParallelDownloaderClientType.ASYNC, false);
+ return new S3ClientConfig(region, endPoint, prefix, anonymousAuth,
certificate, profilerLogInterval,
+ writeBufferSize, S3ParallelDownloaderClientType.ASYNC, false);
}
public String getRegion() {
@@ -188,6 +191,10 @@
return disableSslVerify;
}
+ public String getCertificate() {
+ return certificate;
+ }
+
public boolean isForcePathStyle() {
return forcePathStyle;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index ab27ad4..d0bb06c 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -396,10 +396,13 @@
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.endpointOverride(URI.create(config.getEndpoint()));
}
+ ApacheHttpClient.Builder apacheBuilder = ApacheHttpClient.builder();
if (config.isDisableSslVerify()) {
customHttpConfigBuilder.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES,
true);
+ } else if (config.getCertificate() != null &&
!config.getCertificate().isEmpty()) {
+
apacheBuilder.tlsTrustManagersProvider(S3TrustManagerProvider.create(config.getCertificate()));
}
- SdkHttpClient httpClient =
ApacheHttpClient.builder().buildWithDefaults(customHttpConfigBuilder.build());
+ SdkHttpClient httpClient =
apacheBuilder.buildWithDefaults(customHttpConfigBuilder.build());
builder.httpClient(httpClient);
awsClients.setConsumingClient(builder.build());
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
index c1a92a6..d514269 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
@@ -49,6 +49,7 @@
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
+import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
@@ -225,6 +226,10 @@
NettyNioAsyncHttpClient.Builder nettyBuilder =
NettyNioAsyncHttpClient.builder().eventLoopGroup(SHARED_EVENT_LOOP);
+ if (!config.isDisableSslVerify() && config.getCertificate() != null &&
!config.getCertificate().isEmpty()) {
+
nettyBuilder.tlsTrustManagersProvider(S3TrustManagerProvider.create(config.getCertificate()));
+ }
+
SdkAsyncHttpClient nettyClient =
nettyBuilder.buildWithDefaults(httpOptions.build());
if (config.useRoundRobinDnsResolver()) {
try {
@@ -241,6 +246,11 @@
}
private static S3AsyncClient createS3CrtAsyncClient(S3ClientConfig config)
{
+ if (config.getCertificate() != null &&
!config.getCertificate().isEmpty()) {
+ LOGGER.warn("Custom CA certificate is not supported with the CRT
S3 client. "
+ + "The certificate will be ignored for parallel downloads.
"
+ + "Consider using the 'async' parallel downloader client
type instead.");
+ }
S3CrtAsyncClientBuilder builder = S3AsyncClient.crtBuilder();
builder.credentialsProvider(config.createCredentialsProvider());
builder.region(Region.of(config.getRegion()));
@@ -248,6 +258,9 @@
if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) {
builder.endpointOverride(URI.create(config.getEndpoint()));
}
+ if (config.isDisableSslVerify()) {
+
builder.httpConfiguration(S3CrtHttpConfiguration.builder().trustAllCertificatesEnabled(true).build());
+ }
return builder.build();
}
diff --git
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
index 382bb4f..0786034 100644
---
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
+++
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
@@ -67,7 +67,7 @@
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
LOGGER.info("Client created successfully");
int writeBufferSize = StorageUtil.getIntSizeInBytes(5,
StorageUtil.StorageUnit.MEGABYTE);
- S3ClientConfig config = new S3ClientConfig(MOCK_SERVER_REGION,
MOCK_SERVER_HOSTNAME, "", true, 0,
+ S3ClientConfig config = new S3ClientConfig(MOCK_SERVER_REGION,
MOCK_SERVER_HOSTNAME, "", true, "", 0,
writeBufferSize,
S3ClientConfig.S3ParallelDownloaderClientType.ASYNC, false);
CLOUD_CLIENT = new S3CloudClient(config,
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index abe6560..fd7a7e0 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -54,6 +54,7 @@
CLOUD_STORAGE_PREFIX(STRING, ""),
CLOUD_STORAGE_REGION(STRING, ""),
CLOUD_STORAGE_ENDPOINT(STRING, ""),
+ CLOUD_STORAGE_CERTIFICATE(STRING, ""),
CLOUD_STORAGE_ANONYMOUS_AUTH(BOOLEAN, false),
CLOUD_STORAGE_CACHE_POLICY(STRING, "selective"),
// 80% of the total disk space
@@ -106,6 +107,7 @@
case CLOUD_STORAGE_PREFIX:
case CLOUD_STORAGE_REGION:
case CLOUD_STORAGE_ENDPOINT:
+ case CLOUD_STORAGE_CERTIFICATE:
case CLOUD_STORAGE_ANONYMOUS_AUTH:
case CLOUD_STORAGE_CACHE_POLICY:
case CLOUD_STORAGE_ALLOCATION_PERCENTAGE:
@@ -147,6 +149,8 @@
return "The cloud storage endpoint";
case CLOUD_STORAGE_ANONYMOUS_AUTH:
return "Indicates whether or not anonymous auth should be
used for the cloud storage";
+ case CLOUD_STORAGE_CERTIFICATE:
+ return "The certificate to use to validate the cloud
storage server";
case CLOUD_STORAGE_CACHE_POLICY:
return "The caching policy (either eager, lazy or
selective). 'eager' caching will download"
+ "all partitions upon booting, whereas 'lazy'
caching will download a file upon"
@@ -254,6 +258,10 @@
return accessor.getBoolean(Option.CLOUD_STORAGE_ANONYMOUS_AUTH);
}
+ public String getStorageCertificate() {
+ return accessor.getString(Option.CLOUD_STORAGE_CERTIFICATE);
+ }
+
public CloudCachePolicy getCloudCachePolicy() {
return
CloudCachePolicy.fromName(accessor.getString(Option.CLOUD_STORAGE_CACHE_POLICY));
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21028?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: phoenix
Gerrit-Change-Id: I0af4a59ff219de1a5213d90719a2995f8ebfe7b3
Gerrit-Change-Number: 21028
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <[email protected]>
