>From Michael Blow <[email protected]>: Michael Blow has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21028?usp=email )
Change subject: [ASTERIXDB-3744][*DB][STO] Support explicit blob storage certificates ...................................................................... [ASTERIXDB-3744][*DB][STO] Support explicit blob storage certificates Generated-by: Claude Opus 4.6 (via GitHub Copilot Agent, partial) Ext-ref: MB-68233 Change-Id: I0af4a59ff219de1a5213d90719a2995f8ebfe7b3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21028 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Tested-by: Michael Blow <[email protected]> Reviewed-by: Ian Maxon <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java A asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3TrustManagerProvider.java M asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java 13 files changed, 297 insertions(+), 18 deletions(-) Approvals: Ian Maxon: Looks good to me, approved Jenkins: Verified Michael Blow: Looks good to me, but someone else must approve; Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java index c0ddfdf..480420a 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java @@ -37,6 +37,9 @@ public static final String S3_REGION_DEFAULT = "us-west-2"; public static final String S3_SERVICE_ENDPOINT_PLACEHOLDER = "%serviceEndpoint%"; public static final String S3_SERVICE_ENDPOINT_DEFAULT = "http://127.0.0.1:8001";; + public static final String S3_API_CERTIFICATES_PLACEHOLDER = "%apiCertificates%"; + public static final String S3_DDL_CERTIFICATES_PLACEHOLDER = "%ddlCertificates%"; + public static final String S3_HTTP_CERTIFICATES_PLACEHOLDER = "%httpCertificates%"; public static final String S3_SERVICE_ENDPOINT_KEY = "s3mockEndpoint"; public static final String S3_TEMPLATE = "(\"accessKeyId\"=\"" + S3_ACCESS_KEY_ID_DEFAULT + "\"),\n" + "(\"secretAccessKey\"=\"" + S3_SECRET_ACCESS_KEY_DEFAULT + "\"),\n" + "(\"region\"=\"" diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 81ad57a..425df54 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -2531,6 +2531,9 @@ str = str.replace(TestConstants.S3_SERVICE_ENDPOINT_PLACEHOLDER, getS3ServiceEndpointDefault()); str = str.replace(TestConstants.S3_ACCESS_KEY_ID_PLACEHOLDER, getS3AccessKeyIdDefault()); str = str.replace(TestConstants.S3_SECRET_ACCESS_KEY_PLACEHOLDER, getS3SecretAccessKeyDefault()); + str = str.replace(TestConstants.S3_API_CERTIFICATES_PLACEHOLDER, getS3ApiCertificatesDefault()); + str = str.replace(TestConstants.S3_DDL_CERTIFICATES_PLACEHOLDER, getS3DdlCertificatesDefault()); + str = str.replace(TestConstants.S3_HTTP_CERTIFICATES_PLACEHOLDER, getS3HttpCertificatesDefault()); return str; } @@ -2543,6 +2546,10 @@ return TestConstants.S3_SECRET_ACCESS_KEY_DEFAULT; } + protected String getS3CertificatesDefault() { + return null; + } + protected String getS3ServiceEndpointDefault() { return TestConstants.S3_SERVICE_ENDPOINT_DEFAULT; } @@ -2551,6 +2558,36 @@ return TestConstants.S3_REGION_DEFAULT; } + protected String getS3ApiCertificatesDefault() { + String certificates = getS3CertificatesDefault(); + return certificates == null || certificates.isBlank() ? "" + : "&certificates=" + URLEncoder.encode(writeAsJsonString(List.of(certificates)), UTF_8); + } + + protected String getS3DdlCertificatesDefault() { + String certificates = getS3CertificatesDefault(); + return certificates == null || certificates.isBlank() ? "" + : ",\n\"certificates\":" + writeAsJsonString(List.of(certificates)); + } + + protected String getS3HttpCertificatesDefault() { + String certificates = getS3CertificatesDefault(); + return certificates == null || certificates.isBlank() ? "" + : ", \\\"certificates\\\":" + escapeForHttpBodyString(writeAsJsonString(List.of(certificates))); + } + + private static String escapeForHttpBodyString(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private static String writeAsJsonString(Object value) { + try { + return OBJECT_WRITER.writeValueAsString(value); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialize S3 test value", e); + } + } + protected String setS3Template(String str) { return str.replace("%template%", TestConstants.S3_TEMPLATE); } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 6d7deed..ca1d858 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -27,6 +27,7 @@ "cloud.storage.bucket" : "", "cloud.storage.cache.policy" : "selective", "cloud.storage.debug.mode.enabled" : false, + "cloud.storage.certificates" : [ ], "cloud.storage.debug.sweep.threshold.size" : 1073741824, "cloud.storage.disable.ssl.verify" : false, "cloud.storage.disk.monitor.interval" : 120, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index 08922ef..51c0437 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -27,6 +27,7 @@ "cloud.storage.bucket" : "", "cloud.storage.cache.policy" : "selective", "cloud.storage.debug.mode.enabled" : false, + "cloud.storage.certificates" : [ ], "cloud.storage.debug.sweep.threshold.size" : 1073741824, "cloud.storage.disable.ssl.verify" : false, "cloud.storage.disk.monitor.interval" : 120, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index ecc29f0..35bb0e9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -27,6 +27,7 @@ "cloud.storage.bucket" : "", "cloud.storage.cache.policy" : "selective", "cloud.storage.debug.mode.enabled" : false, + "cloud.storage.certificates" : [ ], "cloud.storage.debug.sweep.threshold.size" : 1073741824, "cloud.storage.disable.ssl.verify" : false, "cloud.storage.disk.monitor.interval" : 120, diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java index c9046a7..4ff5ed0 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.cloud.clients.aws.s3; +import java.util.Collection; +import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -36,6 +38,7 @@ private final String endpoint; private final String prefix; private final boolean anonymousAuth; + private final Collection<String> certificates; private final long profilerLogInterval; private final int writeBufferSize; private final long tokenAcquireTimeout; @@ -52,22 +55,23 @@ private final boolean roundRobinDnsResolver; public S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth, - long profilerLogInterval, int writeBufferSize, S3ParallelDownloaderClientType parallelDownloaderClientType, - boolean roundRobinDnsResolver) { - this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, 1, 0, 0, 0, false, false, - false, 0, 0, -1, parallelDownloaderClientType, roundRobinDnsResolver); + Collection<String> certificates, long profilerLogInterval, int writeBufferSize, + S3ParallelDownloaderClientType parallelDownloaderClientType, boolean roundRobinDnsResolver) { + this(region, endpoint, prefix, anonymousAuth, certificates, profilerLogInterval, writeBufferSize, 1, 0, 0, 0, + false, false, false, 0, 0, -1, parallelDownloaderClientType, roundRobinDnsResolver); } private S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth, - long profilerLogInterval, int writeBufferSize, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds, - int readMaxRequestsPerSeconds, int requestsMaxHttpConnections, boolean forcePathStyle, - boolean disableSslVerify, boolean storageListEventuallyConsistent, int requestsMaxPendingHttpConnections, - int requestsHttpConnectionAcquireTimeout, int s3ReadTimeoutInSeconds, + Collection<String> certificates, long profilerLogInterval, int writeBufferSize, long tokenAcquireTimeout, + int writeMaxRequestsPerSeconds, int readMaxRequestsPerSeconds, int requestsMaxHttpConnections, + boolean forcePathStyle, boolean disableSslVerify, boolean storageListEventuallyConsistent, + int requestsMaxPendingHttpConnections, int requestsHttpConnectionAcquireTimeout, int s3ReadTimeoutInSeconds, S3ParallelDownloaderClientType parallelDownloaderClientType, boolean roundRobinDnsResolver) { this.region = Objects.requireNonNull(region, "region"); this.endpoint = endpoint; this.prefix = Objects.requireNonNull(prefix, "prefix"); this.anonymousAuth = anonymousAuth; + this.certificates = Objects.requireNonNull(certificates, "certificates"); this.profilerLogInterval = profilerLogInterval; this.writeBufferSize = writeBufferSize; this.tokenAcquireTimeout = tokenAcquireTimeout; @@ -87,11 +91,11 @@ public static S3ClientConfig of(CloudProperties cloudProperties) { return new S3ClientConfig(cloudProperties.getStorageRegion(), cloudProperties.getStorageEndpoint(), cloudProperties.getStoragePrefix(), cloudProperties.isStorageAnonymousAuth(), - cloudProperties.getProfilerLogInterval(), cloudProperties.getWriteBufferSize(), - cloudProperties.getTokenAcquireTimeout(), cloudProperties.getWriteMaxRequestsPerSecond(), - cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getRequestsMaxHttpConnections(), - cloudProperties.isStorageForcePathStyle(), cloudProperties.isStorageDisableSSLVerify(), - cloudProperties.isStorageListEventuallyConsistent(), + cloudProperties.getStorageCertificates(), cloudProperties.getProfilerLogInterval(), + cloudProperties.getWriteBufferSize(), cloudProperties.getTokenAcquireTimeout(), + cloudProperties.getWriteMaxRequestsPerSecond(), cloudProperties.getReadMaxRequestsPerSecond(), + cloudProperties.getRequestsMaxHttpConnections(), cloudProperties.isStorageForcePathStyle(), + cloudProperties.isStorageDisableSSLVerify(), cloudProperties.isStorageListEventuallyConsistent(), cloudProperties.getRequestsMaxPendingHttpConnections(), cloudProperties.getRequestsHttpConnectionAcquireTimeout(), cloudProperties.getS3ReadTimeoutInSeconds(), S3ParallelDownloaderClientType.valueOf(cloudProperties.getS3ParallelDownloaderClientType()), @@ -125,10 +129,11 @@ // Dummy values; String region = ""; String prefix = ""; + Collection<String> certificates = Collections.emptyList(); boolean anonymousAuth = false; - return new S3ClientConfig(region, endPoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, - S3ParallelDownloaderClientType.ASYNC, false); + return new S3ClientConfig(region, endPoint, prefix, anonymousAuth, certificates, profilerLogInterval, + writeBufferSize, S3ParallelDownloaderClientType.ASYNC, false); } public String getRegion() { @@ -188,6 +193,10 @@ return disableSslVerify; } + public Collection<String> getCertificates() { + return certificates; + } + public boolean isForcePathStyle() { return forcePathStyle; } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java index ab27ad4..9c98651 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java @@ -396,10 +396,13 @@ if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) { builder.endpointOverride(URI.create(config.getEndpoint())); } + ApacheHttpClient.Builder apacheBuilder = ApacheHttpClient.builder(); if (config.isDisableSslVerify()) { customHttpConfigBuilder.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true); + } else if (!config.getCertificates().isEmpty()) { + apacheBuilder.tlsTrustManagersProvider(S3TrustManagerProvider.create(config.getCertificates())); } - SdkHttpClient httpClient = ApacheHttpClient.builder().buildWithDefaults(customHttpConfigBuilder.build()); + SdkHttpClient httpClient = apacheBuilder.buildWithDefaults(customHttpConfigBuilder.build()); builder.httpClient(httpClient); awsClients.setConsumingClient(builder.build()); diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java index c1a92a6..a64e07b 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java @@ -49,6 +49,7 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder; +import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload; @@ -225,6 +226,10 @@ NettyNioAsyncHttpClient.Builder nettyBuilder = NettyNioAsyncHttpClient.builder().eventLoopGroup(SHARED_EVENT_LOOP); + if (!config.isDisableSslVerify() && !config.getCertificates().isEmpty()) { + nettyBuilder.tlsTrustManagersProvider(S3TrustManagerProvider.create(config.getCertificates())); + } + SdkAsyncHttpClient nettyClient = nettyBuilder.buildWithDefaults(httpOptions.build()); if (config.useRoundRobinDnsResolver()) { try { @@ -241,6 +246,11 @@ } private static S3AsyncClient createS3CrtAsyncClient(S3ClientConfig config) { + if (!config.getCertificates().isEmpty()) { + LOGGER.warn("Custom CA certificate is not supported with the CRT S3 client. " + + "The certificate will be ignored for parallel downloads. " + + "Consider using the 'async' parallel downloader client type instead."); + } S3CrtAsyncClientBuilder builder = S3AsyncClient.crtBuilder(); builder.credentialsProvider(config.createCredentialsProvider()); builder.region(Region.of(config.getRegion())); @@ -248,6 +258,9 @@ if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) { builder.endpointOverride(URI.create(config.getEndpoint())); } + if (config.isDisableSslVerify()) { + builder.httpConfiguration(S3CrtHttpConfiguration.builder().trustAllCertificatesEnabled(true).build()); + } return builder.build(); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3TrustManagerProvider.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3TrustManagerProvider.java new file mode 100644 index 0000000..7100e46 --- /dev/null +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3TrustManagerProvider.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.cloud.clients.aws.s3; + +import static org.apache.hyracks.util.annotations.AiProvenance.Agent.CLAUDE_OPUS_4_6; +import static org.apache.hyracks.util.annotations.AiProvenance.ContributionKind.GENERATED; +import static org.apache.hyracks.util.annotations.AiProvenance.Tool.GITHUB_COPILOT; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.security.KeyStore; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; + +import org.apache.hyracks.util.annotations.AiProvenance; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import software.amazon.awssdk.http.TlsTrustManagersProvider; + +/** + * Provides a {@link TlsTrustManagersProvider} that trusts the default system certificates + * plus any additional PEM-encoded CA certificates supplied via the blobStorageCertificates setting. + */ +@AiProvenance(agent = CLAUDE_OPUS_4_6, tool = GITHUB_COPILOT, contributionKind = GENERATED) +final class S3TrustManagerProvider { + + private static final Logger LOGGER = LogManager.getLogger(); + + private S3TrustManagerProvider() { + throw new AssertionError("do not instantiate"); + } + + /** + * Creates a {@link TlsTrustManagersProvider} that merges the given PEM certificate(s) + * with the JVM default trust store. + * + * @param pemCertificates PEM-encoded CA certificate(s) + * @return a provider that returns trust managers trusting both the system CAs and the custom certificate(s) + */ + static TlsTrustManagersProvider create(Collection<String> pemCertificates) { + String pemCertificateData = String.join("\n", pemCertificates); + return () -> buildTrustManagers(pemCertificateData); + } + + private static TrustManager[] buildTrustManagers(String pemCertificate) { + try { + // Parse the PEM certificate(s) + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + Collection<? extends java.security.cert.Certificate> certs = + cf.generateCertificates(new ByteArrayInputStream(pemCertificate.getBytes(StandardCharsets.UTF_8))); + if (certs.isEmpty()) { + LOGGER.warn("No certificates found in the provided PEM data; falling back to system defaults"); + return getDefaultTrustManagers(); + } + + // Load the default trust store + TrustManagerFactory defaultTmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + defaultTmf.init((KeyStore) null); + + // Build a new key store with the default certs + custom certs + KeyStore customKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + customKeyStore.load(null, null); + + // Import default trusted certs + TrustManager[] defaultTrustManagers = defaultTmf.getTrustManagers(); + for (TrustManager tm : defaultTrustManagers) { + if (tm instanceof javax.net.ssl.X509TrustManager) { + for (X509Certificate cert : ((javax.net.ssl.X509TrustManager) tm).getAcceptedIssuers()) { + String alias = cert.getSubjectX500Principal().getName(); + customKeyStore.setCertificateEntry(alias, cert); + } + } + } + + // Import custom CA certs + List<? extends java.security.cert.Certificate> certList = new ArrayList<>(certs); + for (int i = 0; i < certList.size(); i++) { + customKeyStore.setCertificateEntry("custom-ca-" + i, certList.get(i)); + } + + TrustManagerFactory customTmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + customTmf.init(customKeyStore); + LOGGER.info("Custom CA certificate(s) loaded successfully ({} certificate(s))", certs.size()); + return customTmf.getTrustManagers(); + } catch (Exception e) { + LOGGER.warn("Failed to load custom CA certificate; falling back to system defaults", e); + return getDefaultTrustManagers(); + } + } + + private static TrustManager[] getDefaultTrustManagers() { + try { + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init((KeyStore) null); + return tmf.getTrustManagers(); + } catch (Exception e) { + throw new IllegalStateException("Unable to initialize default TrustManagerFactory", e); + } + } +} diff --git a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java index 382bb4f..005fade 100644 --- a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java +++ b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java @@ -19,6 +19,7 @@ package org.apache.asterix.cloud.s3; import java.net.URI; +import java.util.Collections; import org.apache.asterix.cloud.AbstractLSMTest; import org.apache.asterix.cloud.clients.ICloudGuardian; @@ -67,8 +68,9 @@ client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build()); LOGGER.info("Client created successfully"); int writeBufferSize = StorageUtil.getIntSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE); - S3ClientConfig config = new S3ClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "", true, 0, - writeBufferSize, S3ClientConfig.S3ParallelDownloaderClientType.ASYNC, false); + S3ClientConfig config = + new S3ClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "", true, Collections.emptyList(), 0, + writeBufferSize, S3ClientConfig.S3ParallelDownloaderClientType.ASYNC, false); CLOUD_CLIENT = new S3CloudClient(config, ICloudGuardian.NoOpCloudGuardian.INSTANCE); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java index abe6560..9a7b4ed 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java @@ -25,9 +25,13 @@ import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.STRING; +import static org.apache.hyracks.control.common.config.OptionTypes.STRING_ARRAY; import static org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType; import static org.apache.hyracks.util.StorageUtil.StorageUnit.GIGABYTE; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -54,6 +58,7 @@ CLOUD_STORAGE_PREFIX(STRING, ""), CLOUD_STORAGE_REGION(STRING, ""), CLOUD_STORAGE_ENDPOINT(STRING, ""), + CLOUD_STORAGE_CERTIFICATES(STRING_ARRAY, new String[0]), CLOUD_STORAGE_ANONYMOUS_AUTH(BOOLEAN, false), CLOUD_STORAGE_CACHE_POLICY(STRING, "selective"), // 80% of the total disk space @@ -106,6 +111,7 @@ case CLOUD_STORAGE_PREFIX: case CLOUD_STORAGE_REGION: case CLOUD_STORAGE_ENDPOINT: + case CLOUD_STORAGE_CERTIFICATES: case CLOUD_STORAGE_ANONYMOUS_AUTH: case CLOUD_STORAGE_CACHE_POLICY: case CLOUD_STORAGE_ALLOCATION_PERCENTAGE: @@ -147,6 +153,8 @@ return "The cloud storage endpoint"; case CLOUD_STORAGE_ANONYMOUS_AUTH: return "Indicates whether or not anonymous auth should be used for the cloud storage"; + case CLOUD_STORAGE_CERTIFICATES: + return "The certificates to use to validate the cloud storage server"; case CLOUD_STORAGE_CACHE_POLICY: return "The caching policy (either eager, lazy or selective). 'eager' caching will download" + "all partitions upon booting, whereas 'lazy' caching will download a file upon" @@ -254,6 +262,11 @@ return accessor.getBoolean(Option.CLOUD_STORAGE_ANONYMOUS_AUTH); } + public Collection<String> getStorageCertificates() { + String[] certificates = accessor.getStringArray(Option.CLOUD_STORAGE_CERTIFICATES); + return certificates == null || certificates.length == 0 ? Collections.emptyList() : List.of(certificates); + } + public CloudCachePolicy getCloudCachePolicy() { return CloudCachePolicy.fromName(accessor.getString(Option.CLOUD_STORAGE_CACHE_POLICY)); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index 0eb935e..7b9dca8 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -354,6 +354,7 @@ public static final String CONTAINER_NAME_FIELD_NAME = "container"; public static final String SUBPATH = "subpath"; public static final String PREFIX_DEFAULT_DELIMITER = "/"; + public static final String CERTIFICATES_FIELD_NAME = "certificates"; public static final String DISABLE_SSL_VERIFY_FIELD_NAME = "disableSslVerify"; public static final Pattern COMPUTED_FIELD_PATTERN = Pattern.compile("\\{[^{}:]+:[^{}:]+}"); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java index 8c81c1b..aa06e98 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java @@ -20,6 +20,7 @@ import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE; import static org.apache.asterix.common.exceptions.ErrorCode.LONG_LIVED_CREDENTIALS_NEEDED_TO_ASSUME_ROLE; +import static org.apache.asterix.external.util.ExternalDataConstants.CERTIFICATES_FIELD_NAME; import static org.apache.asterix.external.util.ExternalDataUtils.getDisableSslVerify; import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix; import static org.apache.asterix.external.util.ExternalDataUtils.isDeltaTable; @@ -67,7 +68,14 @@ import static org.apache.asterix.external.util.aws.s3.S3Constants.PATH_STYLE_ADDRESSING_FIELD_NAME; import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -75,6 +83,10 @@ import java.util.function.BiPredicate; import java.util.regex.Matcher; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; + import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; @@ -98,6 +110,7 @@ import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.SdkHttpConfigurationOption; +import software.amazon.awssdk.http.TlsTrustManagersProvider; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -116,6 +129,19 @@ public class S3Utils { private static final Logger LOGGER = LogManager.getLogger(); + private static final class StaticTrustManagersProvider implements TlsTrustManagersProvider { + private final TrustManager[] trustManagers; + + private StaticTrustManagersProvider(TrustManager[] trustManagers) { + this.trustManagers = trustManagers; + } + + @Override + public TrustManager[] trustManagers() { + return trustManagers; + } + } + private S3Utils() { throw new AssertionError("do not instantiate"); } @@ -138,6 +164,7 @@ boolean crossRegion = getCrossRegion(configuration); boolean disableSslVerify = getDisableSslVerify(configuration); + String certificates = configuration.get(CERTIFICATES_FIELD_NAME); S3ClientBuilder builder = S3Client.builder(); builder.region(region); @@ -149,12 +176,57 @@ builder.forcePathStyle(pathStyleAddressing); if (disableSslVerify) { disableSslVerify(builder); + } else if (certificates != null && !certificates.isBlank()) { + builder.httpClient(createHttpClient(certificates)); } awsClients.setConsumingClient(builder.build()); return awsClients; } + static SdkHttpClient createHttpClient(String pemCertificates) throws CompilationException { + TrustManager[] trustManagers = buildTrustManagers(pemCertificates); + TlsTrustManagersProvider trustManagersProvider = new StaticTrustManagersProvider(trustManagers); + return ApacheHttpClient.builder().tlsTrustManagersProvider(trustManagersProvider).build(); + } + + static TrustManager[] buildTrustManagers(String pemCertificates) throws CompilationException { + try { + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + Collection<? extends Certificate> certificates = certificateFactory.generateCertificates( + new ByteArrayInputStream(pemCertificates.getBytes(StandardCharsets.UTF_8))); + if (certificates.isEmpty()) { + throw new IllegalArgumentException("No certificates found in the supplied PEM data"); + } + + TrustManagerFactory defaultTrustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + defaultTrustManagerFactory.init((KeyStore) null); + + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null, null); + + int alias = 0; + for (TrustManager trustManager : defaultTrustManagerFactory.getTrustManagers()) { + if (trustManager instanceof X509TrustManager x509TrustManager) { + for (X509Certificate certificate : x509TrustManager.getAcceptedIssuers()) { + keyStore.setCertificateEntry("default-ca-" + alias++, certificate); + } + } + } + for (Certificate certificate : certificates) { + keyStore.setCertificateEntry("custom-ca-" + alias++, certificate); + } + + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + return trustManagerFactory.getTrustManagers(); + } catch (Exception e) { + throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, e, getMessageOrToString(e)); + } + } + private static void disableSslVerify(S3ClientBuilder builder) { AttributeMap.Builder attributeMapBuilder = AttributeMap.builder(); attributeMapBuilder.put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21028?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: phoenix Gerrit-Change-Id: I0af4a59ff219de1a5213d90719a2995f8ebfe7b3 Gerrit-Change-Number: 21028 Gerrit-PatchSet: 7 Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Ritik Raj <[email protected]>
