>From Michael Blow <[email protected]>: Michael Blow has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20423?usp=email )
Change subject: [NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify ...................................................................... [NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify Also, update test framework to drop created databases on cleanup Ext-ref: MB-68204 Change-Id: If708d39892cb777d4e7f0eba606417bd82692ab2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20423 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Tested-by: Michael Blow <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java M asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java 9 files changed, 129 insertions(+), 21 deletions(-) Approvals: Michael Blow: Looks good to me, but someone else must approve; Verified Hussain Towaileb: Looks good to me, approved Jenkins: Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java index ce42408..d33617b 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java @@ -21,19 +21,16 @@ import static org.apache.hyracks.util.file.FileUtil.joinPath; import java.io.File; -import java.net.URI; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import io.findify.s3mock.S3Mock; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -76,11 +73,7 @@ s3MockServer.start(); LOGGER.info("S3 mock server started successfully"); - S3ClientBuilder builder = S3Client.builder(); - URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server - builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) - .endpointOverride(endpoint); - S3Client client = builder.build(); + S3Client client = LocalCloudUtilAdobeMock.getS3Client(); client.createBucket(CreateBucketRequest.builder().bucket(CLOUD_STORAGE_BUCKET).build()); LOGGER.info("Created bucket {} for cloud storage", CLOUD_STORAGE_BUCKET); @@ -101,6 +94,22 @@ return s3MockServer; } + public static void recreateBucket(String bucketName, S3Client client) { + String verb = "Created"; + try { + client.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()); + verb = "Recreated"; + } catch (Exception e) { + // ignore any failure + } + try { + client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); + LOGGER.info("{} bucket {}", verb, bucketName); + } finally { + client.close(); + } + } + public static void stopS3MockServer() { shutdownSilently(); // since they are running on same port, we need to shut down other mock server as well diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java index ba7f856..8790632 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java @@ -106,4 +106,13 @@ s3Mock = null; } } + + public static S3Client getS3Client() { + S3ClientBuilder builder = S3Client.builder(); + URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); + builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create()) + .endpointOverride(endpoint); + S3Client client = builder.build(); + return client; + } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index ee85995..58c92fb 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -74,6 +74,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -84,6 +85,7 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Queue; +import java.util.SequencedSet; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -2728,17 +2730,38 @@ return stripLineComments(stripJavaComments(statement)); } - public void cleanup(String testCase, List<String> badtestcases) throws Exception { + public final void cleanup(String testCase, SequencedSet<String> badtestcases) throws Exception { + cleanup(testCase, badtestcases, false); + } + + public void cleanup(String testCase, SequencedSet<String> badtestcases, boolean skipClusterCleanup) + throws Exception { + if (skipClusterCleanup) { + return; + } try { List<Pair<String, DataverseName>> toBeDropped = new ArrayList<>(); listUserDefinedDataverses(toBeDropped); if (!toBeDropped.isEmpty()) { badtestcases.add(testCase); - LOGGER.info("Last test left some garbage. Dropping dataverses: " + StringUtils.join(toBeDropped, ',')); + LOGGER.info("Last test leaked scopes. Dropping scopes: {}", + toBeDropped.stream() + .map(pair -> (pair.getFirst() == null ? "" : (pair.getFirst() + SqlppStatementUtil.DOT)) + + pair.getSecond().getCanonicalForm()) + .collect(Collectors.joining(", ", "[", "]"))); + // TODO(mblow): consider that dataverses can have references from views in other dataverses for (Pair<String, DataverseName> dv : toBeDropped) { dropDataverse(dv); } } + List<String> databases = listUserDefinedDatabases(); + if (!databases.isEmpty()) { + LOGGER.info("Last test leaked databases. Dropping databases: " + databases); + badtestcases.add(testCase); + for (String db : databases) { + dropDatabase(db); + } + } } catch (Throwable th) { th.printStackTrace(); throw th; @@ -2768,6 +2791,25 @@ } } + protected List<String> listUserDefinedDatabases() throws Exception { + String query = "select value(DatabaseName) from Metadata.`Database` where Creator.Name != \"@sys\""; + URI uri = getEndpoint(Servlets.QUERY_SERVICE); + try { + InputStream resultStream = executeQueryService(query, OutputFormat.CLEAN_JSON, uri, + constructQueryParameters(query, OutputFormat.CLEAN_JSON, new ArrayList<Parameter>()), false, + StandardCharsets.UTF_8, code -> code == HttpStatus.SC_OK, false); + JsonNode result = extractResult(IOUtils.toString(resultStream, UTF_8)); + List<String> databases = new ArrayList<>(); + for (int i = 0; i < result.size(); i++) { + databases.add(result.get(i).asText()); + } + return databases; + } catch (Exception e) { + // assuming databases are not enabled in this cluster + return Collections.emptyList(); + } + } + protected void dropDataverse(Pair<String, DataverseName> dv) throws Exception { StringBuilder dropStatement = new StringBuilder(); dropStatement.append("drop dataverse "); @@ -2783,8 +2825,18 @@ ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON); } - protected void listDatasets(DataverseName dataverseName, List<Pair<String, DatasetConfig.DatasetType>> outDatasets) - throws Exception { + protected void dropDatabase(String db) throws Exception { + StringBuilder dropStatement = new StringBuilder(); + dropStatement.append("DROP DATABASE "); + SqlppStatementUtil.enclose(dropStatement, db); + dropStatement.append(";"); + InputStream resultStream = executeQueryService(dropStatement.toString(), getEndpoint(Servlets.QUERY_SERVICE), + OutputFormat.CLEAN_JSON, UTF_8); + ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON); + } + + protected void listDatasets(DataverseName dataverseName, + Collection<Pair<String, DatasetConfig.DatasetType>> outDatasets) throws Exception { String query = "select d.DatasetName, d.DatasetType from Metadata.`Dataset` d where d.DataverseName = '" + dataverseName.getCanonicalForm() + "'"; InputStream resultStream = executeQueryService(query, getEndpoint(Servlets.QUERY_SERVICE), diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java index d704d8e..b7e8565 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java @@ -30,7 +30,9 @@ import java.lang.management.RuntimeMXBean; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; +import java.util.SequencedSet; import org.apache.asterix.app.external.ExternalUDFLibrarian; import org.apache.asterix.app.external.IExternalUDFLibrarian; @@ -54,7 +56,7 @@ private static final boolean cleanupOnStart = true; private static final boolean cleanupOnStop = true; - private static final List<String> badTestCases = new ArrayList<>(); + private static final SequencedSet<String> badTestCases = new LinkedHashSet<>(); private static TestExecutor testExecutor; private static ExternalUDFLibrarian librarian; diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java index f4536a1..889a6a4 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java @@ -29,9 +29,10 @@ import com.azure.identity.DefaultAzureCredentialBuilder; public class AzBlobStorageClientConfig { - private final int writeBufferSize; // Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id static final int DELETE_BATCH_SIZE = 256; + + private final int writeBufferSize; private final String region; private final String endpoint; private final String prefix; @@ -42,15 +43,16 @@ private final long tokenAcquireTimeout; private final int writeMaxRequestsPerSeconds; private final int readMaxRequestsPerSeconds; + private final boolean storageDisableSSLVerify; public AzBlobStorageClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth, long profilerLogInterval, String bucket, int writeBufferSize) { - this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, bucket, 1, 0, 0, writeBufferSize); + this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, bucket, 1, 0, 0, writeBufferSize, false); } public AzBlobStorageClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth, long profilerLogInterval, String bucket, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds, - int readMaxRequestsPerSeconds, int writeBufferSize) { + int readMaxRequestsPerSeconds, int writeBufferSize, boolean storageDisableSSLVerify) { this.region = Objects.requireNonNull(region, "region"); this.endpoint = endpoint; this.prefix = Objects.requireNonNull(prefix, "prefix"); @@ -61,6 +63,7 @@ this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds; this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds; this.writeBufferSize = writeBufferSize; + this.storageDisableSSLVerify = storageDisableSSLVerify; } public static AzBlobStorageClientConfig of(CloudProperties cloudProperties) { @@ -68,7 +71,8 @@ cloudProperties.getStoragePrefix(), cloudProperties.isStorageAnonymousAuth(), cloudProperties.getProfilerLogInterval(), cloudProperties.getStorageBucket(), cloudProperties.getTokenAcquireTimeout(), cloudProperties.getWriteMaxRequestsPerSecond(), - cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getWriteBufferSize()); + cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getWriteBufferSize(), + cloudProperties.isStorageDisableSSLVerify()); } public static AzBlobStorageClientConfig of(Map<String, String> configuration, int writeBufferSize) { @@ -111,6 +115,10 @@ return anonymousAuth; } + public boolean isStorageDisableSSLVerify() { + return storageDisableSSLVerify; + } + public DefaultAzureCredential createCredentialsProvider() { return new DefaultAzureCredentialBuilder().build(); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java index d7b680e..bdb551b 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java @@ -58,6 +58,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.azure.core.http.netty.NettyAsyncHttpClientBuilder; import com.azure.core.http.rest.PagedIterable; import com.azure.core.http.rest.Response; import com.azure.core.util.BinaryData; @@ -79,6 +80,11 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import reactor.netty.http.client.HttpClient; + public class AzBlobStorageCloudClient implements ICloudClient { private static final String BUCKET_ROOT_PATH = ""; public static final String AZURITE_ENDPOINT = "http://127.0.0.1:15055/devstoreaccount1/";; @@ -402,6 +408,24 @@ blobServiceClientBuilder.endpoint(getEndpoint(config)); blobServiceClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS); configCredentialsToAzClient(blobServiceClientBuilder, config); + + // Disable SSL verification if the config property is set + if (config.isStorageDisableSSLVerify()) { + try { + // Create SSL context that trusts all certificates + SslContext sslContext = + SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build(); + + // Create a base Reactor Netty HttpClient with SSL verification disabled + HttpClient baseHttpClient = HttpClient.create().secure(sslSpec -> sslSpec.sslContext(sslContext)); + + // Configure the Azure HTTP client with the base client + blobServiceClientBuilder.httpClient(new NettyAsyncHttpClientBuilder(baseHttpClient).build()); + } catch (Exception e) { + throw new RuntimeException("Failed to disable SSL verification", e); + } + } + return blobServiceClientBuilder.buildClient(); } diff --git a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java index 1f49fd9..0d5a9ba 100644 --- a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java +++ b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java @@ -63,7 +63,7 @@ LOGGER.info("Az Blob Client created successfully"); int writeBufferSize = StorageUtil.getIntSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE); AzBlobStorageClientConfig config = new AzBlobStorageClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "", - true, 0, PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize); + true, 0, PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize, false); CLOUD_CLIENT = new AzBlobStorageCloudClient(config, ICloudGuardian.NoOpCloudGuardian.INSTANCE); } diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java index 35bf7e4..1360917 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java @@ -21,8 +21,10 @@ import java.io.File; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; +import java.util.SequencedSet; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.IdentitiyResolverFactory; @@ -61,7 +63,7 @@ private static final String EXTERNAL_LIBRARY_TEST_GROUP = "lib"; - private static final List<String> badTestCases = new ArrayList<>(); + private static final SequencedSet<String> badTestCases = new LinkedHashSet<>(); private static String reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath(); diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java index ab4e19f..507a156 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java @@ -24,8 +24,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Random; +import java.util.SequencedSet; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -135,7 +137,7 @@ private final TestCaseContext tcCtx; private static final TestExecutor testExecutor = new TestExecutor(); - private static final List<String> badTestCases = new ArrayList<>(); + private static final SequencedSet<String> badTestCases = new LinkedHashSet<>(); private static HyracksVirtualCluster cluster; private final KillCommand killType; private static boolean clusterActive = false; -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20423?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: phoenix Gerrit-Change-Id: If708d39892cb777d4e7f0eba606417bd82692ab2 Gerrit-Change-Number: 20423 Gerrit-PatchSet: 4 Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]>
