This is an automated email from the ASF dual-hosted git repository. adulceanu pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push: new 0a6f99367f OAK-10615 - Azure Service Principal Support in oak-run segment-copy, compact, console (#1280) 0a6f99367f is described below commit 0a6f99367fd20bf215b4154a76e4a6e422acd2a5 Author: Andrei Dulceanu <dulce...@users.noreply.github.com> AuthorDate: Fri Jan 26 16:00:04 2024 +0200 OAK-10615 - Azure Service Principal Support in oak-run segment-copy, compact, console (#1280) * OAK-10615 - Azure Service Principal Support in oak-run First un-tested implementation * Fixed ToolUtilsTest * Added test for warning when connecting with AZURE_SECRET_KEY instead of service principal * Added SegmentCopyAzureServicePrincipalToTarTest * Fixed warning test when connecting with AZURE_SECRET_KEY Bumped azurite version to 3.29.0 Increased oak.segment.azure package version after refactorings * Bumped azurite to 3.29.0 Increased again oak-run size to include apache commons lang3 needed for running azure commands * Updated Jackrabbit Oak Site Documentation --- .../cloud/azure/blobstorage/AzuriteDockerRule.java | 2 +- .../site/markdown/nodestore/segment/overview.md | 2 +- oak-run/pom.xml | 7 +- .../segment/azure/AzureSegmentStoreService.java | 13 +-- .../oak/segment/azure/AzureUtilities.java | 22 +++++ .../jackrabbit/oak/segment/azure/package-info.java | 2 +- .../oak/segment/azure/tool/ToolUtils.java | 37 ++++++- .../SegmentCopyAzureServicePrincipalToTarTest.java | 80 +++++++++++++++ .../segment/azure/tool/SegmentCopyTestBase.java | 13 ++- .../azure/AzureSegmentStoreServiceTest.java | 10 +- .../oak/segment/azure/tool/ToolUtilsTest.java | 108 ++++++++++++++++++--- 11 files changed, 254 insertions(+), 42 deletions(-) diff --git a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java index 3eaf27980a..cb709aca29 100644 --- a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java +++ b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzuriteDockerRule.java @@ -38,7 +38,7 @@ import java.util.concurrent.atomic.AtomicReference; public class AzuriteDockerRule extends ExternalResource { - private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.19.0"); + private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:3.29.0"); public static final String ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; public static final String ACCOUNT_NAME = "devstoreaccount1"; private static final AtomicReference<Exception> STARTUP_EXCEPTION = new AtomicReference<>(); diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md b/oak-doc/src/site/markdown/nodestore/segment/overview.md index cd2133c058..7e07b02ca2 100644 --- a/oak-doc/src/site/markdown/nodestore/segment/overview.md +++ b/oak-doc/src/site/markdown/nodestore/segment/overview.md @@ -688,7 +688,7 @@ Besides the local storage in TAR files (previously known as TarMK), support for **Connection Instructions**: -* **Microsoft Azure** The `cloud-prefix` for MS Azure is `az`, therefore a valid connection argument would be `az:https://myaccount.blob.core.windows.net/container/repository`, where the part after `:` is the Azure URL identifier for the _repository_ directory inside the specified _container_ of the _myaccount_ Azure storage account. The last missing piece is the secret key which will be supplied as an environment variable, i.e. `AZURE_SECRET_KEY`. +* **Microsoft Azure** The `cloud-prefix` for MS Azure is `az`, therefore a valid connection argument would be `az:https://myaccount.blob.core.windows.net/container/repository`, where the part after `:` is the Azure URL identifier for the _repository_ directory inside the specified _container_ of the _myaccount_ Azure storage account. Default authentication to Microsoft Entra ID with service principal credentials supplied via `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` and `AZURE_TENANT_ID` [...] * **Amazon AWS** The `cloud-prefix` for Amazon AWS is `aws`, therefore a valid connection argument would be `aws:bucket;root_directory;journal_table;lock_table` where the part after `:` defines the _root_directory_ inside the specified _bucket_ in S3 and the _journal_table_ and _lock_table_ tables within DynamoDB services. The other portion to connect to AWS is the credentials which will be supplied by placing a credentials file with ~/.aws folder. diff --git a/oak-run/pom.xml b/oak-run/pom.xml index 0c2057422f..4f2e9d5356 100644 --- a/oak-run/pom.xml +++ b/oak-run/pom.xml @@ -34,6 +34,7 @@ <jetty.version>9.4.53.v20231009</jetty.version> <!-- Size History: + + 78.7 MB Apache Commons Lang ยป 3.0 (OAK-10615) + 78 MB Azure Identity client library for Java (OAK-10604) + 60 MB Groovy 2.5 (OAK-10066) + 56 MB MongoDB Java driver 3.12.7 (OAK-9357) @@ -48,7 +49,7 @@ + 41 MB build failing on the release profile (OAK-6250) + 38 MB. Initial value. Current 35MB plus a 10% --> - <max.jar.size>78300000</max.jar.size> + <max.jar.size>78700000</max.jar.size> </properties> <build> @@ -314,6 +315,10 @@ <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> <dependency> <groupId>com.h2database</groupId> <artifactId>h2</artifactId> diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java index 2e02cae99d..1b5640c76d 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java @@ -18,9 +18,6 @@ */ package org.apache.jackrabbit.oak.segment.azure; -import com.azure.core.credential.TokenRequestContext; -import com.azure.identity.ClientSecretCredential; -import com.azure.identity.ClientSecretCredentialBuilder; import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.StorageCredentialsToken; @@ -46,6 +43,7 @@ import java.security.InvalidKeyException; import java.util.Hashtable; import java.util.Objects; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom; import static org.osgi.framework.Constants.SERVICE_PID; @Component( @@ -126,14 +124,7 @@ public class AzureSegmentStoreService { @NotNull private static AzurePersistence createPersistenceFromServicePrincipalCredentials(Configuration configuration) throws IOException { - ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder() - .clientId(configuration.clientId()) - .clientSecret(configuration.clientSecret()) - .tenantId(configuration.tenantId()) - .build(); - - String accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes("https://storage.azure.com/.default")).getToken(); - StorageCredentialsToken storageCredentialsToken = new StorageCredentialsToken(configuration.accountName(), accessToken); + StorageCredentialsToken storageCredentialsToken = storageCredentialAccessTokenFrom(configuration.accountName(), configuration.clientId(), configuration.clientSecret(), configuration.tenantId()); try { CloudStorageAccount cloud = new CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX, configuration.accountName()); diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java index d37ccf8b97..46e410e398 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java @@ -27,10 +27,14 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import com.azure.core.credential.TokenRequestContext; +import com.azure.identity.ClientSecretCredential; +import com.azure.identity.ClientSecretCredentialBuilder; import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.ResultContinuation; import com.microsoft.azure.storage.ResultSegment; import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsToken; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageUri; import com.microsoft.azure.storage.blob.BlobListingDetails; @@ -45,6 +49,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class AzureUtilities { + public static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; + public static final String AZURE_SECRET_KEY = "AZURE_SECRET_KEY"; + public static final String AZURE_TENANT_ID = "AZURE_TENANT_ID"; + public static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID"; + public static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET"; + + private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default"; private static final Logger log = LoggerFactory.getLogger(AzureUtilities.class); @@ -117,6 +128,17 @@ public final class AzureUtilities { return container.getDirectoryReference(dir); } + public static StorageCredentialsToken storageCredentialAccessTokenFrom(String accountName, String clientId, String clientSecret, String tenantId) { + ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder() + .clientId(clientId) + .clientSecret(clientSecret) + .tenantId(tenantId) + .build(); + + String accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)).getToken(); + return new StorageCredentialsToken(accountName, accessToken); + } + private static ResultSegment<ListBlobItem> listBlobsInSegments(CloudBlobDirectory directory, ResultContinuation token) throws IOException { ResultSegment<ListBlobItem> result = null; diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java index bbe447ca4f..46bf646176 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java @@ -15,7 +15,7 @@ * limitations under the License. */ @Internal(since = "1.0.0") -@Version("2.1.0") +@Version("2.2.0") package org.apache.jackrabbit.oak.segment.azure; import org.apache.jackrabbit.oak.commons.annotations.Internal; diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java index 325d648fef..51984f440c 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java @@ -18,7 +18,16 @@ */ package org.apache.jackrabbit.oak.segment.azure.tool; -import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.*; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI; +import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri; import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions; import java.io.File; @@ -29,6 +38,7 @@ import java.text.MessageFormat; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.commons.Buffer; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; @@ -54,12 +64,14 @@ import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Utility class for common stuff pertaining to tooling. */ public class ToolUtils { - + private static final Logger log = LoggerFactory.getLogger(ToolUtils.class); private static final Environment ENVIRONMENT = new Environment(); private ToolUtils() { @@ -170,7 +182,7 @@ public class ToolUtils { if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) { credentials = new StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE)); } else { - credentials = getStorageCredentialsAccountAndKey(accountName, environment); + credentials = getStorageCredentialsFromAccountAndEnv(accountName, environment); } String uri = config.get(KEY_STORAGE_URI); @@ -185,8 +197,23 @@ public class ToolUtils { } @NotNull - private static StorageCredentials getStorageCredentialsAccountAndKey(String accountName, Environment environment) { - String key = environment.getVariable("AZURE_SECRET_KEY"); + private static StorageCredentials getStorageCredentialsFromAccountAndEnv(String accountName, Environment environment) { + String clientId = environment.getVariable(AZURE_CLIENT_ID); + String clientSecret = environment.getVariable(AZURE_CLIENT_SECRET); + String tenantId = environment.getVariable(AZURE_TENANT_ID); + + if (!StringUtils.isAnyBlank(clientId, clientSecret, tenantId)) { + try { + return storageCredentialAccessTokenFrom(accountName, clientId, clientSecret, tenantId); + } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { + throw new IllegalArgumentException( + "Could not connect to the Azure Storage. Please verify if AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables are correctly set!"); + } + } else { + log.warn("AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."); + } + + String key = environment.getVariable(AZURE_SECRET_KEY); try { return new StorageCredentialsAccountAndKey(accountName, key); } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { diff --git a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java new file mode 100644 index 0000000000..892d380151 --- /dev/null +++ b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package oak.apache.jackrabbit.oak.segment.azure.tool; + +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; +import org.junit.Test; + +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; +import static org.junit.Assume.assumeNotNull; + +public class SegmentCopyAzureServicePrincipalToTarTest extends SegmentCopyTestBase { + private static final Environment ENVIRONMENT = new Environment(); + private static final String CONTAINER_NAME = "oak"; + private static final String DIR = "repository"; + private static final String SEGMENT_STORE_PATH_FORMAT = "https://%s.blob.core.windows.net/%s/%s"; + + @Test + @Override + public void testSegmentCopy() throws Exception { + assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_TENANT_ID)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_ID)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET)); + + super.testSegmentCopy(); + } + + @Override + protected SegmentNodeStorePersistence getSrcPersistence() { + String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME); + String path = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, CONTAINER_NAME, DIR); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT); + + return new AzurePersistence(cloudBlobDirectory); + } + + @Override + protected SegmentNodeStorePersistence getDestPersistence() { + return getTarPersistence(); + } + + @Override + protected String getSrcPathOrUri() { + String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME); + + StringBuilder sb = new StringBuilder(); + sb.append("az:"); + sb.append(String.format(SEGMENT_STORE_PATH_FORMAT, accountName, CONTAINER_NAME, DIR)); + + return sb.toString(); + } + + @Override + protected String getDestPathOrUri() { + return getTarPersistencePathOrUri(); + } +} diff --git a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java index 0113734b87..a732dbeb36 100644 --- a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java +++ b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java @@ -18,7 +18,11 @@ */ package oak.apache.jackrabbit.oak.segment.azure.tool; -import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.*; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ; +import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE; import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newFileStore; import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newSegmentNodeStorePersistence; import static org.junit.Assert.assertEquals; @@ -50,7 +54,12 @@ import org.apache.jackrabbit.oak.segment.azure.tool.SegmentCopy; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; import org.apache.jackrabbit.oak.segment.file.FileStore; -import org.apache.jackrabbit.oak.segment.spi.monitor.*; +import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor; +import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor; +import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitor; +import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitorAdapter; import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileReader; import org.apache.jackrabbit.oak.segment.spi.persistence.ManifestFile; diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java index 117609d0b6..4da1f99779 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreServiceTest.java @@ -40,6 +40,11 @@ import org.junit.Rule; import org.junit.Test; import org.osgi.util.converter.Converters; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; + import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD; import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE; import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST; @@ -65,11 +70,6 @@ public class AzureSegmentStoreServiceTest { private static final EnumSet<SharedAccessBlobPermissions> READ_WRITE = EnumSet.of(READ, LIST, CREATE, WRITE, ADD); private static final ImmutableSet<String> BLOBS = ImmutableSet.of("blob1", "blob2"); - private static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; - private static final String AZURE_TENANT_ID = "AZURE_TENANT_ID"; - private static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID"; - private static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET"; - private CloudBlobContainer container; @Before diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java index 11999453d1..6ae1a0abba 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java @@ -18,59 +18,95 @@ */ package org.apache.jackrabbit.oak.segment.azure.tool; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.Appender; +import ch.qos.logback.core.read.ListAppender; import com.microsoft.azure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; + +import java.net.URISyntaxException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.jetbrains.annotations.NotNull; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; +import org.slf4j.LoggerFactory; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNotNull; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mockStatic; public class ToolUtilsTest { - private static final String CONTAINER_URL = "https://myaccount.blob.core.windows.net/oak-test"; - private static final String REPO_DIR = "repository"; - private static final String SEGMENT_STORE_PATH = CONTAINER_URL + '/' + REPO_DIR; + private static final Environment ENVIRONMENT = new Environment(); + + private static final String CONTAINER_URL_FORMAT = "https://%s.blob.core.windows.net/%s"; + private static final String SEGMENT_STORE_PATH_FORMAT = CONTAINER_URL_FORMAT + "/%s"; + + private static final String DEFAULT_ACCOUNT_NAME = "myaccount"; + private static final String DEFAULT_CONTAINER_NAME = "oak"; + private static final String DEFAULT_REPO_DIR = "repository"; + private static final String DEFAULT_CONTAINER_URL = String.format(CONTAINER_URL_FORMAT, DEFAULT_ACCOUNT_NAME, DEFAULT_CONTAINER_NAME); + private static final String DEFAULT_SEGMENT_STORE_PATH = String.format(SEGMENT_STORE_PATH_FORMAT, DEFAULT_ACCOUNT_NAME, DEFAULT_CONTAINER_NAME, DEFAULT_REPO_DIR); + public static final String AZURE_SECRET_KEY_WARNING = "AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."; private final TestEnvironment environment = new TestEnvironment(); @Test public void createCloudBlobDirectoryWithAccessKey() { - environment.setVariable("AZURE_SECRET_KEY", AzuriteDockerRule.ACCOUNT_KEY); + environment.setVariable(AZURE_SECRET_KEY, AzuriteDockerRule.ACCOUNT_KEY); + + final ListAppender<ILoggingEvent> logAppender = subscribeAppender(); StorageCredentialsAccountAndKey credentials = expectCredentials( StorageCredentialsAccountAndKey.class, - () -> ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH, environment) + () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment), + DEFAULT_CONTAINER_URL ); - - assertEquals("myaccount", credentials.getAccountName()); + + assertTrue(checkLogContainsMessage(AZURE_SECRET_KEY_WARNING, logAppender.list.stream().map(ILoggingEvent::getFormattedMessage).collect(Collectors.toList()))); + assertEquals(Level.WARN, logAppender.list.get(0).getLevel()); + + assertEquals(DEFAULT_ACCOUNT_NAME, credentials.getAccountName()); assertEquals(AzuriteDockerRule.ACCOUNT_KEY, credentials.exportBase64EncodedKey()); + unsubscribe(logAppender); } @Test public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() { - environment.setVariable("AZURE_SECRET_KEY", null); + environment.setVariable(AZURE_SECRET_KEY, null); assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH) + ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment) ); } @Test public void createCloudBlobDirectoryFailsWhenAccessKeyIsInvalid() { - environment.setVariable("AZURE_SECRET_KEY", "invalid"); + environment.setVariable(AZURE_SECRET_KEY, "invalid"); assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH) + ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment) ); } @@ -80,28 +116,70 @@ public class ToolUtilsTest { StorageCredentialsSharedAccessSignature credentials = expectCredentials( StorageCredentialsSharedAccessSignature.class, - () -> ToolUtils.createCloudBlobDirectory(SEGMENT_STORE_PATH + '?' + sasToken) + () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken), + DEFAULT_CONTAINER_URL ); assertEquals(sasToken, credentials.getToken()); assertNull("AccountName should be null when SAS credentials are used", credentials.getAccountName()); } - private static <T extends StorageCredentials> T expectCredentials(Class<T> clazz, Runnable body) { + @Test + public void createCloudBlobDirectoryWithServicePrincipal() throws URISyntaxException, StorageException { + assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_TENANT_ID)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_ID)); + assumeNotNull(ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET)); + + String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME); + String containerName = "oak"; + String segmentStorePath = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, containerName, DEFAULT_REPO_DIR); + + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT); + assertNotNull(cloudBlobDirectory); + assertEquals(containerName, cloudBlobDirectory.getContainer().getName()); + } + + private static <T extends StorageCredentials> T expectCredentials(Class<T> clazz, Runnable body, String containerUrl) { ArgumentCaptor<T> credentialsCaptor = ArgumentCaptor.forClass(clazz); try (MockedStatic<AzureUtilities> mockedAzureUtilities = mockStatic(AzureUtilities.class)) { body.run(); mockedAzureUtilities.verify(() -> AzureUtilities.cloudBlobDirectoryFrom( credentialsCaptor.capture(), - eq(CONTAINER_URL), - eq(REPO_DIR) + eq(containerUrl), + eq(DEFAULT_REPO_DIR) ) ); return credentialsCaptor.getValue(); } } + private boolean checkLogContainsMessage(String toCheck, List<String> messages) { + for (String message : messages) { + if (message.equals(toCheck)) { + return true; + } + } + + return false; + } + + private ListAppender<ILoggingEvent> subscribeAppender() { + ListAppender<ILoggingEvent> appender = new ListAppender<ILoggingEvent>(); + appender.setContext((LoggerContext) LoggerFactory.getILoggerFactory()); + appender.setName("asynclogcollector"); + appender.start(); + ((LoggerContext) LoggerFactory.getILoggerFactory()).getLogger( + ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME).addAppender(appender); + return appender; + } + + private void unsubscribe(@NotNull final Appender<ILoggingEvent> appender) { + ((LoggerContext) LoggerFactory.getILoggerFactory()).getLogger( + ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME).detachAppender(appender); + } + static class TestEnvironment extends Environment { private final Map<String, String> envs = new HashMap<>();