This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch bucket-browser-azure in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/bucket-browser-azure by this push: new 6c5c086 [DLAB-1552] Updated auth for azure bucket browser 6c5c086 is described below commit 6c5c086a32523d96fcd0eb754b281614043a9381 Author: Oleh Fuks <olegfuk...@gmail.com> AuthorDate: Thu Jul 16 13:53:13 2020 +0300 [DLAB-1552] Updated auth for azure bucket browser --- .../billing/azure/CalculateBillingServiceImpl.java | 2 +- .../com/epam/dlab/model/azure}/AzureAuthFile.java | 2 +- services/provisioning-service/pom.xml | 5 ++ .../service/impl/azure/BucketServiceAzureImpl.java | 86 +++++++++++++++++++--- 4 files changed, 81 insertions(+), 14 deletions(-) diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java index 3b3d60b..7b29677 100644 --- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java +++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java @@ -20,12 +20,12 @@ package com.epam.dlab.billing.azure; import com.epam.dlab.MongoKeyWords; -import com.epam.dlab.billing.azure.config.AzureAuthFile; import com.epam.dlab.billing.azure.config.BillingConfigurationAzure; import com.epam.dlab.billing.azure.model.AzureDailyResourceInvoice; import com.epam.dlab.billing.azure.model.BillingPeriod; import com.epam.dlab.dto.billing.BillingData; import com.epam.dlab.exceptions.DlabException; +import com.epam.dlab.model.azure.AzureAuthFile; import com.epam.dlab.util.mongo.modules.IsoDateModule; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/AzureAuthFile.java b/services/dlab-model/src/main/java/com/epam/dlab/model/azure/AzureAuthFile.java similarity index 96% rename from services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/AzureAuthFile.java rename to services/dlab-model/src/main/java/com/epam/dlab/model/azure/AzureAuthFile.java index 1d1993c..21b1bf2 100644 --- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/config/AzureAuthFile.java +++ b/services/dlab-model/src/main/java/com/epam/dlab/model/azure/AzureAuthFile.java @@ -17,7 +17,7 @@ * under the License. */ -package com.epam.dlab.billing.azure.config; +package com.epam.dlab.model.azure; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/services/provisioning-service/pom.xml b/services/provisioning-service/pom.xml index 043f596..f164361 100644 --- a/services/provisioning-service/pom.xml +++ b/services/provisioning-service/pom.xml @@ -107,6 +107,11 @@ <version>12.6.0</version> </dependency> <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-identity</artifactId> + <version>1.0.6</version> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-core</artifactId> <version>2.11.0</version> diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java index f35dbc6..db0344c 100644 --- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java +++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java @@ -19,34 +19,52 @@ package com.epam.dlab.backendapi.service.impl.azure; +import com.azure.identity.ClientSecretCredentialBuilder; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobItem; +import com.epam.dlab.backendapi.ProvisioningServiceApplicationConfiguration; import com.epam.dlab.backendapi.service.BucketService; import com.epam.dlab.dto.bucket.BucketDTO; import com.epam.dlab.exceptions.DlabException; +import com.epam.dlab.model.azure.AzureAuthFile; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; +import java.io.FileNotFoundException; +import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.stream.Collectors; @Slf4j public class BucketServiceAzureImpl implements BucketService { + private final AzureAuthFile azureAuthFile; + + @Inject + public BucketServiceAzureImpl(ProvisioningServiceApplicationConfiguration configuration) throws Exception { + azureAuthFile = getAzureAuthFile(configuration); + } + @Override public List<BucketDTO> getObjects(String bucket) { try { - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); - BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + AzureStorageAccount account = getAzureStorageAccount(bucket); + BlobServiceClient blobServiceClient = getBlobServiceClient(account.getStorageAccount()); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(account.getContainer()); return blobContainerClient.listBlobs() .stream() - .map(blob -> toBucketDTO(bucket, blob)) + .map(blob -> toBucketDTO(account.getContainer(), blob)) .collect(Collectors.toList()); } catch (Exception e) { log.error("Cannot retrieve objects from bucket {}. Reason: {}", bucket, e.getMessage()); @@ -58,8 +76,9 @@ public class BucketServiceAzureImpl implements BucketService { public void uploadObject(String bucket, String object, InputStream stream, long fileSize) { log.info("Uploading file {} to bucket {}", object, bucket); try { - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); - BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + AzureStorageAccount account = getAzureStorageAccount(bucket); + BlobServiceClient blobServiceClient = getBlobServiceClient(account.getStorageAccount()); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(account.getContainer()); BlobClient blobClient = blobContainerClient.getBlobClient(object); blobClient.upload(stream, fileSize); } catch (Exception e) { @@ -73,8 +92,9 @@ public class BucketServiceAzureImpl implements BucketService { public void downloadObject(String bucket, String object, HttpServletResponse resp) { log.info("Downloading file {} from bucket {}", object, bucket); try (ServletOutputStream outputStream = resp.getOutputStream()) { - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); - BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + AzureStorageAccount account = getAzureStorageAccount(bucket); + BlobServiceClient blobServiceClient = getBlobServiceClient(account.getStorageAccount()); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(account.getContainer()); BlobClient blobClient = blobContainerClient.getBlobClient(object); blobClient.download(outputStream); } catch (Exception e) { @@ -87,8 +107,9 @@ public class BucketServiceAzureImpl implements BucketService { @Override public void deleteObjects(String bucket, List<String> objects) { try { - BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); - BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + AzureStorageAccount account = getAzureStorageAccount(bucket); + BlobServiceClient blobServiceClient = getBlobServiceClient(account.getStorageAccount()); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(account.getContainer()); objects.forEach(object -> blobContainerClient.getBlobClient(object).delete()); } catch (Exception e) { log.error("Cannot delete objects {} from bucket {}. Reason: {}", objects, bucket, e.getMessage()); @@ -97,13 +118,54 @@ public class BucketServiceAzureImpl implements BucketService { } private BucketDTO toBucketDTO(String bucket, BlobItem blob) { - final String size = FileUtils.byteCountToDisplaySize(blob.getProperties().getContentLength()); String lastModifiedDate = blob.getProperties().getLastModified().format(DateTimeFormatter.ofPattern(DATE_FORMAT)); return BucketDTO.builder() .bucket(bucket) .object(blob.getName()) .lastModifiedDate(lastModifiedDate) - .size(size) + .size(String.valueOf(blob.getProperties().getContentLength())) .build(); } + + private AzureAuthFile getAzureAuthFile(ProvisioningServiceApplicationConfiguration configuration) throws Exception { + final String authFile = configuration.getCloudConfiguration().getAzureAuthFile(); + Path path = Paths.get(authFile); + if (path.toFile().exists()) { + try { + return new ObjectMapper().readValue(path.toFile(), AzureAuthFile.class); + } catch (IOException e) { + log.error("Cannot parse azure auth file {}", authFile, e); + throw new IOException("Cannot parse azure auth file " + authFile); + } catch (Exception e) { + log.error("Something went wrong while parsing azure auth file {}", authFile, e); + throw new Exception("Something went wrong while parsing azure auth file " + authFile); + } + } else { + throw new FileNotFoundException("Cannot find azure auth file for path" + authFile); + } + } + + private BlobServiceClient getBlobServiceClient(String storageAccount) { + final String endpoint = String.format("https://%s.blob.core.windows.net", storageAccount); + return new BlobServiceClientBuilder() + .endpoint(endpoint) + .credential(new ClientSecretCredentialBuilder() + .clientId(azureAuthFile.getClientId()) + .clientSecret(azureAuthFile.getClientSecret()) + .tenantId(azureAuthFile.getTenantId()) + .build()) + .buildClient(); + } + + private AzureStorageAccount getAzureStorageAccount(String bucket) { + String[] a = bucket.split("\\."); + return new AzureStorageAccount(a[0], a[1]); + } + + @Getter + @AllArgsConstructor + private static class AzureStorageAccount { + private final String storageAccount; + private final String container; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org