This is an automated email from the ASF dual-hosted git repository. ykinash pushed a commit to branch DATALAB-2556 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5337e93ff4d633676fd2617324406948ed5d89fb Author: KinashYurii <[email protected]> AuthorDate: Thu Nov 4 16:36:10 2021 +0200 [DATALAB-2556] -- added logs and small refactoring --- .../gcp/service/impl/BillingServiceImpl.java | 4 +- .../epam/datalab/dto/status/EnvResourceList.java | 4 + .../response/handlers/EdgeCallbackHandler.java | 1 + .../response/handlers/ProjectCallbackHandler.java | 4 +- .../response/handlers/ResourceCallbackHandler.java | 5 +- .../resources/callback/ProjectCallback.java | 35 ++-- .../CheckInfrastructureStatusScheduler.java | 5 +- .../service/impl/BillingServiceImpl.java | 17 +- .../impl/InfrastructureInfoServiceImpl.java | 17 +- .../epam/datalab/backendapi/util/BillingUtils.java | 202 +++++++++++++++------ .../datalab/backendapi/util/RequestBuilder.java | 3 +- 11 files changed, 207 insertions(+), 90 deletions(-) diff --git a/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java b/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java index b88caba..f766784 100644 --- a/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java +++ b/services/billing-gcp/src/main/java/com/epam/datalab/billing/gcp/service/impl/BillingServiceImpl.java @@ -43,7 +43,9 @@ public class BillingServiceImpl implements BillingService { @Override public List<BillingData> getBillingData() { try { - return billingDAO.getBillingData(); + List<BillingData> billingData = billingDAO.getBillingData(); + log.info("TEST LOG BILLING: billingData: {}", billingData); + return billingData; } catch (Exception e) { log.error("Can not update billing due to: {}", e.getMessage(), e); return Collections.emptyList(); diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java index 0db1930..620a071 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/status/EnvResourceList.java @@ -46,6 +46,10 @@ public class EnvResourceList { .add("cluster", clusterList); } + public boolean isEmpty(){ + return hostList.isEmpty() && clusterList.isEmpty(); + } + @Override public String toString() { return toStringHelper(this).toString(); diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java index 8ebec10..52df8ab 100644 --- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java +++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/EdgeCallbackHandler.java @@ -59,6 +59,7 @@ public class EdgeCallbackHandler<E extends EdgeInfo, T extends UploadFileResult< } protected T parseOutResponse(JsonNode resultNode, T baseStatus) { + log.info("TEST LOG!!!:resultNode: {}, base status: {} ", resultNode, baseStatus); if (resultNode != null && (getAction() == DockerAction.CREATE || getAction() == DockerAction.START) && UserInstanceStatus.of(baseStatus.getStatus()) != UserInstanceStatus.FAILED) { diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java index 1629c05..e8a6566 100644 --- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java +++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ProjectCallbackHandler.java @@ -57,10 +57,12 @@ public class ProjectCallbackHandler extends ResourceCallbackHandler<ProjectResul @Override protected ProjectResult parseOutResponse(JsonNode resultNode, ProjectResult baseStatus) { + log.info("TEST LOG!!!: resultNode: {} , baseStatus: {}", resultNode, baseStatus); + baseStatus.setProjectName(projectName); baseStatus.setEndpointName(endpointName); if (resultNode != null && - Arrays.asList(DockerAction.CREATE, DockerAction.RECREATE, DockerAction.START).contains(getAction()) && + Arrays.asList(DockerAction.CREATE, DockerAction.RECREATE).contains(getAction()) && UserInstanceStatus.of(baseStatus.getStatus()) != UserInstanceStatus.FAILED) { try { final EdgeInfo projectEdgeInfo = mapper.readValue(resultNode.toString(), clazz); diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java index 5a20b37..6999968 100644 --- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java +++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourceCallbackHandler.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Data; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +40,8 @@ import java.util.Date; public abstract class ResourceCallbackHandler<T extends StatusBaseDTO<?>> implements FileHandlerCallback { private static final Logger log = LoggerFactory.getLogger(ResourceCallbackHandler.class); - final ObjectMapper mapper = new ObjectMapper().configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true); + final ObjectMapper mapper = new ObjectMapper() + .configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true); private static final String STATUS_FIELD = "status"; protected static final String RESPONSE_NODE = "response"; @@ -127,6 +129,7 @@ public abstract class ResourceCallbackHandler<T extends StatusBaseDTO<?>> implem log.error("Could not {} resource for user: {}, UUID: {}", action, user, uuid); result.setErrorMessage(getTextValue(resultNode.get(ERROR_NODE))); } + log.info("TEST LOG!!!: resultNode: {}, result: {}", resultNode, result); result = parseOutResponse(resultNode, result); selfServicePost(result); return !UserInstanceStatus.FAILED.equals(status); diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java index 07a327f..d5c521a 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java @@ -23,7 +23,6 @@ import com.epam.datalab.backendapi.dao.EndpointDAO; import com.epam.datalab.backendapi.dao.GpuDAO; import com.epam.datalab.backendapi.dao.ProjectDAO; import com.epam.datalab.backendapi.domain.RequestId; -import com.epam.datalab.backendapi.schedulers.CheckInfrastructureStatusScheduler; import com.epam.datalab.backendapi.service.ExploratoryService; import com.epam.datalab.dto.UserInstanceStatus; import com.epam.datalab.dto.base.project.ProjectResult; @@ -49,39 +48,31 @@ public class ProjectCallback { private final ExploratoryService exploratoryService; private final RequestId requestId; private final GpuDAO gpuDAO; - private final CheckInfrastructureStatusScheduler scheduler; @Inject public ProjectCallback(ProjectDAO projectDAO, EndpointDAO endpointDAO, ExploratoryService exploratoryService, RequestId requestId, - GpuDAO gpuDAO, CheckInfrastructureStatusScheduler scheduler) { + GpuDAO gpuDAO) { this.projectDAO = projectDAO; this.exploratoryService = exploratoryService; this.requestId = requestId; this.gpuDAO = gpuDAO; - this.scheduler = scheduler; } @POST public Response updateProjectStatus(ProjectResult projectResult) { - try { - requestId.checkAndRemove(projectResult.getRequestId()); - final String projectName = projectResult.getProjectName(); - final UserInstanceStatus status = UserInstanceStatus.of(projectResult.getStatus()); - if (projectResult.getEdgeInfo() != null) { - saveGpuForProject(projectResult, projectName); - } - if (UserInstanceStatus.RUNNING == status && Objects.nonNull(projectResult.getEdgeInfo())) { - projectDAO.updateEdgeInfo(projectName, projectResult.getEndpointName(), projectResult.getEdgeInfo()); - } else { - updateExploratoriesStatusIfNeeded(status, projectResult.getProjectName(), projectResult.getEndpointName()); - projectDAO.updateEdgeStatus(projectName, projectResult.getEndpointName(), status); - } - } catch (Exception e) { - log.error(e.toString()); - log.error(e.getMessage()); - log.info("Run scheduler"); - scheduler.execute(null); + log.info("TEST LOG!!!: projectResult: {}", projectResult); + requestId.checkAndRemove(projectResult.getRequestId()); + final String projectName = projectResult.getProjectName(); + final UserInstanceStatus status = UserInstanceStatus.of(projectResult.getStatus()); + if (projectResult.getEdgeInfo() != null) { + saveGpuForProject(projectResult, projectName); + } + if (UserInstanceStatus.RUNNING == status && Objects.nonNull(projectResult.getEdgeInfo())) { + projectDAO.updateEdgeInfo(projectName, projectResult.getEndpointName(), projectResult.getEdgeInfo()); + } else { + updateExploratoriesStatusIfNeeded(status, projectResult.getProjectName(), projectResult.getEndpointName()); + projectDAO.updateEdgeStatus(projectName, projectResult.getEndpointName(), status); } return Response.ok().build(); } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java index 91dda92..5d593db 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java @@ -51,7 +51,7 @@ import static com.epam.datalab.dto.UserInstanceStatus.*; public class CheckInfrastructureStatusScheduler implements Job { private static final List<UserInstanceStatus> statusesToCheck = - Arrays.asList(CREATING, RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED); + Arrays.asList(STARTING, CREATING, RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED); private final InfrastructureInfoService infrastructureInfoService; private final SecurityService securityService; @@ -148,7 +148,8 @@ public class CheckInfrastructureStatusScheduler implements Job { } private boolean noEmrCreating(UserComputationalResource c) { - return ! (c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER)); + log.info("TEST LOG!!!: status: {}, template name: {}", c.getStatus(), c.getTemplateName()); + return !(c.getStatus().equals(CREATING.name()) && c.getTemplateName().contains(AWS_EMR_CLUSTER)); } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java index df800e0..e65901a 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java @@ -22,6 +22,7 @@ package com.epam.datalab.backendapi.service.impl; import com.epam.datalab.auth.UserInfo; import com.epam.datalab.backendapi.conf.SelfServiceApplicationConfiguration; import com.epam.datalab.backendapi.dao.BillingDAO; +import com.epam.datalab.backendapi.dao.ExploratoryDAO; import com.epam.datalab.backendapi.dao.ImageExploratoryDAO; import com.epam.datalab.backendapi.dao.ProjectDAO; import com.epam.datalab.backendapi.domain.*; @@ -72,12 +73,13 @@ public class BillingServiceImpl implements BillingService { private final RESTService billingService; private final ImageExploratoryDAO imageExploratoryDao; private final BillingDAO billingDAO; + private final ExploratoryDAO exploratoryDAO; @Inject public BillingServiceImpl(ProjectService projectService, ProjectDAO projectDAO, EndpointService endpointService, ExploratoryService exploratoryService, SelfServiceApplicationConfiguration configuration, @Named(ServiceConsts.BILLING_SERVICE_NAME) RESTService billingService, - ImageExploratoryDAO imageExploratoryDao, BillingDAO billingDAO) { + ImageExploratoryDAO imageExploratoryDao, BillingDAO billingDAO, ExploratoryDAO exploratoryDAO) { this.projectService = projectService; this.projectDAO = projectDAO; this.endpointService = endpointService; @@ -86,6 +88,7 @@ public class BillingServiceImpl implements BillingService { this.billingService = billingService; this.imageExploratoryDao = imageExploratoryDao; this.billingDAO = billingDAO; + this.exploratoryDAO = exploratoryDAO; } @Override @@ -156,6 +159,8 @@ public class BillingServiceImpl implements BillingService { .stream() .collect(Collectors.toMap(e -> e, e -> getBillingData(userInfo, e))); + log.info("TEST LOG!!!: billingDataMap: {}", billingDataMap); + billingDataMap.forEach((endpointDTO, billingData) -> { log.info("Updating billing information for endpoint {}. Billing data {}", endpointDTO.getName(), billingData); if (!billingData.isEmpty()) { @@ -209,10 +214,18 @@ public class BillingServiceImpl implements BillingService { final Stream<BillingReportLine> billableSharedEndpoints = endpoints .stream() .flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), configuration.getServiceBaseName())); + + log.info("TEST LOG!!!: userInstance from service: {}", exploratoryService.findAll(projects)); + log.info("TEST LOG!!!: userInstance from DB: {}", exploratoryDAO.fetchExploratoryFieldsForProject("proj")); + + final Stream<BillingReportLine> billableUserInstances = exploratoryService.findAll(projects) .stream() .filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId())) .flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount())); + + log.info("TEST LOG!!!: billableUserInstances: {}", billableUserInstances.collect(Collectors.toList())); + final Stream<BillingReportLine> customImages = projects .stream() .map(p -> imageExploratoryDao.getImagesForProject(p.getName())) @@ -222,7 +235,7 @@ public class BillingServiceImpl implements BillingService { final Map<String, BillingReportLine> billableResources = Stream.of(ssnBillingDataStream, billableEdges, billableSharedEndpoints, billableUserInstances, customImages) .flatMap(s -> s) .collect(Collectors.toMap(BillingReportLine::getDatalabId, b -> b)); - log.debug("Billable resources are: {}", billableResources); + log.info("Billable resources are: {}", billableResources); return billableResources; } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java index 5ed48f6..4ea885b 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java @@ -146,12 +146,16 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService .build(); EndpointDTO endpointDTO = endpointService.get(endpoint); - log.info("Send request to provisioning service:\n POST:{}, with EnvResources: {}", INFRASTRUCTURE_STATUS, - envResourceList); - String uuid = provisioningService.post(endpointDTO.getUrl() + INFRASTRUCTURE_STATUS, user.getAccessToken(), - requestBuilder.newInfrastructureStatus(user.getName(), endpointDTO.getCloudProvider(), envResourceList), - String.class); - requestId.put(user.getName(), uuid); + if (envResourceList.isEmpty()) { + log.info("EnvResources is empty: {} , didn't send request to provisioning service", envResourceList); + } else { + log.info("Send request to provisioning service:\n POST:{}, with EnvResources: {}", INFRASTRUCTURE_STATUS, + envResourceList); + String uuid = provisioningService.post(endpointDTO.getUrl() + INFRASTRUCTURE_STATUS, user.getAccessToken(), + requestBuilder.newInfrastructureStatus(user.getName(), endpointDTO.getCloudProvider(), envResourceList), + String.class); + requestId.put(user.getName(), uuid); + } } private List<BillingReport> getExploratoryBillingData(List<UserInstanceDTO> exploratories) { @@ -192,6 +196,7 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService shared.put("status", endpointDTO.getStatus().toString()); shared.put("edge_node_ip", edge.getPublicIp()); + log.info("TEST LOG!!!: edge: {}", edge); if (edge instanceof EdgeInfoAws) { EdgeInfoAws edgeInfoAws = (EdgeInfoAws) edge; shared.put("user_own_bicket_name", edgeInfoAws.getUserOwnBucketName()); diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java index 83f6774..1e8b31c 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java @@ -26,32 +26,24 @@ import com.epam.datalab.dto.UserInstanceStatus; import com.epam.datalab.dto.base.DataEngineType; import com.epam.datalab.dto.computational.UserComputationalResource; import jersey.repackaged.com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.time.format.FormatStyle; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Objects; -import java.util.Optional; +import java.util.*; +import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.epam.datalab.dto.billing.BillingResourceType.BUCKET; -import static com.epam.datalab.dto.billing.BillingResourceType.COMPUTATIONAL; -import static com.epam.datalab.dto.billing.BillingResourceType.EDGE; -import static com.epam.datalab.dto.billing.BillingResourceType.ENDPOINT; -import static com.epam.datalab.dto.billing.BillingResourceType.EXPLORATORY; -import static com.epam.datalab.dto.billing.BillingResourceType.IMAGE; -import static com.epam.datalab.dto.billing.BillingResourceType.SSN; -import static com.epam.datalab.dto.billing.BillingResourceType.VOLUME; +import static com.epam.datalab.dto.billing.BillingResourceType.*; +@Slf4j public class BillingUtils { private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "deeplearning"}; private static final String[] BILLING_FILTERED_REPORT_HEADERS = {"DataLab ID", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"}; private static final String[] COMPLETE_REPORT_REPORT_HEADERS = {"DataLab ID", "User", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"}; + private static final String REPORT_FIRST_LINE = "Service base name: %s. Available reporting period from: %s to: %s"; private static final String TOTAL_LINE = "Total: %s %s"; private static final String SSN_FORMAT = "%s-ssn"; @@ -60,10 +52,12 @@ public class BillingUtils { private static final String EDGE_VOLUME_FORMAT = "%s-%s-%s-edge-volume-primary"; private static final String PROJECT_ENDPOINT_BUCKET_FORMAT = "%s-%s-%s-bucket"; private static final String ENDPOINT_SHARED_BUCKET_FORMAT = "%s-%s-shared-bucket"; + private static final String VOLUME_PRIMARY_FORMAT = "%s-volume-primary"; private static final String VOLUME_PRIMARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-primary"; private static final String VOLUME_SECONDARY_FORMAT = "%s-volume-secondary"; private static final String VOLUME_SECONDARY_COMPUTATIONAL_FORMAT = "%s-%s-volume-secondary"; + private static final String IMAGE_STANDARD_FORMAT1 = "%s-%s-%s-%s-notebook-image"; private static final String IMAGE_STANDARD_FORMAT2 = "%s-%s-%s-notebook-image"; private static final String IMAGE_CUSTOM_FORMAT = "%s-%s-%s-%s-%s"; @@ -80,9 +74,27 @@ public class BillingUtils { final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project, endpoint).toLowerCase(); return Stream.concat(Stream.of( - BillingReportLine.builder().resourceName(endpoint).user(SHARED_RESOURCE).project(project).datalabId(userEdgeId).resourceType(EDGE).build(), - BillingReportLine.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).datalabId(edgeVolumeId).resourceType(VOLUME).build(), - BillingReportLine.builder().resourceName("Project endpoint shared bucket").user(SHARED_RESOURCE).project(project).datalabId(endpointBucketId).resourceType(BUCKET).build() + BillingReportLine.builder() + .resourceName(endpoint) + .user(SHARED_RESOURCE) + .project(project) + .datalabId(userEdgeId) + .resourceType(EDGE) + .build(), + BillingReportLine.builder() + .resourceName("EDGE volume") + .user(SHARED_RESOURCE) + .project(project) + .datalabId(edgeVolumeId) + .resourceType(VOLUME) + .build(), + BillingReportLine.builder() + .resourceName("Project endpoint shared bucket") + .user(SHARED_RESOURCE) + .project(project) + .datalabId(endpointBucketId) + .resourceType(BUCKET) + .build() ), standardImageBillingDataStream(sbn, project, endpoint) ); @@ -91,8 +103,20 @@ public class BillingUtils { public static Stream<BillingReportLine> ssnBillingDataStream(String sbn) { final String ssnId = String.format(SSN_FORMAT, sbn); return Stream.of( - BillingReportLine.builder().user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceName("SSN").datalabId(ssnId).resourceType(SSN).build(), - BillingReportLine.builder().user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceName("SSN Volume").datalabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)).resourceType(VOLUME).build() + BillingReportLine + .builder() + .user(SHARED_RESOURCE) + .project(SHARED_RESOURCE) + .resourceName("SSN") + .datalabId(ssnId) + .resourceType(SSN) + .build(), + BillingReportLine.builder() + .user(SHARED_RESOURCE) + .project(SHARED_RESOURCE) + .resourceName("SSN Volume") + .datalabId(String.format(VOLUME_PRIMARY_FORMAT, ssnId)) + .resourceType(VOLUME).build() ); } @@ -100,39 +124,93 @@ public class BillingUtils { final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint).toLowerCase(); final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint).toLowerCase(); return Stream.concat(Stream.of( - BillingReportLine.builder().resourceName("Endpoint shared bucket").user(SHARED_RESOURCE).project(SHARED_RESOURCE).datalabId(projectEndpointBucketId).resourceType(BUCKET).build(), - BillingReportLine.builder().resourceName("Endpoint").user(SHARED_RESOURCE).project(SHARED_RESOURCE).datalabId(endpointId).resourceType(ENDPOINT).build() + BillingReportLine.builder() + .resourceName("Endpoint shared bucket") + .user(SHARED_RESOURCE) + .project(SHARED_RESOURCE) + .datalabId(projectEndpointBucketId) + .resourceType(BUCKET) + .build(), + BillingReportLine.builder() + .resourceName("Endpoint") + .user(SHARED_RESOURCE) + .project(SHARED_RESOURCE) + .datalabId(endpointId) + .resourceType(ENDPOINT).build() ), standardImageBillingDataStream(sbn, endpoint)); } public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount) { + log.info("TEST LOG!!!: exploratoryBillingDataStream"); + log.info("TEST LOG!!!: userInstance: {}", userInstance); + + final Stream<BillingReportLine> computationalStream = userInstance.getResources() .stream() .filter(cr -> cr.getComputationalId() != null) .flatMap(cr -> { final String computationalId = cr.getComputationalId().toLowerCase(); return Stream.concat(Stream.of( - withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(computationalId).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr)) - .exploratoryName(userInstance.getExploratoryName()).build(), - withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId)).resourceType(VOLUME).build(), - withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId)).resourceType(VOLUME).build(), - withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m")) - .resourceType(VOLUME).build(), - withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).datalabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m")) - .resourceType(VOLUME).build() + withUserProjectEndpoint(userInstance) + .resourceName(cr.getComputationalName()) + .datalabId(computationalId) + .resourceType(COMPUTATIONAL) + .shape(getComputationalShape(cr)) + .exploratoryName(userInstance.getExploratoryName()) + .build(), + withUserProjectEndpoint(userInstance) + .resourceName(cr.getComputationalName()) + .datalabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId)) + .resourceType(VOLUME) + .build(), + withUserProjectEndpoint(userInstance) + .resourceName(cr.getComputationalName()) + .datalabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId)) + .resourceType(VOLUME) + .build(), + withUserProjectEndpoint(userInstance) + .resourceName(cr.getComputationalName()) + .datalabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m")) + .resourceType(VOLUME) + .build(), + withUserProjectEndpoint(userInstance) + .resourceName(cr.getComputationalName()) + .datalabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m")) + .resourceType(VOLUME) + .build() ), getSlaveVolumes(userInstance, cr, maxSparkInstanceCount) ); }); + log.info("TEST LOG!!!: computationalStream: {}", computationalStream); + log.info("TEST LOG!!!: computationalStream: {}", computationalStream.collect(Collectors.toList())); + + final String exploratoryName = userInstance.getExploratoryName(); final String exploratoryId = userInstance.getExploratoryId().toLowerCase(); final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId); final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId); final Stream<BillingReportLine> exploratoryStream = Stream.of( - withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(exploratoryId).resourceType(EXPLORATORY).shape(userInstance.getShape()).build(), - withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(primaryVolumeId).resourceType(VOLUME).build(), - withUserProjectEndpoint(userInstance).resourceName(exploratoryName).datalabId(secondaryVolumeId).resourceType(VOLUME).build()); + withUserProjectEndpoint(userInstance) + .resourceName(exploratoryName) + .datalabId(exploratoryId) + .resourceType(EXPLORATORY) + .shape(userInstance.getShape()) + .build(), + withUserProjectEndpoint(userInstance) + .resourceName(exploratoryName) + .datalabId(primaryVolumeId) + .resourceType(VOLUME) + .build(), + withUserProjectEndpoint(userInstance) + .resourceName(exploratoryName) + .datalabId(secondaryVolumeId) + .resourceType(VOLUME) + .build()); + + log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream); + log.info("TEST LOG!!!: exploratoryStream: {}", exploratoryStream.collect(Collectors.toList())); return Stream.concat(computationalStream, exploratoryStream); } @@ -156,10 +234,19 @@ public class BillingUtils { } private static BillingReportLine.BillingReportLineBuilder withUserProjectEndpoint(UserInstanceDTO userInstance) { - return BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject()).endpoint(userInstance.getEndpoint()); + return BillingReportLine.builder() + .user(userInstance.getUser()) + .project(userInstance.getProject()) + .endpoint(userInstance.getEndpoint()); } public static String getComputationalShape(UserComputationalResource resource) { + log.info("TEST LOG!!! getComputationalShape"); + log.info("TEST LOG!!! UserComputationalResource: {}", resource); + log.info("TEST LOG!!! getTemplateName: {}", resource.getTemplateName()); + log.info("TEST LOG!!! getDataengineShape: {}", resource.getDataengineShape()); + log.info("TEST LOG!!! getMasterNodeShape: {}", resource.getMasterNodeShape()); + return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ? String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) : String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getTotalInstanceCount() - 1, resource.getSlaveNodeShape()); @@ -168,37 +255,46 @@ public class BillingUtils { private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String endpoint) { List<BillingReportLine> list = new ArrayList<>(); for (String notebook : AVAILABLE_NOTEBOOKS) { - list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).datalabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase()) - .user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceType(IMAGE).build()); + list.add(BillingReportLine.builder() + .resourceName(IMAGE_NAME) + .datalabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase()) + .user(SHARED_RESOURCE) + .project(SHARED_RESOURCE) + .resourceType(IMAGE) + .build()); } - return list.stream(); } private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) { List<BillingReportLine> list = new ArrayList<>(); for (String notebook : AVAILABLE_NOTEBOOKS) { - list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).datalabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase()) - .project(project).user(SHARED_RESOURCE).resourceType(IMAGE).build()); + list.add(BillingReportLine + .builder() + .resourceName(IMAGE_NAME) + .datalabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase()) + .project(project) + .user(SHARED_RESOURCE) + .resourceType(IMAGE) + .build()); } - return list.stream(); } - /** - * @param sbn Service Base Name - * @param from formatted date, like 2020-04-07 - * @param to formatted date, like 2020-05-07 - * @param locale user's locale - * @return line, like: - * "Service base name: SERVICE_BASE_NAME. Available reporting period from: 2020-04-07 to: 2020-04-07" - */ - public static String getFirstLine(String sbn, LocalDate from, LocalDate to, String locale) { - return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, sbn, - Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY), - Optional.ofNullable(to).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY))), - CSVFormatter.SEPARATOR, '\"'); - } + /** + * @param sbn Service Base Name + * @param from formatted date, like 2020-04-07 + * @param to formatted date, like 2020-05-07 + * @param locale user's locale + * @return line, like: + * "Service base name: SERVICE_BASE_NAME. Available reporting period from: 2020-04-07 to: 2020-04-07" + */ + public static String getFirstLine(String sbn, LocalDate from, LocalDate to, String locale) { + return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, sbn, + Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY), + Optional.ofNullable(to).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY))), + CSVFormatter.SEPARATOR, '\"'); + } /** * headerType there are two types of header according user role diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java index ad50429..7b1ea0a 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java @@ -171,7 +171,7 @@ public class RequestBuilder { } - T t = exploratoryCreate.withExploratoryName(exploratory.getName()) + return exploratoryCreate.withExploratoryName(exploratory.getName()) .withNotebookImage(exploratory.getDockerImage()) .withApplicationName(getApplicationNameFromImage(exploratory.getDockerImage())) .withGitCreds(exploratoryGitCredsDTO.getGitCreds()) @@ -184,7 +184,6 @@ public class RequestBuilder { .withGPUCount(exploratory.getGpuCount()) .withGPUType(exploratory.getGpuType()) .withEnabledGPU(exploratory.getEnabledGPU()); - return t; } @SuppressWarnings("unchecked") --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
