This is an automated email from the ASF dual-hosted git repository. ykinash pushed a commit to branch DATALAB-2556 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 66455cb133ca49c782a4a6af513320b184727552 Author: KinashYurii <[email protected]> AuthorDate: Wed Nov 10 23:06:14 2021 +0200 [DATALAB] -- fixed jackson dependency, added test log, fixed billing null value, --- services/datalab-model/pom.xml | 6 +++++ .../epam/datalab/dto/UserEnvironmentResources.java | 5 +++- .../epam/datalab/dto/project/ProjectCreateDTO.java | 2 ++ .../handlers/ResourcesStatusCallbackHandler.java | 7 +++++ .../resources/aws/InfrastructureResourceAws.java | 1 + .../CheckInfrastructureStatusScheduler.java | 9 +++++-- .../schedulers/billing/BillingScheduler.java | 2 +- .../service/impl/BillingServiceImpl.java | 30 +++++++++++++++++++++- .../epam/datalab/backendapi/util/BillingUtils.java | 6 ++++- .../datalab/backendapi/util/RequestBuilder.java | 4 ++- 10 files changed, 65 insertions(+), 7 deletions(-) diff --git a/services/datalab-model/pom.xml b/services/datalab-model/pom.xml index 2f7878a..a27ff0b 100644 --- a/services/datalab-model/pom.xml +++ b/services/datalab-model/pom.xml @@ -74,6 +74,12 @@ <scope>provided</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.11.2</version> + </dependency> + </dependencies> </project> \ No newline at end of file diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java index 3c74d45..0fc9c27 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/UserEnvironmentResources.java @@ -20,11 +20,13 @@ package com.epam.datalab.dto; import com.epam.datalab.dto.status.EnvResourceList; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects.ToStringHelper; import lombok.EqualsAndHashCode; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) +@JsonIgnoreProperties public class UserEnvironmentResources extends ResourceSysBaseDTO<UserEnvironmentResources> { @JsonProperty("edge_list_resources") private EnvResourceList resourceList; @@ -46,6 +48,7 @@ public class UserEnvironmentResources extends ResourceSysBaseDTO<UserEnvironment /** * Set the list of resources (hosts, clusters, storages). */ + @JsonProperty("edge_list_resources") public UserEnvironmentResources withResourceList(EnvResourceList resourceList) { setResourceList(resourceList); return this; diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java index 09cbd46..c780d4d 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/project/ProjectCreateDTO.java @@ -23,7 +23,9 @@ import com.epam.datalab.dto.ResourceBaseDTO; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data @Builder public class ProjectCreateDTO extends ResourceBaseDTO<ProjectCreateDTO> { diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java index 0acb3a4..778b774 100644 --- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java +++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/response/handlers/ResourcesStatusCallbackHandler.java @@ -52,6 +52,13 @@ public class ResourcesStatusCallbackHandler extends ResourceCallbackHandler<EnvS @JsonCreator public ResourcesStatusCallbackHandler(@JacksonInject RESTService selfService, @JsonProperty("action") DockerAction action, + @JsonProperty("uuid") String uuid, @JsonProperty("user") String user) { + super(selfService, user, uuid, action); + this.datalabHostResources = getEnvResources(null); + } + + @JsonCreator + public ResourcesStatusCallbackHandler(@JacksonInject RESTService selfService, @JsonProperty("action") DockerAction action, @JsonProperty("uuid") String uuid, @JsonProperty("user") String user, EnvResourceList resourceList) { super(selfService, user, uuid, action); diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java index c88002c..2df471e 100644 --- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java +++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/resources/aws/InfrastructureResourceAws.java @@ -44,6 +44,7 @@ public class InfrastructureResourceAws extends InfrastructureService { @Path("/status") @POST public String status(@Auth UserInfo ui, UserEnvironmentResources dto) { + log.info("TEST LOG!!! UserEnvironmentResources:{}", dto); return action(ui.getName(), dto, dto.getCloudSettings().getIamUser(), DockerAction.STATUS); } } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java index 5d593db..018176a 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java @@ -110,7 +110,8 @@ public class CheckInfrastructureStatusScheduler implements Job { .stream() .filter(c -> DataEngineType.SPARK_STANDALONE == DataEngineType.fromDockerImageName(c.getImageName())) .filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus()))) -// .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null) + .filter(this::noEmrCreating) + .filter(c -> c.getInstanceId() != null) .map(r -> new EnvResource() .withId(r.getInstanceId()) .withName(r.getComputationalName()) @@ -135,7 +136,7 @@ public class CheckInfrastructureStatusScheduler implements Job { return userInstanceDTO.getResources().stream() .filter(c -> DataEngineType.CLOUD_SERVICE == DataEngineType.fromDockerImageName(c.getImageName())) .filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus()))) -// .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null) + .filter(c -> c.getInstanceId() != null) .filter(this::noEmrCreating) .map(r -> new EnvResource() .withId(r.getInstanceId()) @@ -167,6 +168,10 @@ public class CheckInfrastructureStatusScheduler implements Job { private List<EnvResource> getEdgeInstances(String endpoint, Map.Entry<String, List<ProjectEndpointDTO>> entry) { return entry.getValue() .stream() + .peek(e -> log.info("TEST LOG!!!: status: {}", e.getStatus())) + .peek(e -> log.info("TEST LOG!!!: e: {}", e)) + .peek(e -> log.info("TEST LOG!!!: statusesToCheck: {}", statusesToCheck)) + .filter(e -> statusesToCheck.contains(e.getStatus())) .filter(e -> e.getName().equals(endpoint)) .filter(e -> Objects.nonNull(e.getEdgeInfo())) diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java index c4dc177..fbc369c 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/billing/BillingScheduler.java @@ -46,7 +46,7 @@ public class BillingScheduler implements Job { try { billingService.updateRemoteBillingData(securityService.getServiceAccountInfo("admin")); } catch (Exception e) { - log.error("Something went wrong {}", e.getMessage(), e); + log.error("Something wents wrong {}", e.getMessage(), e); } } } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java index e65901a..2a6908c 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/BillingServiceImpl.java @@ -46,6 +46,8 @@ import com.epam.datalab.rest.client.RESTService; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.name.Named; +import lombok.AllArgsConstructor; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.http.client.utils.URIBuilder; @@ -99,6 +101,24 @@ public class BillingServiceImpl implements BillingService { .peek(this::appendStatuses) .filter(bd -> CollectionUtils.isEmpty(filter.getStatuses()) || filter.getStatuses().contains(bd.getStatus())) .collect(Collectors.toList()); + + //TEST + List<ProjectResources> collect = billingReportLines.stream() + .filter(x -> x.getShape().equals("0 x null")) + .map(x -> new ProjectResources(x.getResourceName(), x.getProject(), x)) + .collect(Collectors.toList()); + + collect.stream() + .peek(( + x -> System.out.println(exploratoryService.getUserInstance(user.getName(), x.getProject(), x.getResName())) + )) + .peek( + x -> System.out.println( + exploratoryService.getUserInstance(user.getName(), x.getProject(), x.getBillingReportLine().getExploratoryName())) + + ).count(); + +//TEST final LocalDate min = billingReportLines.stream().min(Comparator.comparing(BillingReportLine::getUsageDateFrom)).map(BillingReportLine::getUsageDateFrom).orElse(null); final LocalDate max = billingReportLines.stream().max(Comparator.comparing(BillingReportLine::getUsageDateTo)).map(BillingReportLine::getUsageDateTo).orElse(null); final double sum = billingReportLines.stream().mapToDouble(BillingReportLine::getCost).sum(); @@ -385,4 +405,12 @@ public class BillingServiceImpl implements BillingService { .map(Double::intValue) .orElse(BigDecimal.ZERO.intValue()); } -} + + @Data + @AllArgsConstructor + private class ProjectResources { + private String resName; + private String project; + private BillingReportLine billingReportLine; + } +} \ No newline at end of file diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java index 1e8b31c..3ca334f 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java @@ -151,6 +151,7 @@ public class BillingUtils { .filter(cr -> cr.getComputationalId() != null) .flatMap(cr -> { final String computationalId = cr.getComputationalId().toLowerCase(); + log.info("TEST LOG!!!: cr: {}", cr); return Stream.concat(Stream.of( withUserProjectEndpoint(userInstance) .resourceName(cr.getComputationalName()) @@ -244,10 +245,13 @@ public class BillingUtils { log.info("TEST LOG!!! getComputationalShape"); log.info("TEST LOG!!! UserComputationalResource: {}", resource); log.info("TEST LOG!!! getTemplateName: {}", resource.getTemplateName()); + log.info("TEST LOG!!! getTotalInstanceCount: {}", resource.getTotalInstanceCount()); log.info("TEST LOG!!! getDataengineShape: {}", resource.getDataengineShape()); log.info("TEST LOG!!! getMasterNodeShape: {}", resource.getMasterNodeShape()); + log.info("TEST LOG!!!: DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE: {} == {}", + DataEngineType.fromDockerImageName(resource.getImageName()), DataEngineType.SPARK_STANDALONE); - return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ? + return DataEngineType.fromDockerImageName(resource.getImageName()) != DataEngineType.SPARK_STANDALONE ? String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) : String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getTotalInstanceCount() - 1, resource.getSlaveNodeShape()); } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java index 7b1ea0a..9a77442 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/RequestBuilder.java @@ -654,8 +654,10 @@ public class RequestBuilder { } public UserEnvironmentResources newInfrastructureStatus(String user, CloudProvider cloudProvider, EnvResourceList resourceList) { - return newResourceSysBaseDTO(user, cloudProvider, UserEnvironmentResources.class) + UserEnvironmentResources userEnvironmentResources = newResourceSysBaseDTO(user, cloudProvider, UserEnvironmentResources.class) .withResourceList(resourceList); + log.info("TEST LOG!!! userEnvironmentResources: {}", userEnvironmentResources); + return userEnvironmentResources; } /** --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
