This is an automated email from the ASF dual-hosted git repository. ykinash pushed a commit to branch DATALAB-2544 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 30520309134d93354a05e0a7da5cb2e56655ed25 Author: KinashYurii <[email protected]> AuthorDate: Tue Oct 19 15:58:00 2021 +0300 [DATALAB-2544] -- * denied access for users with "auto" in username * fixed bucket access. --- .../main/java/com/epam/datalab/dto/aws/edge/EdgeInfoAws.java | 7 +++---- .../java/com/epam/datalab/dto/azure/edge/EdgeInfoAzure.java | 7 +++---- .../main/java/com/epam/datalab/dto/base/edge/EdgeInfo.java | 12 ++++++++++++ .../main/java/com/epam/datalab/dto/gcp/edge/EdgeInfoGcp.java | 7 +++---- .../schedulers/CheckInfrastructureStatusScheduler.java | 12 +++++++++++- .../epam/datalab/backendapi/service/KeycloakServiceImpl.java | 4 ---- .../service/impl/InfrastructureInfoServiceImpl.java | 1 + 7 files changed, 33 insertions(+), 17 deletions(-) diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/aws/edge/EdgeInfoAws.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/aws/edge/EdgeInfoAws.java index 813ad11..b117bb1 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/aws/edge/EdgeInfoAws.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/aws/edge/EdgeInfoAws.java @@ -22,16 +22,15 @@ package com.epam.datalab.dto.aws.edge; import com.epam.datalab.dto.base.edge.EdgeInfo; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import com.fasterxml.jackson.annotation.JsonTypeName; +import lombok.*; @Getter @Setter @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @JsonIgnoreProperties(ignoreUnknown = true) +//@JsonTypeName("aws") public class EdgeInfoAws extends EdgeInfo { @JsonProperty("user_own_bicket_name") private String userOwnBucketName; diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/azure/edge/EdgeInfoAzure.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/azure/edge/EdgeInfoAzure.java index c5bba7e..fe5ea88 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/azure/edge/EdgeInfoAzure.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/azure/edge/EdgeInfoAzure.java @@ -22,16 +22,15 @@ package com.epam.datalab.dto.azure.edge; import com.epam.datalab.dto.base.edge.EdgeInfo; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import com.fasterxml.jackson.annotation.JsonTypeName; +import lombok.*; @Getter @Setter @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @JsonIgnoreProperties(ignoreUnknown = true) +//@JsonTypeName("azure") public class EdgeInfoAzure extends EdgeInfo { @JsonProperty("user_storage_account_name") private String userStorageAccountName; diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/base/edge/EdgeInfo.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/base/edge/EdgeInfo.java index 46223a5..a7f1856 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/base/edge/EdgeInfo.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/base/edge/EdgeInfo.java @@ -19,6 +19,8 @@ package com.epam.datalab.dto.base.edge; +import com.epam.datalab.dto.aws.edge.EdgeInfoAws; +import com.epam.datalab.dto.azure.edge.EdgeInfoAzure; import com.epam.datalab.dto.gcp.edge.EdgeInfoGcp; import com.fasterxml.jackson.annotation.*; import lombok.Data; @@ -27,6 +29,16 @@ import java.util.List; @Data @JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo( + use = JsonTypeInfo.Id.CLASS, + include = JsonTypeInfo.As.PROPERTY, + property = "@class" +) +//@JsonSubTypes({ +// @JsonSubTypes.Type(value = EdgeInfoAws.class, name = "aws"), +// @JsonSubTypes.Type(value = EdgeInfoAzure.class, name = "azure"), +// @JsonSubTypes.Type(value = EdgeInfoGcp.class, name = "gcp") +//}) public class EdgeInfo { @JsonProperty("_id") @JsonInclude(JsonInclude.Include.NON_EMPTY) diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/edge/EdgeInfoGcp.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/edge/EdgeInfoGcp.java index 490ce54..ed2789d 100644 --- a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/edge/EdgeInfoGcp.java +++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/edge/EdgeInfoGcp.java @@ -22,16 +22,15 @@ package com.epam.datalab.dto.gcp.edge; import com.epam.datalab.dto.base.edge.EdgeInfo; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import com.fasterxml.jackson.annotation.JsonTypeName; +import lombok.*; @Getter @Setter @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @JsonIgnoreProperties(ignoreUnknown = true) +//@JsonTypeName("gcp") public class EdgeInfoGcp extends EdgeInfo { @JsonProperty("user_own_bucket_name") private String userOwnBucketName; diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java index ab5f7e0..00162f6 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/schedulers/CheckInfrastructureStatusScheduler.java @@ -32,6 +32,7 @@ import com.epam.datalab.backendapi.service.SecurityService; import com.epam.datalab.dto.UserInstanceDTO; import com.epam.datalab.dto.UserInstanceStatus; import com.epam.datalab.dto.base.DataEngineType; +import com.epam.datalab.dto.computational.UserComputationalResource; import com.epam.datalab.dto.status.EnvResource; import com.epam.datalab.model.ResourceType; import com.google.inject.Inject; @@ -50,13 +51,15 @@ import static com.epam.datalab.dto.UserInstanceStatus.*; public class CheckInfrastructureStatusScheduler implements Job { private static final List<UserInstanceStatus> statusesToCheck = - Arrays.asList(RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED); + Arrays.asList(CREATING, RUNNING, STOPPING, RECONFIGURING, STOPPED, TERMINATING, TERMINATED); private final InfrastructureInfoService infrastructureInfoService; private final SecurityService securityService; private final EndpointService endpointService; private final ExploratoryDAO exploratoryDAO; private final ProjectService projectService; + private static final String AWS_EMR_CLUSTER = "AWS EMR cluster"; + @Inject public CheckInfrastructureStatusScheduler(InfrastructureInfoService infrastructureInfoService, SecurityService securityService, @@ -107,6 +110,7 @@ public class CheckInfrastructureStatusScheduler implements Job { .stream() .filter(c -> DataEngineType.SPARK_STANDALONE == DataEngineType.fromDockerImageName(c.getImageName())) .filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus()))) + .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null) .map(r -> new EnvResource() .withId(r.getInstanceId()) .withName(r.getComputationalName()) @@ -131,6 +135,8 @@ public class CheckInfrastructureStatusScheduler implements Job { return userInstanceDTO.getResources().stream() .filter(c -> DataEngineType.CLOUD_SERVICE == DataEngineType.fromDockerImageName(c.getImageName())) .filter(c -> statusesToCheck.contains(UserInstanceStatus.of(c.getStatus()))) + .filter(c -> c.getComputationalId() != null && c.getInstanceId() != null) + .filter(this::noEmrCreating) .map(r -> new EnvResource() .withId(r.getInstanceId()) .withName(r.getComputationalName()) @@ -141,6 +147,10 @@ public class CheckInfrastructureStatusScheduler implements Job { .collect(Collectors.toList()); } + private boolean noEmrCreating(UserComputationalResource c) { + return !c.getStatus().equals(CREATING.name()) && !c.getTemplateName().contains(AWS_EMR_CLUSTER); + } + private List<EnvResource> getEdgeInstances(String endpoint) { return projectService.getProjectsByEndpoint(endpoint) diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/KeycloakServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/KeycloakServiceImpl.java index 518e14e..f418e65 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/KeycloakServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/KeycloakServiceImpl.java @@ -80,14 +80,10 @@ public class KeycloakServiceImpl implements KeycloakService { } private AccessTokenResponse requestToken(Form requestForm) { - log.info("TEST LOG!!!: access token form : {}", requestForm); - final String credentials = Base64.encodeAsString(String.join(":", conf.getResource(), String.valueOf(conf.getCredentials().get("secret")))); String url = conf.getAuthServerUrl() + String.format(URI, conf.getRealm()); String header = "Basic " + credentials; - log.info("TEST LOG!!!: post with: url: {}, cred: {}, header: {} : {}", url, credentials, HttpHeaders.AUTHORIZATION, header); - final Response response = httpClient.target(url) .request() diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java index fe2f340..5ed48f6 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java @@ -191,6 +191,7 @@ public class InfrastructureInfoServiceImpl implements InfrastructureInfoService shared.put("status", endpointDTO.getStatus().toString()); shared.put("edge_node_ip", edge.getPublicIp()); + if (edge instanceof EdgeInfoAws) { EdgeInfoAws edgeInfoAws = (EdgeInfoAws) edge; shared.put("user_own_bicket_name", edgeInfoAws.getUserOwnBucketName()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
