This is an automated email from the ASF dual-hosted git repository. dyankiv pushed a commit to branch DATALAB-2809 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 8a94b9401119c8aede38e5d2f27372bb4eefc170 Author: Denys Yankiv <[email protected]> AuthorDate: Mon Jun 27 02:33:19 2022 +0300 initial support for sharing images --- .../backendapi/dao/ImageExploratoryDAO.java | 3 + .../backendapi/dao/ImageExploratoryDAOImpl.java | 17 +++- .../resources/ImageExploratoryResource.java | 15 ++- .../backendapi/resources/dto/ImageInfoRecord.java | 2 +- ...Record.java => ImageProjectGroupsShareDTO.java} | 28 +----- .../backendapi/resources/dto/UserRoleDTO.java | 6 +- .../epam/datalab/backendapi/roles/RoleType.java | 1 + .../service/ImageExploratoryService.java | 9 ++ .../service/impl/ImageExploratoryServiceImpl.java | 102 +++++++++++++++++++-- .../main/resources/mongo/image/mongo_roles.json | 10 ++ 10 files changed, 149 insertions(+), 44 deletions(-) diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAO.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAO.java index 4d97437e6..8b9ccc84c 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAO.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAO.java @@ -38,10 +38,13 @@ public interface ImageExploratoryDAO { List<ImageInfoRecord> getImages(String user, String dockerImage, String project, String endpoint, ImageStatus... statuses); + List<ImageInfoRecord> getImages(String project, String endpoint, String dockerImage); + List<ImageInfoRecord> getImagesOfUser(String user, String project); List<ImageInfoRecord> getImagesForProject(String project); + List<ImageInfoRecord> getAllImages(); Optional<ImageInfoRecord> getImage(String user, String name, String project, String endpoint); List<Library> getLibraries(String user, String imageFullName, String project, String endpoint, LibStatus status); diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAOImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAOImpl.java index 95f4e0a1a..9fb160585 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAOImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/ImageExploratoryDAOImpl.java @@ -54,7 +54,6 @@ public class ImageExploratoryDAOImpl extends BaseDAO implements ImageExploratory private static final String DOCKER_IMAGE = "dockerImage"; private static final String PROJECT = "project"; private static final String ENDPOINT = "endpoint"; - private static final String CREATION_DATE = "creationDate"; @Override public boolean exist(String image, String project) { @@ -80,6 +79,13 @@ public class ImageExploratoryDAOImpl extends BaseDAO implements ImageExploratory ImageInfoRecord.class); } + @Override + public List<ImageInfoRecord> getImages(String project, String endpoint, String dockerImage) { + return find(MongoCollections.IMAGES, + imageProjectEndpointDockerCondition(project, endpoint, dockerImage), + ImageInfoRecord.class); + } + @Override public List<ImageInfoRecord> getImagesOfUser(String user, String project) { return find(MongoCollections.IMAGES, @@ -94,6 +100,11 @@ public class ImageExploratoryDAOImpl extends BaseDAO implements ImageExploratory ImageInfoRecord.class); } + @Override + public List<ImageInfoRecord> getAllImages() { + return find(MongoCollections.IMAGES, ImageInfoRecord.class); + } + @Override public Optional<ImageInfoRecord> getImage(String user, String name, String project, String endpoint) { return findOne(MongoCollections.IMAGES, userImageCondition(user, name, project, endpoint), ImageInfoRecord.class); @@ -158,6 +169,10 @@ public class ImageExploratoryDAOImpl extends BaseDAO implements ImageExploratory return and(eq(USER, user), eq(PROJECT, project)); } + private Bson imageProjectEndpointDockerCondition(String project, String endpoint, String dockerImage){ + return and(eq(PROJECT, project), eq(ENDPOINT, endpoint), eq(DOCKER_IMAGE,dockerImage)); + } + private Document getUpdatedFields(Image image) { return new Document(STATUS, image.getStatus().toString()) .append(IMAGE_FULL_NAME, image.getFullName()) diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ImageExploratoryResource.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ImageExploratoryResource.java index 8c5c76861..819ae04bc 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ImageExploratoryResource.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ImageExploratoryResource.java @@ -21,10 +21,7 @@ package com.epam.datalab.backendapi.resources; import com.epam.datalab.auth.UserInfo; import com.epam.datalab.backendapi.domain.RequestId; -import com.epam.datalab.backendapi.resources.dto.ExploratoryImageCreateFormDTO; -import com.epam.datalab.backendapi.resources.dto.ImageFilter; -import com.epam.datalab.backendapi.resources.dto.ImageInfoRecord; -import com.epam.datalab.backendapi.resources.dto.ProjectImagesInfo; +import com.epam.datalab.backendapi.resources.dto.*; import com.epam.datalab.backendapi.service.ImageExploratoryService; import com.google.inject.Inject; import io.dropwizard.auth.Auth; @@ -113,7 +110,7 @@ public class ImageExploratoryResource { @Path("user") public Response getImagesForUser(@Auth UserInfo ui, @Valid @NotNull ImageFilter imageFilter) { log.debug("Getting images for user {} with filter {}", ui.getName(), imageFilter); - final List<ProjectImagesInfo> images = imageExploratoryService.getImagesOfUserWithFilter(ui ,imageFilter); + final List<ProjectImagesInfo> images = imageExploratoryService.getImagesOfUserWithFilter(ui, imageFilter); return Response.ok(images).build(); } @@ -127,4 +124,12 @@ public class ImageExploratoryResource { log.debug("Getting image with name {} for user {}", name, ui.getName()); return Response.ok(imageExploratoryService.getImage(ui.getName(), name, project, endpoint)).build(); } + + @POST + @Path("share") + public Response shareImageWithProjectGroups(@Auth UserInfo ui, @Valid @NotNull ImageProjectGroupsShareDTO dto) { + log.debug("Sharing user image {} with project {} groups", dto.getImageName(), dto.getProjectName()); + imageExploratoryService.shareImageWithProjectGroups(ui, dto.getImageName(), dto.getProjectName(), dto.getEndpoint()); + return Response.ok(imageExploratoryService.getImagesOfUser(ui)).build(); + } } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageInfoRecord.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageInfoRecord.java index d1dd17b1f..4417b211a 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageInfoRecord.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageInfoRecord.java @@ -26,7 +26,6 @@ import com.epam.datalab.model.library.Library; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; -import java.time.LocalDateTime; import java.util.Date; import java.util.List; import java.util.Map; @@ -43,6 +42,7 @@ public class ImageInfoRecord { private final String application; private final String instanceName; private final CloudProvider cloudProvider; + private final String dockerImage; private final String fullName; private final ImageStatus status; private final boolean isShared; diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageInfoRecord.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageProjectGroupsShareDTO.java similarity index 52% copy from services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageInfoRecord.java copy to services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageProjectGroupsShareDTO.java index d1dd17b1f..17613954c 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageInfoRecord.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ImageProjectGroupsShareDTO.java @@ -19,35 +19,13 @@ package com.epam.datalab.backendapi.resources.dto; -import com.epam.datalab.cloud.CloudProvider; -import com.epam.datalab.dto.aws.computational.ClusterConfig; -import com.epam.datalab.dto.exploratory.ImageStatus; -import com.epam.datalab.model.library.Library; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; -import java.time.LocalDateTime; -import java.util.Date; -import java.util.List; -import java.util.Map; - @Data @JsonIgnoreProperties(ignoreUnknown = true) -public class ImageInfoRecord { - private final String name; - private final Date timestamp; - private final String description; - private final String project; +public class ImageProjectGroupsShareDTO { + private final String imageName; + private final String projectName; private final String endpoint; - private final String user; - private final String application; - private final String instanceName; - private final CloudProvider cloudProvider; - private final String fullName; - private final ImageStatus status; - private final boolean isShared; - private final List<ClusterConfig> clusterConfig; - private final String exploratoryURL; - private final List<Library> libraries; - private final Map<String, List<Library>> computationalLibraries; } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/UserRoleDTO.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/UserRoleDTO.java index 62514f328..cdf652157 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/UserRoleDTO.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/UserRoleDTO.java @@ -42,15 +42,17 @@ public class UserRoleDTO { private Set<String> pages; private Set<String> computationals; private Set<String> exploratories; + private Set<String> images; @JsonProperty("exploratory_shapes") private Set<String> exploratoryShapes; @JsonProperty("computational_shapes") private Set<String> computationalShapes; private Set<String> groups; - private enum Type { + public enum Type { NOTEBOOK, COMPUTATIONAL, + IMAGE, NOTEBOOK_SHAPE, COMPUTATIONAL_SHAPE, BILLING, @@ -59,6 +61,6 @@ public class UserRoleDTO { } public static List<Type> cloudSpecificTypes() { - return Arrays.asList(Type.NOTEBOOK, Type.COMPUTATIONAL, Type.NOTEBOOK_SHAPE, Type.COMPUTATIONAL_SHAPE); + return Arrays.asList(Type.NOTEBOOK, Type.COMPUTATIONAL, Type.IMAGE, Type.NOTEBOOK_SHAPE, Type.COMPUTATIONAL_SHAPE); } } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/RoleType.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/RoleType.java index 998e01f39..1b27567ce 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/RoleType.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/RoleType.java @@ -25,6 +25,7 @@ package com.epam.datalab.backendapi.roles; public enum RoleType { COMPUTATIONAL("computationals"), EXPLORATORY("exploratories"), + IMAGE("images"), EXPLORATORY_SHAPES("exploratory_shapes"), COMPUTATIONAL_SHAPES("computational_shapes"), PAGE("pages"); diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/ImageExploratoryService.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/ImageExploratoryService.java index 99973f1a6..98c439b70 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/ImageExploratoryService.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/ImageExploratoryService.java @@ -35,10 +35,19 @@ public interface ImageExploratoryService { List<ImageInfoRecord> getNotFailedImages(String user, String dockerImage, String project, String endpoint); + List<ImageInfoRecord> getNotFailedImages(String dockerImage, String project, String endpoint); + ImageInfoRecord getImage(String user, String name, String project, String endpoint); List<ImageInfoRecord> getImagesForProject(String project); List<ProjectImagesInfo> getImagesOfUser(UserInfo user); List<ProjectImagesInfo> getImagesOfUserWithFilter(UserInfo user, ImageFilter imageFilter); + + void shareImageWithProjectGroups(UserInfo user, String imageName, String projectName, String endpoint); + + List<ImageInfoRecord> getSharedImages(UserInfo user); + + List<ImageInfoRecord> getSharedImages(UserInfo userInfo, String dockerImage, String project, String endpoint); + } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ImageExploratoryServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ImageExploratoryServiceImpl.java index d2ec48e0f..5c9eaec4e 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ImageExploratoryServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ImageExploratoryServiceImpl.java @@ -25,18 +25,20 @@ import com.epam.datalab.backendapi.annotation.Info; import com.epam.datalab.backendapi.annotation.Project; import com.epam.datalab.backendapi.annotation.ResourceName; import com.epam.datalab.backendapi.annotation.User; -import com.epam.datalab.backendapi.dao.ExploratoryDAO; -import com.epam.datalab.backendapi.dao.ExploratoryLibDAO; -import com.epam.datalab.backendapi.dao.ImageExploratoryDAO; +import com.epam.datalab.backendapi.dao.*; import com.epam.datalab.backendapi.domain.EndpointDTO; import com.epam.datalab.backendapi.domain.ProjectDTO; import com.epam.datalab.backendapi.resources.dto.ImageFilter; import com.epam.datalab.backendapi.resources.dto.ImageInfoRecord; import com.epam.datalab.backendapi.resources.dto.ProjectImagesInfo; +import com.epam.datalab.backendapi.resources.dto.UserRoleDTO; +import com.epam.datalab.backendapi.roles.RoleType; +import com.epam.datalab.backendapi.roles.UserRoles; import com.epam.datalab.backendapi.service.EndpointService; import com.epam.datalab.backendapi.service.ImageExploratoryService; import com.epam.datalab.backendapi.service.ProjectService; import com.epam.datalab.backendapi.util.RequestBuilder; +import com.epam.datalab.cloud.CloudProvider; import com.epam.datalab.constants.ServiceConsts; import com.epam.datalab.dto.UserInstanceDTO; import com.epam.datalab.dto.UserInstanceStatus; @@ -49,15 +51,18 @@ import com.epam.datalab.model.exploratory.Image; import com.epam.datalab.model.library.Library; import com.epam.datalab.rest.client.RESTService; import com.epam.datalab.rest.contracts.ExploratoryAPI; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.io.InputStream; import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -69,13 +74,22 @@ import static com.epam.datalab.backendapi.domain.AuditResourceTypeEnum.IMAGE; public class ImageExploratoryServiceImpl implements ImageExploratoryService { private static final String IMAGE_EXISTS_MSG = "Image with name %s is already exist in project %s"; private static final String IMAGE_NOT_FOUND_MSG = "Image with name %s was not found for user %s"; + private static final String PATH_TO_IMAGE_ROLES = "/mongo/image/mongo_roles.json"; + private static final String IMAGE_FULL_CONTROL_ROLE = "imgFull_%s_%s_%s"; + private static final ObjectMapper MAPPER = new ObjectMapper(); @Inject private ExploratoryDAO exploratoryDAO; @Inject private ImageExploratoryDAO imageExploratoryDao; @Inject private ExploratoryLibDAO libDAO; + + @Inject + private UserRoleDAO userRoleDAO; + + @Inject + private UserGroupDAO userGroupDAO; @Inject @Named(ServiceConsts.PROVISIONING_SERVICE_NAME) private RESTService provisioningService; @@ -119,7 +133,6 @@ public class ImageExploratoryServiceImpl implements ImageExploratoryService { .withProject(project) .withExploratoryName(exploratoryName) .withStatus(UserInstanceStatus.CREATING_IMAGE)); - EndpointDTO endpointDTO = endpointService.get(userInstance.getEndpoint()); return provisioningService.post(endpointDTO.getUrl() + ExploratoryAPI.EXPLORATORY_IMAGE, user.getAccessToken(), @@ -136,6 +149,19 @@ public class ImageExploratoryServiceImpl implements ImageExploratoryService { .withExploratoryName(exploratoryName) .withStatus(UserInstanceStatus.RUNNING)); imageExploratoryDao.updateImageFields(image); + + // Create image roles + if (image.getStatus().equals(ImageStatus.CREATED)){ + List<UserRoleDTO> imageRoles = getUserImageRoleFromFile(); + imageRoles.stream().forEach(role -> { + role.setId(String.format(role.getId(), image.getProject(), image.getEndpoint(), image.getName())); + role.setDescription(String.format(role.getDescription(), image.getFullName())); + role.setCloud(CloudProvider.valueOf(image.getCloudProvider())); + role.setImages(new HashSet<>(Collections.singletonList(image.getFullName()))); + }); + userRoleDAO.insert(imageRoles); + } + if (newNotebookIp != null) { log.debug("Changing exploratory ip with name {} for user {} to {}", exploratoryName, image.getUser(), newNotebookIp); @@ -149,6 +175,11 @@ public class ImageExploratoryServiceImpl implements ImageExploratoryService { return imageExploratoryDao.getImages(user, dockerImage, project, endpoint, ImageStatus.CREATED, ImageStatus.CREATING); } + @Override + public List<ImageInfoRecord> getNotFailedImages(String dockerImage, String project, String endpoint) { + return imageExploratoryDao.getImages(project, endpoint, dockerImage); + } + @Override public ImageInfoRecord getImage(String user, String name, String project, String endpoint) { return imageExploratoryDao.getImage(user, name, project, endpoint).orElseThrow(() -> @@ -165,8 +196,9 @@ public class ImageExploratoryServiceImpl implements ImageExploratoryService { log.debug("Loading list of images for user {}", user.getName()); return projectService.getUserProjects(user, Boolean.FALSE) .stream() - .map( p-> { - List<ImageInfoRecord> images = imageExploratoryDao.getImagesOfUser(user.getName(), p.getName()); + .map(p -> { + List<ImageInfoRecord> images = imageExploratoryDao.getImagesOfUser(user.getName(), p.getName()); + images.addAll(getSharedImages(user, p.getName())); return ProjectImagesInfo.builder() .project(p.getName()) .images(images) @@ -180,8 +212,8 @@ public class ImageExploratoryServiceImpl implements ImageExploratoryService { log.debug("Loading list of images for user {}", user.getName()); return projectService.getUserProjects(user, Boolean.FALSE) .stream() - .map( p-> { - List<ImageInfoRecord> images = imageExploratoryDao.getImagesOfUser(user.getName(), p.getName()); + .map(p -> { + List<ImageInfoRecord> images = imageExploratoryDao.getImagesOfUser(user.getName(), p.getName()); return ProjectImagesInfo.builder() .project(p.getName()) .images(images) @@ -190,6 +222,14 @@ public class ImageExploratoryServiceImpl implements ImageExploratoryService { .collect(Collectors.toList()); } + @Override + public void shareImageWithProjectGroups(UserInfo user, String imageName, String projectName, String endpoint) { + Set<String> projectGroups = projectService.get(projectName).getGroups(); + userRoleDAO.addGroupToRole(projectGroups, + Collections.singleton(String.format(IMAGE_FULL_CONTROL_ROLE, + projectName, endpoint, imageName))); + } + private Map<String, List<Library>> fetchComputationalLibs(List<Library> libraries) { return libraries.stream() .filter(resourceTypePredicate(ResourceType.COMPUTATIONAL)) @@ -210,4 +250,46 @@ public class ImageExploratoryServiceImpl implements ImageExploratoryService { private Predicate<Library> resourceTypePredicate(ResourceType resourceType) { return l -> resourceType == l.getType(); } + + private List<UserRoleDTO> getUserImageRoleFromFile() { + try (InputStream is = getClass().getResourceAsStream(PATH_TO_IMAGE_ROLES)) { + return MAPPER.readValue(is, new TypeReference<List<UserRoleDTO>>() { + }); + } catch (IOException e) { + log.error("Can not marshall datalab image roles due to: {}", e.getMessage(), e); + throw new IllegalStateException("Can not marshall datalab image roles due to: " + e.getMessage()); + } + } + + public List<ImageInfoRecord> getSharedImages(UserInfo userInfo) { + List<ImageInfoRecord> sharedImages = imageExploratoryDao.getAllImages().stream() + .filter(img -> !img.getUser().equals(userInfo.getName())) + .filter(img -> + UserRoles.checkAccess(userInfo, RoleType.IMAGE, img.getFullName(), userInfo.getRoles())) + .collect(Collectors.toList()); + log.info("Shared with user {} images : {}", userInfo.getName(), sharedImages); + return sharedImages; + } + + public List<ImageInfoRecord> getSharedImages(UserInfo userInfo, String dockerImage, String project, String endpoint) { + List<ImageInfoRecord> sharedImages = imageExploratoryDao.getAllImages().stream() + .filter(img -> img.getStatus().equals(ImageStatus.CREATED)) + .filter(img -> !img.getUser().equals(userInfo.getName())) + .filter(img -> img.getDockerImage().equals(dockerImage) && img.getProject().equals(project) && img.getEndpoint().equals(endpoint)) + .filter(img -> UserRoles.checkAccess(userInfo, RoleType.IMAGE, img.getFullName(), userInfo.getRoles())) + .collect(Collectors.toList()); + log.info("Found shared with user {} images {}", userInfo.getName(), sharedImages); + return sharedImages; + } + + public List<ImageInfoRecord> getSharedImages(UserInfo userInfo, String project){ + List<ImageInfoRecord> sharedImages = imageExploratoryDao.getAllImages().stream() + .filter(img -> img.getStatus().equals(ImageStatus.CREATED)) + .filter(img -> !img.getUser().equals(userInfo.getName())) + .filter(img -> img.getProject().equals(project) ) + .filter(img -> UserRoles.checkAccess(userInfo, RoleType.IMAGE, img.getFullName(), userInfo.getRoles())) + .collect(Collectors.toList()); + log.info("Found shared with user {} images {}", userInfo.getName(), sharedImages); + return sharedImages; + } } diff --git a/services/self-service/src/main/resources/mongo/image/mongo_roles.json b/services/self-service/src/main/resources/mongo/image/mongo_roles.json new file mode 100644 index 000000000..e9efc3437 --- /dev/null +++ b/services/self-service/src/main/resources/mongo/image/mongo_roles.json @@ -0,0 +1,10 @@ +[ + { + "_id": "imgFull_%s_%s_%s", + "description": "Full control of %s image", + "type": "IMAGE", + "groups": [ + "$anyuser" + ] + } +] \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
