This is an automated email from the ASF dual-hosted git repository. ppapou pushed a commit to branch DATALAB-2245 in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5cc3e7a3edd718ac08501c21f22bd37f944f3aee Author: Pavel Papou <[email protected]> AuthorDate: Thu Mar 31 21:58:45 2022 -0400 [DATALAB-2245] Export Audit report - The audit report is exported to CSV file --- .../com/epam/datalab/backendapi/dao/AuditDAO.java | 3 ++ .../epam/datalab/backendapi/dao/AuditDAOImpl.java | 43 ++++++++++++++++-- .../datalab/backendapi/domain/AuditReportLine.java | 28 ++++++++++++ .../backendapi/resources/AuditResource.java | 18 ++++++++ .../datalab/backendapi/service/AuditService.java | 4 ++ .../backendapi/service/impl/AuditServiceImpl.java | 20 ++++++-- .../epam/datalab/backendapi/util/AuditUtils.java | 53 ++++++++++++++++++++++ 7 files changed, 162 insertions(+), 7 deletions(-) diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAO.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAO.java index ae840fb..249b85f 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAO.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAO.java @@ -21,6 +21,7 @@ package com.epam.datalab.backendapi.dao; import com.epam.datalab.backendapi.domain.AuditDTO; import com.epam.datalab.backendapi.domain.AuditPaginationDTO; +import com.epam.datalab.backendapi.domain.AuditReportLine; import java.util.List; @@ -28,4 +29,6 @@ public interface AuditDAO { void save(AuditDTO audit); List<AuditPaginationDTO> getAudit(List<String> users, List<String> projects, List<String> resourceNames, List<String> resourceTypes, String dateStart, String dateEnd, int pageNumber, int pageSize); + + List<AuditReportLine> aggregateAuditReport(List<String> users, List<String> projects, List<String> resourceNames, List<String> resourceTypes, String dateStart, String dateEnd, int pageNumber, int pageSize); } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAOImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAOImpl.java index 34c14a2..2da9274 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAOImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/dao/AuditDAOImpl.java @@ -21,6 +21,7 @@ package com.epam.datalab.backendapi.dao; import com.epam.datalab.backendapi.domain.AuditDTO; import com.epam.datalab.backendapi.domain.AuditPaginationDTO; +import com.epam.datalab.backendapi.domain.AuditReportLine; import com.epam.datalab.exceptions.DatalabException; import com.mongodb.client.model.Facet; import com.mongodb.client.model.Filters; @@ -33,6 +34,7 @@ import org.bson.conversions.Bson; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; +import java.time.ZoneId; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -69,6 +71,8 @@ public class AuditDAOImpl extends BaseDAO implements AuditDAO { private static final String USER_FACET = "userFacet"; private static final String PROJECT_FACET = "projectFacet"; private static final String RESOURCE_TYPE_FACET = "typeFacet"; + private static final String DATALAB_ID = "datalabId"; + private static final String ACTION = "action"; @Override public void save(AuditDTO audit) { @@ -78,6 +82,28 @@ public class AuditDAOImpl extends BaseDAO implements AuditDAO { @Override public List<AuditPaginationDTO> getAudit(List<String> users, List<String> projects, List<String> resourceNames, List<String> resourceTypes, String dateStart, String dateEnd, int pageNumber, int pageSize) { + List<Bson> facets = getFacets(users, projects, resourceNames, resourceTypes, dateStart, dateEnd, pageNumber, pageSize); + return StreamSupport.stream(aggregate(AUDIT_COLLECTION, facets).spliterator(), false) + .map(this::toAuditPaginationDTO) + .collect(Collectors.toList()); + } + + public List<AuditReportLine> aggregateAuditReport(List<String> users, List<String> projects, List<String> resourceNames, List<String> resourceTypes, String dateStart, String dateEnd, + int pageNumber, int pageSize) { + List<Bson> facets = getFacets(users, projects, resourceNames, resourceTypes, dateStart, dateEnd, pageNumber, pageSize); + List<Document> auditDocuments = new ArrayList<>(); + StreamSupport.stream(aggregate(AUDIT_COLLECTION, facets).spliterator(), false) + .peek(System.out::println) + .map(document -> (ArrayList<Document>)document.get(AUDIT_FACET)) + .forEach(auditDocuments::addAll); + + return auditDocuments.stream() + .map(this::toAuditReport) + .collect(Collectors.toList()); + } + + private List<Bson> getFacets(List<String> users, List<String> projects, List<String> resourceNames, List<String> resourceTypes, String dateStart, String dateEnd, + int pageNumber, int pageSize){ List<Bson> valuesPipeline = new ArrayList<>(); List<Bson> countPipeline = new ArrayList<>(); List<Bson> matchCriteria = matchCriteria(users, projects, resourceNames, resourceTypes, dateStart, dateEnd); @@ -95,12 +121,9 @@ public class AuditDAOImpl extends BaseDAO implements AuditDAO { List<Bson> resourceNameFilter = Collections.singletonList(group(getGroupingFields(RESOURCE_NAME_FIELD))); List<Bson> resourceTypeFilter = Collections.singletonList(group(getGroupingFields(RESOURCE_TYPE_FIELD))); - List<Bson> facets = Collections.singletonList(facet(new Facet(AUDIT_FACET, valuesPipeline), new Facet(TOTAL_COUNT_FACET, countPipeline), + return Collections.singletonList(facet(new Facet(AUDIT_FACET, valuesPipeline), new Facet(TOTAL_COUNT_FACET, countPipeline), new Facet(RESOURCE_NAME_FACET, resourceNameFilter), new Facet(USER_FACET, userFilter), new Facet(PROJECT_FACET, projectFilter), new Facet(RESOURCE_TYPE_FACET, resourceTypeFilter))); - return StreamSupport.stream(aggregate(AUDIT_COLLECTION, facets).spliterator(), false) - .map(this::toAuditPaginationDTO) - .collect(Collectors.toList()); } private List<Bson> matchCriteria(List<String> users, List<String> projects, List<String> resourceNames, List<String> resourceTypes, String dateStart, String dateEnd) { @@ -144,6 +167,18 @@ public class AuditDAOImpl extends BaseDAO implements AuditDAO { .build(); } + private AuditReportLine toAuditReport(Document doc) { + return AuditReportLine.builder() + .datalabId(doc.getString(DATALAB_ID)) + .project(doc.getString(PROJECT)) + .resourceName(doc.getString(RESOURCE_NAME_FIELD)) + .action(doc.getString(ACTION)) + .user(doc.getString(USER)) + .resourceType(doc.getString(RESOURCE_TYPE_FIELD)) + .timestamp(doc.getDate(TIMESTAMP_FIELD).toInstant().atZone(ZoneId.systemDefault()).toLocalDate()) + .build(); + } + private Set<String> getFilter(Document document, String facet, String field) { return ((List<Document>) document.get(facet)) .stream() diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/AuditReportLine.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/AuditReportLine.java new file mode 100644 index 0000000..9b7dd0a --- /dev/null +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/AuditReportLine.java @@ -0,0 +1,28 @@ +package com.epam.datalab.backendapi.domain; + +import com.epam.datalab.dto.UserInstanceStatus; +import com.epam.datalab.dto.billing.BillingResourceType; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDate; + + +@Data +@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@AllArgsConstructor +@NoArgsConstructor +public class AuditReportLine { + private String datalabId; + private String action; + private String resourceType; + private String resourceName; + private String project; + private String user; + private LocalDate timestamp; +} diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/AuditResource.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/AuditResource.java index aa04fd0..c544b49 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/AuditResource.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/AuditResource.java @@ -33,6 +33,7 @@ import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -67,4 +68,21 @@ public class AuditResource { .ok(auditService.getAudit(users, projects, resourceNames, resourceTypes, dateStart, dateEnd, pageNumber, pageSize)) .build(); } + + @POST + @Path("/report/download") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + public Response downloadAuditReport(@Auth UserInfo userInfo, + @QueryParam("users") StringList users, + @QueryParam("projects") StringList projects, + @QueryParam("resource-names") StringList resourceNames, + @QueryParam("resource-types") StringList resourceTypes, + @QueryParam("date-start") String dateStart, + @QueryParam("date-end") String dateEnd, + @QueryParam("page-number") int pageNumber, + @QueryParam("page-size") int pageSize) { + return Response.ok(auditService.downloadAuditReport(users, projects, resourceNames, resourceTypes, dateStart, dateEnd, pageNumber, pageSize)) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"audit-report.csv\"") + .build(); + } } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/AuditService.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/AuditService.java index 6dc4ab5..8afe546 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/AuditService.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/AuditService.java @@ -22,6 +22,7 @@ package com.epam.datalab.backendapi.service; import com.epam.datalab.backendapi.domain.AuditCreateDTO; import com.epam.datalab.backendapi.domain.AuditDTO; import com.epam.datalab.backendapi.domain.AuditPaginationDTO; +import com.epam.datalab.model.StringList; import java.util.List; @@ -31,4 +32,7 @@ public interface AuditService { void save(String user, AuditCreateDTO audit); List<AuditPaginationDTO> getAudit(List<String> users, List<String> projects, List<String> resourceNames, List<String> resourceTypes, String dateStart, String dateEnd, int pageNumber, int pageSize); + + String downloadAuditReport(StringList users, StringList projects, StringList resourceNames, StringList resourceTypes, String dateStart, String dateEnd, int pageNumber, int pageSize); + } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/AuditServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/AuditServiceImpl.java index 161bf2b..bfaf714 100644 --- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/AuditServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/AuditServiceImpl.java @@ -20,12 +20,13 @@ package com.epam.datalab.backendapi.service.impl; import com.epam.datalab.backendapi.dao.AuditDAO; -import com.epam.datalab.backendapi.domain.AuditCreateDTO; -import com.epam.datalab.backendapi.domain.AuditDTO; -import com.epam.datalab.backendapi.domain.AuditPaginationDTO; +import com.epam.datalab.backendapi.domain.*; import com.epam.datalab.backendapi.service.AuditService; +import com.epam.datalab.backendapi.util.AuditUtils; +import com.epam.datalab.model.StringList; import com.google.inject.Inject; +import java.time.LocalDate; import java.util.List; import static com.epam.datalab.backendapi.domain.AuditActionEnum.FOLLOW_LINK; @@ -60,4 +61,17 @@ public class AuditServiceImpl implements AuditService { String dateStart, String dateEnd, int pageNumber, int pageSize) { return auditDAO.getAudit(users, projects, resourceNames, resourceTypes, dateStart, dateEnd, pageNumber, pageSize); } + + public String downloadAuditReport(StringList users, StringList projects, StringList resourceNames, StringList resourceTypes, String dateStart, String dateEnd, int pageNumber, int pageSize) { + + List<AuditReportLine> auditReportLines = auditDAO.aggregateAuditReport(users, projects, resourceNames, resourceTypes, dateStart, dateEnd, pageNumber, pageSize); + final LocalDate dateFrom = LocalDate.parse(dateStart); + final LocalDate dateTo = LocalDate.parse(dateEnd); + //Locale can be added as parameter in method + StringBuilder reportHead = new StringBuilder(AuditUtils.getFirstLine(dateFrom, dateTo, "en-GB")); + reportHead.append(AuditUtils.getHeader()); + auditReportLines.forEach(r -> reportHead.append(AuditUtils.printLine(r))); + + return reportHead.toString(); + } } diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/AuditUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/AuditUtils.java new file mode 100644 index 0000000..a33c85e --- /dev/null +++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/AuditUtils.java @@ -0,0 +1,53 @@ +package com.epam.datalab.backendapi.util; + +import com.epam.datalab.backendapi.domain.AuditReportLine; +import com.epam.datalab.backendapi.domain.BillingReportLine; +import com.epam.datalab.dto.UserInstanceStatus; +import jersey.repackaged.com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.*; + +public class AuditUtils { + + private static final String REPORT_FIRST_LINE = "Available reporting period from: %s to: %s"; + private static final String[] AUDIT_REPORT_HEADER = {"Date", "User", "Action", "Project", "Resource type", "Resource"}; + + /** + * @param from formatted date, like 2020-04-07 + * @param to formatted date, like 2020-05-07 + * @param locale user's locale + * @return line, like: + * Available reporting period from: 2020-04-07 to: 2020-04-07" + */ + public static String getFirstLine(LocalDate from, LocalDate to, String locale) { + return CSVFormatter.formatLine(Lists.newArrayList(String.format(REPORT_FIRST_LINE, + Optional.ofNullable(from).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY), + Optional.ofNullable(to).map(date -> date.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM).withLocale(Locale.forLanguageTag(locale)))).orElse(StringUtils.EMPTY))), + CSVFormatter.SEPARATOR, '\"'); + } + + public static String printLine(AuditReportLine line) { + List<String> lines = new ArrayList<>(); + lines.add(getOrEmpty(line.getTimestamp().toString())); + lines.add(getOrEmpty(line.getUser())); + lines.add(getOrEmpty(line.getAction())); + lines.add(getOrEmpty(line.getProject())); + lines.add(getOrEmpty(line.getResourceType()));; + lines.add(getOrEmpty(line.getResourceName())); + return CSVFormatter.formatLine(lines, CSVFormatter.SEPARATOR); + } + + public static String getHeader() { + return CSVFormatter.formatLine(Arrays.asList(AuditUtils.AUDIT_REPORT_HEADER), CSVFormatter.SEPARATOR); + } + + private static String getOrEmpty(String s) { + + return Objects.nonNull(s) ? s : StringUtils.EMPTY; + } + +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
