METRON-1685 Retrieve Pcap results in raw binary format (merrimanr) closes apache/metron#1123
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a5a51399 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a5a51399 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a5a51399 Branch: refs/heads/master Commit: a5a51399d2eafd2535d79bb13ee0d4d8eb2e2d23 Parents: 3e5ef41 Author: merrimanr <merrim...@gmail.com> Authored: Fri Jul 20 09:37:34 2018 -0500 Committer: rmerriman <merrim...@gmail.com> Committed: Fri Jul 20 09:37:34 2018 -0500 ---------------------------------------------------------------------- metron-interface/metron-rest/README.md | 10 ++++ .../metron/rest/controller/PcapController.java | 40 ++++++++++++++ .../apache/metron/rest/service/PcapService.java | 5 +- .../rest/service/impl/PcapServiceImpl.java | 17 ++++++ .../PcapControllerIntegrationTest.java | 44 ++++++++++++++++ .../rest/service/impl/PcapServiceImplTest.java | 55 ++++++++++++++++++++ 6 files changed, 170 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 7b3a263..4a7102f 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -256,6 +256,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)| | [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)| | [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)| +| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)| | [ `GET /api/v1/search/search`](#get-apiv1searchsearch)| | [ `POST /api/v1/search/search`](#get-apiv1searchsearch)| | [ `POST /api/v1/search/group`](#get-apiv1searchgroup)| @@ -523,6 +524,15 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Returns: * 200 - Returns PDML in json format. * 404 - Job or page is missing. + +### `POST /api/v1/pcap/{jobId}/raw` + * Description: Download Pcap Results for a page. + * Input: + * jobId - Job ID of submitted job + * page - Page number + * Returns: + * 200 - Returns Pcap as a file download. + * 404 - Job or page is missing. ### `POST /api/v1/search/search` * Description: Searches the indexing store. GUIDs must be quoted to ensure correct results. http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java index 47bc6a0..23bb0b9 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java @@ -21,6 +21,8 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.pcap.FixedPcapRequest; import org.apache.metron.rest.model.pcap.PcapStatus; @@ -37,10 +39,18 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + @RestController @RequestMapping("/api/v1/pcap") public class PcapController { + private static final String PCAP_FILENAME_FORMAT = "pcap_%s_%s.pcap"; + @Autowired private PcapService pcapQueryService; @@ -99,4 +109,34 @@ public class PcapController { } } + @ApiOperation(value = "Download Pcap Results for a page.") + @ApiResponses(value = { + @ApiResponse(message = "Returns Pcap as a file download.", code = 200), + @ApiResponse(message = "Job or page is missing.", code = 404) + }) + @RequestMapping(value = "/{jobId}/raw", method = RequestMethod.GET) + void raw(@ApiParam(name="jobId", value="Job ID of submitted job", required=true)@PathVariable String jobId, + @ApiParam(name="page", value="Page number", required=true)@RequestParam Integer page, + @RequestParam(defaultValue = "", required = false) String fileName, + final HttpServletRequest request, final HttpServletResponse response) throws RestException { + try (InputStream inputStream = pcapQueryService.getRawPcap(SecurityUtils.getCurrentUser(), jobId, page); + OutputStream output = response.getOutputStream()) { + response.reset(); + if (inputStream == null) { + response.setStatus(HttpStatus.NOT_FOUND.value()); + } else { + response.setContentType("application/octet-stream"); + if (fileName.isEmpty()) { + fileName = String.format(PCAP_FILENAME_FORMAT, jobId, page); + } + response.setHeader("Content-Disposition", "attachment; filename=\"" + fileName + "\""); + int size = IOUtils.copy(inputStream, output); + response.setContentLength(size); + output.flush(); + } + } catch (IOException e) { + throw new RestException(e); + } + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java index 9421ce3..f84735d 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java @@ -20,9 +20,10 @@ package org.apache.metron.rest.service; import org.apache.hadoop.fs.Path; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.pcap.FixedPcapRequest; + +import java.io.InputStream; import org.apache.metron.rest.model.pcap.PcapStatus; import org.apache.metron.rest.model.pcap.Pdml; -import org.apache.metron.rest.model.pcap.Pdml; public interface PcapService { @@ -35,4 +36,6 @@ public interface PcapService { Path getPath(String username, String jobId, Integer page) throws RestException; Pdml getPdml(String username, String jobId, Integer page) throws RestException; + + InputStream getRawPcap(String username, String jobId, Integer page) throws RestException; } http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java index 7894b1a..e341184 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java @@ -18,6 +18,7 @@ package org.apache.metron.rest.service.impl; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -42,6 +43,8 @@ import org.springframework.stereotype.Service; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; @Service public class PcapServiceImpl implements PcapService { @@ -146,6 +149,20 @@ public class PcapServiceImpl implements PcapService { return pdml; } + public InputStream getRawPcap(String username, String jobId, Integer page) throws RestException { + InputStream inputStream = null; + Path path = getPath(username, jobId, page); + try { + FileSystem fileSystem = getFileSystem(); + if (path!= null && fileSystem.exists(path)) { + inputStream = fileSystem.open(path); + } + } catch (IOException e) { + throw new RestException(e); + } + return inputStream; + } + protected void setPcapOptions(String username, PcapRequest pcapRequest) throws IOException { PcapOptions.JOB_NAME.put(pcapRequest, "jobName"); PcapOptions.USERNAME.put(pcapRequest, username); http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java index 2fa64cd..6189d2c 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java @@ -25,21 +25,28 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import java.io.File; import java.util.Arrays; import java.util.List; import java.util.Map; import org.adrianwalker.multilinestring.Multiline; import org.apache.hadoop.fs.Path; +import org.apache.commons.io.FileUtils; import org.apache.metron.common.Constants; import org.apache.metron.job.JobStatus; import org.apache.metron.job.Pageable; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.job.JobStatus; +import org.apache.metron.job.Pageable; import org.apache.metron.pcap.PcapHelper; import org.apache.metron.pcap.PcapPages; import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; import org.apache.metron.rest.mock.MockPcapJob; +import org.apache.metron.rest.mock.MockPcapToPdmlScriptWrapper; import org.apache.metron.rest.model.PcapResponse; import org.apache.metron.rest.service.PcapService; import org.junit.Assert; @@ -301,5 +308,42 @@ public class PcapControllerIntegrationTest { .andExpect(status().isNotFound()); } + @Test + public void testRawDownload() throws Exception { + String pcapFileContents = "pcap file contents"; + FileUtils.write(new File("./target/pcapFile"), pcapFileContents, "UTF8"); + + MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); + + mockPcapJob.setStatus(new JobStatus().withJobId("jobId").withState(JobStatus.State.RUNNING)); + + this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.jobId").value("jobId")) + .andExpect(jsonPath("$.jobStatus").value("RUNNING")); + + Pageable<Path> pageable = new PcapPages(Arrays.asList(new Path("./target/pcapFile"))); + mockPcapJob.setIsDone(true); + mockPcapJob.setPageable(pageable); + + this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=1").with(httpBasic(user, password))) + .andExpect(status().isOk()) + .andExpect(header().string("Content-Disposition", "attachment; filename=\"pcap_jobId_1.pcap\"")) + .andExpect(header().string("Content-Length", Integer.toString(pcapFileContents.length()))) + .andExpect(content().contentType(MediaType.parseMediaType("application/octet-stream"))) + .andExpect(content().bytes(pcapFileContents.getBytes())); + + this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=1&fileName=pcapFile.pcap").with(httpBasic(user, password))) + .andExpect(status().isOk()) + .andExpect(header().string("Content-Disposition", "attachment; filename=\"pcapFile.pcap\"")) + .andExpect(header().string("Content-Length", Integer.toString(pcapFileContents.length()))) + .andExpect(content().contentType(MediaType.parseMediaType("application/octet-stream"))) + .andExpect(content().bytes(pcapFileContents.getBytes())); + + this.mockMvc.perform(get(pcapUrl + "/jobId/raw?page=2").with(httpBasic(user, password))) + .andExpect(status().isNotFound()); + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java index d818c77..3c6d506 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java @@ -19,6 +19,7 @@ package org.apache.metron.rest.service.impl; import org.adrianwalker.multilinestring.Multiline; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.metron.common.Constants; @@ -471,4 +472,58 @@ public class PcapServiceImplTest { pcapService.getPdml("user", "jobId", 1); } + @Test + public void getRawShouldProperlyReturnInputStream() throws Exception { + FSDataInputStream inputStream = mock(FSDataInputStream.class); + Path path = new Path("./target"); + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), new PcapToPdmlScriptWrapper())); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + when(fileSystem.exists(path)).thenReturn(true); + doReturn(path).when(pcapService).getPath("user", "jobId", 1); + when(fileSystem.open(path)).thenReturn(inputStream); + + Assert.assertEquals(inputStream, pcapService.getRawPcap("user", "jobId", 1)); + } + + @Test + public void getRawShouldReturnNullOnInvalidPage() throws Exception { + Path path = new Path("/some/path"); + + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper)); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + + assertNull(pcapService.getRawPcap("user", "jobId", 1)); + } + + @Test + public void getRawShouldReturnNullOnNonexistentPath() throws Exception { + Path path = new Path("/some/path"); + + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper)); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + when(fileSystem.exists(path)).thenReturn(false); + doReturn(path).when(pcapService).getPath("user", "jobId", 1); + + assertNull(pcapService.getRawPcap("user", "jobId", 1)); + } + + @Test + public void getRawShouldThrowException() throws Exception { + exception.expect(RestException.class); + exception.expectMessage("some exception"); + + Path path = new Path("./target"); + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, new PcapJobSupplier(), new InMemoryJobManager<>(), pcapToPdmlScriptWrapper)); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + when(fileSystem.exists(path)).thenReturn(true); + doReturn(path).when(pcapService).getPath("user", "jobId", 1); + when(fileSystem.open(path)).thenThrow(new IOException("some exception")); + + pcapService.getRawPcap("user", "jobId", 1); + } + }