METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes apache/metron#1125
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d0e87576 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d0e87576 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d0e87576 Branch: refs/heads/master Commit: d0e87576944b6370eb1d9a0c98d0450f2c51f052 Parents: e78bc53 Author: merrimanr <merrim...@gmail.com> Authored: Mon Jul 23 09:09:37 2018 -0500 Committer: rmerriman <merrim...@gmail.com> Committed: Mon Jul 23 09:09:37 2018 -0500 ---------------------------------------------------------------------- .../rest/model/pcap/FixedPcapRequest.java | 1 + .../metron/rest/model/pcap/PcapRequest.java | 4 +- .../rest/model/pcap/QueryPcapOptions.java | 35 +++++++++ .../rest/model/pcap/QueryPcapRequest.java | 59 ++++++++++++++ metron-interface/metron-rest/README.md | 10 ++- .../metron/rest/controller/PcapController.java | 15 +++- .../apache/metron/rest/service/PcapService.java | 8 +- .../rest/service/impl/PcapServiceImpl.java | 13 ++-- .../PcapControllerIntegrationTest.java | 81 +++++++++++++------- .../apache/metron/rest/mock/MockPcapJob.java | 13 ++-- .../rest/service/impl/PcapServiceImplTest.java | 55 +++++++++++-- 11 files changed, 240 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java index a2d345b..d91aac7 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java @@ -87,6 +87,7 @@ public class FixedPcapRequest extends PcapRequest { FixedPcapOptions.INCLUDE_REVERSE.put(this, includeReverse); } + @Override public void setFields() { Map<String, String> fields = new HashMap<>(); if (getIpSrcAddr() != null) { http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java index 64ed932..44721fa 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java @@ -22,7 +22,7 @@ import org.apache.metron.pcap.config.PcapOptions; import java.util.HashMap; -public class PcapRequest extends AbstractMapDecorator<String, Object> { +public abstract class PcapRequest extends AbstractMapDecorator<String, Object> { public PcapRequest() { super(new HashMap<>()); @@ -78,4 +78,6 @@ public class PcapRequest extends AbstractMapDecorator<String, Object> { public void setNumReducers(Integer numReducers) { PcapOptions.NUM_REDUCERS.put(this, numReducers); } + + public abstract void setFields(); } http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java new file mode 100644 index 0000000..1ae3da6 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model.pcap; + +import org.apache.metron.common.configuration.ConfigOption; + +public enum QueryPcapOptions implements ConfigOption { + QUERY("query"); + + String key; + + QueryPcapOptions(String key) { + this.key = key; + } + + @Override + public String getKey() { + return key; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java new file mode 100644 index 0000000..0da3e69 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapRequest.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model.pcap; + +import org.apache.metron.pcap.config.PcapOptions; +import org.apache.metron.pcap.filter.query.QueryPcapFilter; + +public class QueryPcapRequest extends PcapRequest { + + public QueryPcapRequest() { + PcapOptions.FILTER_IMPL.put(this, new QueryPcapFilter.Configurator()); + } + + public String getQuery() { + return QueryPcapOptions.QUERY.get(this, String.class); + } + + public void setQuery(String query) { + QueryPcapOptions.QUERY.put(this, query); + } + + @Override + public void setFields() { + PcapOptions.FIELDS.put(this, getQuery()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + QueryPcapRequest queryPcapRequest = (QueryPcapRequest) o; + + return (super.equals(o)) && + (getQuery() != null ? getQuery().equals(queryPcapRequest.getQuery()) : queryPcapRequest.getQuery() != null); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (getQuery() != null ? getQuery().hashCode() : 0); + return result; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 4a7102f..53f5978 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -254,6 +254,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)| | [ `GET /api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)| | [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)| +| [ `GET /api/v1/pcap/query`](#get-apiv1pcapquery)| | [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)| | [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)| | [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)| @@ -502,12 +503,19 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns 'true' if the status changed and 'false' if it did not. ### `POST /api/v1/pcap/fixed` - * Description: Executes a Fixed Pcap Query. + * Description: Executes a Fixed Filter Pcap Query. * Input: * fixedPcapRequest - A Fixed Pcap Request which includes fixed filter fields like ip source address and protocol * Returns: * 200 - Returns a job status with job ID. +### `POST /api/v1/pcap/query` + * Description: Executes a Query Filter Pcap Query. + * Input: + * queryPcapRequest - A Query Pcap Request which includes Stellar query field + * Returns: + * 200 - Returns a job status with job ID. + ### `POST /api/v1/pcap/{jobId}` * Description: Gets job status for Pcap query job. * Input: http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java index 23bb0b9..be95718 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java @@ -22,11 +22,11 @@ import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.pcap.FixedPcapRequest; import org.apache.metron.rest.model.pcap.PcapStatus; import org.apache.metron.rest.model.pcap.Pdml; +import org.apache.metron.rest.model.pcap.QueryPcapRequest; import org.apache.metron.rest.security.SecurityUtils; import org.apache.metron.rest.service.PcapService; import org.springframework.beans.factory.annotation.Autowired; @@ -54,12 +54,21 @@ public class PcapController { @Autowired private PcapService pcapQueryService; - @ApiOperation(value = "Executes a Fixed Pcap Query.") + @ApiOperation(value = "Executes a Fixed Filter Pcap Query.") @ApiResponses(value = { @ApiResponse(message = "Returns a job status with job ID.", code = 200)}) @RequestMapping(value = "/fixed", method = RequestMethod.POST) ResponseEntity<PcapStatus> fixed(@ApiParam(name="fixedPcapRequest", value="A Fixed Pcap Request" + " which includes fixed filter fields like ip source address and protocol", required=true)@RequestBody FixedPcapRequest fixedPcapRequest) throws RestException { - PcapStatus pcapStatus = pcapQueryService.fixed(SecurityUtils.getCurrentUser(), fixedPcapRequest); + PcapStatus pcapStatus = pcapQueryService.submit(SecurityUtils.getCurrentUser(), fixedPcapRequest); + return new ResponseEntity<>(pcapStatus, HttpStatus.OK); + } + + @ApiOperation(value = "Executes a Query Filter Pcap Query.") + @ApiResponses(value = { @ApiResponse(message = "Returns a job status with job ID.", code = 200)}) + @RequestMapping(value = "/query", method = RequestMethod.POST) + ResponseEntity<PcapStatus> query(@ApiParam(name="queryPcapRequest", value="A Query Pcap Request" + + " which includes Stellar query field", required=true)@RequestBody QueryPcapRequest queryPcapRequest) throws RestException { + PcapStatus pcapStatus = pcapQueryService.submit(SecurityUtils.getCurrentUser(), queryPcapRequest); return new ResponseEntity<>(pcapStatus, HttpStatus.OK); } http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java index f84735d..2ae34a3 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java @@ -19,15 +19,15 @@ package org.apache.metron.rest.service; import org.apache.hadoop.fs.Path; import org.apache.metron.rest.RestException; -import org.apache.metron.rest.model.pcap.FixedPcapRequest; - -import java.io.InputStream; +import org.apache.metron.rest.model.pcap.PcapRequest; import org.apache.metron.rest.model.pcap.PcapStatus; import org.apache.metron.rest.model.pcap.Pdml; +import java.io.InputStream; + public interface PcapService { - PcapStatus fixed(String username, FixedPcapRequest fixedPcapRequest) throws RestException; + PcapStatus submit(String username, PcapRequest pcapRequest) throws RestException; PcapStatus getJobStatus(String username, String jobId) throws RestException; http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java index e341184..77a8934 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java @@ -18,7 +18,6 @@ package org.apache.metron.rest.service.impl; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -32,7 +31,6 @@ import org.apache.metron.pcap.config.PcapOptions; import org.apache.metron.rest.MetronRestConstants; import org.apache.metron.rest.RestException; import org.apache.metron.rest.config.PcapJobSupplier; -import org.apache.metron.rest.model.pcap.FixedPcapRequest; import org.apache.metron.rest.model.pcap.PcapRequest; import org.apache.metron.rest.model.pcap.PcapStatus; import org.apache.metron.rest.model.pcap.Pdml; @@ -43,8 +41,6 @@ import org.springframework.stereotype.Service; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; @Service public class PcapServiceImpl implements PcapService { @@ -65,11 +61,11 @@ public class PcapServiceImpl implements PcapService { } @Override - public PcapStatus fixed(String username, FixedPcapRequest fixedPcapRequest) throws RestException { + public PcapStatus submit(String username, PcapRequest pcapRequest) throws RestException { try { - setPcapOptions(username, fixedPcapRequest); - fixedPcapRequest.setFields(); - pcapJobSupplier.setPcapRequest(fixedPcapRequest); + setPcapOptions(username, pcapRequest); + pcapRequest.setFields(); + pcapJobSupplier.setPcapRequest(pcapRequest); JobStatus jobStatus = jobManager.submit(pcapJobSupplier, username); return jobStatusToPcapStatus(jobStatus); } catch (IOException | JobException e) { @@ -149,6 +145,7 @@ public class PcapServiceImpl implements PcapService { return pdml; } + @Override public InputStream getRawPcap(String username, String jobId, Integer page) throws RestException { InputStream inputStream = null; Path path = getPath(username, jobId, page); http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java index 6189d2c..25956e4 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java @@ -17,37 +17,17 @@ */ package org.apache.metron.rest.controller; -import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; -import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import java.io.File; -import java.util.Arrays; -import java.util.List; -import java.util.Map; import org.adrianwalker.multilinestring.Multiline; -import org.apache.hadoop.fs.Path; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.Path; import org.apache.metron.common.Constants; import org.apache.metron.job.JobStatus; import org.apache.metron.job.Pageable; -import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.job.JobStatus; -import org.apache.metron.job.Pageable; import org.apache.metron.pcap.PcapHelper; import org.apache.metron.pcap.PcapPages; import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; +import org.apache.metron.pcap.filter.query.QueryPcapFilter; import org.apache.metron.rest.mock.MockPcapJob; -import org.apache.metron.rest.mock.MockPcapToPdmlScriptWrapper; -import org.apache.metron.rest.model.PcapResponse; import org.apache.metron.rest.service.PcapService; import org.junit.Assert; import org.junit.Before; @@ -62,6 +42,22 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; +import java.io.File; +import java.util.Arrays; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(TEST_PROFILE) @@ -101,6 +97,20 @@ public class PcapControllerIntegrationTest { @Multiline public static String fixedWithDefaultsJson; + /** + { + "basePath": "/base/path", + "baseInterimResultPath": "/base/interim/result/path", + "finalOutputPath": "/final/output/path", + "startTimeMs": 10, + "endTimeMs": 20, + "numReducers": 2, + "query": "query" + } + */ + @Multiline + public static String queryJson; + @Autowired private PcapService pcapService; @@ -123,17 +133,16 @@ public class PcapControllerIntegrationTest { public void testSecurity() throws Exception { this.mockMvc.perform(post(pcapUrl + "/fixed").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(post(pcapUrl + "/query").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(queryJson)) + .andExpect(status().isUnauthorized()); } @Test public void testFixedRequest() throws Exception { MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); - List<byte[]> results = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes()); - mockPcapJob.setResults(results); mockPcapJob.setStatus(new JobStatus().withState(JobStatus.State.RUNNING)); - PcapResponse expectedReponse = new PcapResponse(); - expectedReponse.setPcaps(results); this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) @@ -187,6 +196,26 @@ public class PcapControllerIntegrationTest { } @Test + public void testQueryRequest() throws Exception { + MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); + mockPcapJob.setStatus(new JobStatus().withState(JobStatus.State.RUNNING)); + + this.mockMvc.perform(post(pcapUrl + "/query").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(queryJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.jobStatus").value("RUNNING")); + + Assert.assertEquals("/base/path", mockPcapJob.getBasePath()); + Assert.assertEquals("/base/interim/result/path", mockPcapJob.getBaseInterrimResultPath()); + Assert.assertEquals("/final/output/path", mockPcapJob.getFinalOutputPath()); + Assert.assertEquals(10000000, mockPcapJob.getStartTimeNs()); + Assert.assertEquals(20000000, mockPcapJob.getEndTimeNs()); + Assert.assertEquals(2, mockPcapJob.getNumReducers()); + Assert.assertTrue(mockPcapJob.getFilterImpl() instanceof QueryPcapFilter.Configurator); + Assert.assertEquals("query", mockPcapJob.getQuery()); + } + + @Test public void testGetStatus() throws Exception { MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob"); http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java index df65635..6a954e8 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java @@ -45,11 +45,10 @@ public class MockPcapJob extends PcapJob<Path> { private Map<String, String> fixedFields; private PcapFilterConfigurator filterImpl; private int recPerFile; - private SequenceFileIterable sequenceFileIterable; + private String query; private Statusable<Path> statusable; public MockPcapJob() { - sequenceFileIterable = mock(SequenceFileIterable.class); statusable = mock(Statusable.class); } @@ -64,6 +63,8 @@ public class MockPcapJob extends PcapJob<Path> { Object fields = PcapOptions.FIELDS.get(configuration, Object.class); if (fields instanceof Map) { this.fixedFields = (Map<String, String>) fields; + } else { + this.query = (String) fields; } this.filterImpl = PcapOptions.FILTER_IMPL.get(configuration, PcapFilterConfigurator.class); this.recPerFile = PcapOptions.NUM_RECORDS_PER_FILE.get(configuration, Integer.class); @@ -92,10 +93,6 @@ public class MockPcapJob extends PcapJob<Path> { when(statusable.isDone()).thenReturn(isDone); } - public void setResults(List<byte[]> pcaps) { - when(sequenceFileIterable.iterator()).thenReturn(pcaps.iterator()); - } - public String getBasePath() { return basePath; } @@ -136,6 +133,10 @@ public class MockPcapJob extends PcapJob<Path> { return fixedFields; } + public String getQuery() { + return query; + } + public PcapFilterConfigurator getFilterImpl() { return filterImpl; } http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java index 3c6d506..e9f8f9f 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java @@ -32,12 +32,14 @@ import org.apache.metron.job.manager.InMemoryJobManager; import org.apache.metron.job.manager.JobManager; import org.apache.metron.pcap.PcapHelper; import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; +import org.apache.metron.pcap.filter.query.QueryPcapFilter; import org.apache.metron.rest.MetronRestConstants; import org.apache.metron.rest.RestException; import org.apache.metron.rest.config.PcapJobSupplier; import org.apache.metron.rest.mock.MockPcapJob; import org.apache.metron.rest.mock.MockPcapJobSupplier; import org.apache.metron.rest.model.pcap.FixedPcapRequest; +import org.apache.metron.rest.model.pcap.QueryPcapRequest; import org.apache.metron.rest.model.pcap.PcapStatus; import org.apache.metron.rest.model.pcap.Pdml; import org.hamcrest.CoreMatchers; @@ -190,7 +192,7 @@ public class PcapServiceImplTest { } @Test - public void fixedShouldProperlyCallPcapJobQuery() throws Exception { + public void submitShouldProperlySubmitFixedPcapRequest() throws Exception { FixedPcapRequest fixedPcapRequest = new FixedPcapRequest(); fixedPcapRequest.setBasePath("basePath"); fixedPcapRequest.setBaseInterimResultPath("baseOutputPath"); @@ -232,7 +234,7 @@ public class PcapServiceImplTest { expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name()); expectedPcapStatus.setDescription("description"); - Assert.assertEquals(expectedPcapStatus, pcapService.fixed("user", fixedPcapRequest)); + Assert.assertEquals(expectedPcapStatus, pcapService.submit("user", fixedPcapRequest)); Assert.assertEquals(expectedPcapStatus, pcapService.jobStatusToPcapStatus(jobManager.getJob("user", "jobId").getStatus())); Assert.assertEquals("basePath", mockPcapJob.getBasePath()); Assert.assertEquals("baseOutputPath", mockPcapJob.getBaseInterrimResultPath()); @@ -253,7 +255,7 @@ public class PcapServiceImplTest { } @Test - public void fixedShouldProperlyCallPcapJobQueryWithDefaults() throws Exception { + public void submitShouldProperlySubmitWithDefaults() throws Exception { long beforeJobTime = System.currentTimeMillis(); FixedPcapRequest fixedPcapRequest = new FixedPcapRequest(); @@ -275,7 +277,7 @@ public class PcapServiceImplTest { expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name()); expectedPcapStatus.setDescription("description"); - Assert.assertEquals(expectedPcapStatus, pcapService.fixed("user", fixedPcapRequest)); + Assert.assertEquals(expectedPcapStatus, pcapService.submit("user", fixedPcapRequest)); Assert.assertEquals("/base/path", mockPcapJob.getBasePath()); Assert.assertEquals("/base/interim/result/path", mockPcapJob.getBaseInterrimResultPath()); Assert.assertEquals("/final/output/path", mockPcapJob.getFinalOutputPath()); @@ -289,6 +291,49 @@ public class PcapServiceImplTest { } @Test + public void submitShouldProperlySubmitQueryPcapRequest() throws Exception { + QueryPcapRequest queryPcapRequest = new QueryPcapRequest(); + queryPcapRequest.setBasePath("basePath"); + queryPcapRequest.setBaseInterimResultPath("baseOutputPath"); + queryPcapRequest.setFinalOutputPath("finalOutputPath"); + queryPcapRequest.setStartTimeMs(1L); + queryPcapRequest.setEndTimeMs(2L); + queryPcapRequest.setNumReducers(2); + queryPcapRequest.setQuery("query"); + MockPcapJob mockPcapJob = new MockPcapJob(); + mockPcapJobSupplier.setMockPcapJob(mockPcapJob); + JobManager jobManager = new InMemoryJobManager<>(); + + PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, mockPcapJobSupplier, jobManager, pcapToPdmlScriptWrapper)); + FileSystem fileSystem = mock(FileSystem.class); + doReturn(fileSystem).when(pcapService).getFileSystem(); + mockPcapJob.setStatus(new JobStatus() + .withJobId("jobId") + .withDescription("description") + .withPercentComplete(0L) + .withState(JobStatus.State.RUNNING)); + + String expectedFields = "query"; + PcapStatus expectedPcapStatus = new PcapStatus(); + expectedPcapStatus.setJobId("jobId"); + expectedPcapStatus.setJobStatus(JobStatus.State.RUNNING.name()); + expectedPcapStatus.setDescription("description"); + + Assert.assertEquals(expectedPcapStatus, pcapService.submit("user", queryPcapRequest)); + Assert.assertEquals(expectedPcapStatus, pcapService.jobStatusToPcapStatus(jobManager.getJob("user", "jobId").getStatus())); + Assert.assertEquals("basePath", mockPcapJob.getBasePath()); + Assert.assertEquals("baseOutputPath", mockPcapJob.getBaseInterrimResultPath()); + Assert.assertEquals("finalOutputPath", mockPcapJob.getFinalOutputPath()); + Assert.assertEquals(1000000, mockPcapJob.getStartTimeNs()); + Assert.assertEquals(2000000, mockPcapJob.getEndTimeNs()); + Assert.assertEquals(2, mockPcapJob.getNumReducers()); + Assert.assertEquals(100, mockPcapJob.getRecPerFile()); + Assert.assertTrue(mockPcapJob.getFilterImpl() instanceof QueryPcapFilter.Configurator); + Map<String, String> actualFixedFields = mockPcapJob.getFixedFields(); + Assert.assertEquals("query", mockPcapJob.getQuery()); + } + + @Test public void fixedShouldThrowRestException() throws Exception { exception.expect(RestException.class); exception.expectMessage("some job exception"); @@ -301,7 +346,7 @@ public class PcapServiceImplTest { doReturn(fileSystem).when(pcapService).getFileSystem(); when(jobManager.submit(pcapJobSupplier, "user")).thenThrow(new JobException("some job exception")); - pcapService.fixed("user", fixedPcapRequest); + pcapService.submit("user", fixedPcapRequest); } @Test
