http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java index e9bb510..dbf4aa4 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java @@ -17,6 +17,16 @@ */ package org.apache.metron.rest.config; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; import kafka.admin.AdminUtils$; import kafka.utils.ZKStringSerializer$; import kafka.utils.ZkUtils; @@ -42,14 +52,6 @@ import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.web.client.RestTemplate; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; @Configuration @Profile(TEST_PROFILE) @@ -76,7 +78,7 @@ public class TestConfig { } @Bean(destroyMethod = "stop") - public ComponentRunner componentRunner(ZKServerComponent zkServerComponent, KafkaComponent kafkaWithZKComponent) { + public ComponentRunner componentRunner(ZKServerComponent zkServerComponent, KafkaComponent kafkaWithZKComponent) throws IOException { ComponentRunner runner = new ComponentRunner.Builder() .withComponent("zk", zkServerComponent) .withCustomShutdownOrder(new String[]{"search", "zk"}) @@ -84,12 +86,19 @@ public class TestConfig { try { runner.start(); File globalConfigFile = new File("src/test/resources/zookeeper/global.json"); + File bundlePropertiesFile = new File("src/test/resources/zookeeper/bundle.properties"); try(BufferedReader r = new BufferedReader(new FileReader(globalConfigFile))){ String globalConfig = IOUtils.toString(r); ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(), zkServerComponent.getConnectionString()); } catch (Exception e) { throw new IllegalStateException("Unable to upload global config", e); } + try(BufferedReader r = new BufferedReader(new FileReader(bundlePropertiesFile))){ + String bundleProperties = IOUtils.toString(r); + ConfigurationsUtils.writeGlobalBundlePropertiesToZookeeper(bundleProperties.getBytes(), zkServerComponent.getConnectionString()); + }catch(Exception e) { + throw new IllegalStateException("Unable to upload bundle properties"); + } } catch (UnableToStartException e) { e.printStackTrace(); }
http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java index 8888eb0..f12b415 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java @@ -131,8 +131,8 @@ public class GrokControllerIntegrationTest { .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$").isNotEmpty()); - String statement = FileUtils.readFileToString(new File("../../metron-platform/metron-parsers/src/main/resources/patterns/squid")); - this.mockMvc.perform(get(grokUrl + "/get/statement?path=/patterns/squid").with(httpBasic(user,password))) + String statement = FileUtils.readFileToString(new File("../../metron-platform/metron-parsers/src/main/resources/patterns/common")); + this.mockMvc.perform(get(grokUrl + "/get/statement?path=/patterns/common").with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))) .andExpect(content().bytes(statement.getBytes())); http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java index 9e6d408..5c00cd2 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java @@ -101,8 +101,8 @@ public class KafkaControllerIntegrationTest { class SampleDataRunner implements Runnable { - private boolean stop = false; - private String path = "../../metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput"; + private boolean stop = false; + private String path = "../../metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/raw/test.raw"; @Override public void run() { http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/ParserExtensionControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/ParserExtensionControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/ParserExtensionControllerIntegrationTest.java new file mode 100644 index 0000000..49721ca --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/ParserExtensionControllerIntegrationTest.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.service.HdfsService; +import org.apache.metron.test.utils.ResourceCopier; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.hamcrest.Matchers.notNullValue; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class ParserExtensionControllerIntegrationTest { + @Autowired + private TestRestTemplate restTemplate; + @Autowired + private HdfsService hdfsService; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String parserExtUrl = "/api/v1/ext/parsers"; + private String user = "user"; + private String password = "password"; + private String extPath = "./target/remote/extension_contrib_lib/"; + private String fileContents = "file contents"; + + @BeforeClass + public static void beforeClass() throws Exception{ + ResourceCopier.copyResources(Paths.get("./src/test/resources"), Paths.get( "./target/remote"), false); + } + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @After + public void takeDown() throws Exception { + + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(parserExtUrl).with(csrf()).contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")).content(fileContents)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(parserExtUrl)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(delete(parserExtUrl).with(csrf())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + final File bundle = new File("./target/remote/metron-parser-test-assembly-0.4.0-archive.tar.gz"); + final MockMultipartFile multipartFile = new MockMultipartFile("extensionTgz","metron-parser-test-assembly-0.4.0-archive.tar.gz","", new FileInputStream(bundle)); + + HashMap<String, String> contentTypeParams = new HashMap<String, String>(); + contentTypeParams.put("boundary", "265001916915724"); + MediaType mediaType = new MediaType("multipart", "form-data", contentTypeParams); + + // INSTALL ASYNC + MvcResult result = this.mockMvc.perform(MockMvcRequestBuilders.fileUpload(parserExtUrl).file(multipartFile).with(httpBasic(user, password)).contentType(mediaType)) + .andReturn(); + + this.mockMvc.perform(asyncDispatch(result)) + .andExpect(status().isCreated()); + + // INSTALL ASYNC AGAIN AND FAIL + result = this.mockMvc.perform(MockMvcRequestBuilders.fileUpload(parserExtUrl).file(multipartFile).with(httpBasic(user, password)).contentType(mediaType)) + .andReturn(); + + this.mockMvc.perform(asyncDispatch(result)) + .andExpect(status().isForbidden()); + + // GET ONE + this.mockMvc.perform(get(parserExtUrl + "/metron-parser-test-assembly-0_4_0").with(httpBasic(user, password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.extensionAssemblyName").value("metron-parser-test-assembly-0_4_0")) + .andExpect(jsonPath("$.extensionBundleName").value("metron-parser-test-bundle-0.4.0.bundle")) + .andExpect(jsonPath("$.extensionsBundleID").value("metron-parser-test-bundle")) + .andExpect(jsonPath("$.extensionsBundleVersion").value("0.4.0")) + .andExpect(jsonPath("$.parserExtensionParserNames[0]").value("test")); + + // GET ALL + this.mockMvc.perform(get(parserExtUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.metron-parser-test-assembly-0_4_0.extensionAssemblyName == 'metron-parser-test-assembly-0_4_0' && " + + "@.metron-parser-test-assembly-0_4_0.extensionBundleName == 'metron-parser-test-bundle-0.4.0.bundle' && " + + "@.metron-parser-test-assembly-0_4_0.extensionsBundleID == 'metron-parser-test-bundle' && " + + "@.metron-parser-test-assembly-0_4_0.extensionsBundleVersion == '0.4.0' && " + + "@.metron-parser-test-assembly-0_4_0.parserExtensionParserNames[0] == 'test')]").exists()); + + // DELETE ASYNC + result = this.mockMvc.perform(delete(parserExtUrl + "/metron-parser-test-assembly-0_4_0").with(httpBasic(user, password))).andReturn(); + this.mockMvc.perform(asyncDispatch(result)) + .andExpect(status().isOk()); + + // GET ONE + this.mockMvc.perform(get(parserExtUrl + "/metron-parser-test-assembly-0_4_0").with(httpBasic(user, password))) + .andExpect(status().isNotFound()); + + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java index dd4eff7..57a9624 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java @@ -146,8 +146,7 @@ public class SensorEnrichmentConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(content().bytes("{}".getBytes())); + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))); this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isCreated()) http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java index cebcde6..4534781 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java @@ -94,8 +94,7 @@ public class SensorIndexingConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(content().bytes("{}".getBytes())); + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))); this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isCreated()) http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java index f5ea23d..ffe6b9a 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java @@ -17,11 +17,15 @@ */ package org.apache.metron.rest.controller; +import java.io.FileInputStream; +import java.util.HashMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.io.FileUtils; +import org.apache.metron.bundles.BundleSystem; +import org.apache.metron.bundles.util.BundleProperties; import org.apache.metron.rest.MetronRestConstants; -import org.apache.metron.rest.service.GrokService; import org.apache.metron.rest.service.SensorParserConfigService; +import org.apache.metron.rest.service.impl.SensorParserConfigServiceImpl; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,6 +44,8 @@ import java.io.IOException; import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; @@ -57,7 +63,7 @@ public class SensorParserConfigControllerIntegrationTest { /** { - "parserClassName": "org.apache.metron.parsers.GrokParser", + "parserClassName": "org.apache.metron.parsers.grok.GrokParser", "sensorTopic": "squidTest", "parserConfig": { "patternLabel": "SQUIDTEST", @@ -81,21 +87,21 @@ public class SensorParserConfigControllerIntegrationTest { /** { - "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", - "sensorTopic":"broTest", + "parserClassName":"org.apache.metron.parsers.json.JSONMapParser", + "sensorTopic":"jsonTest", "parserConfig": {}, "readMetadata": true, "mergeMetadata": true } */ @Multiline - public static String broJson; + public static String jsonMapJson; /** { "sensorParserConfig": { - "parserClassName": "org.apache.metron.parsers.GrokParser", + "parserClassName": "org.apache.metron.parsers.grok.GrokParser", "sensorTopic": "squidTest", "parserConfig": { "patternLabel": "SQUID_DELIMITED", @@ -165,6 +171,8 @@ public class SensorParserConfigControllerIntegrationTest { @Autowired private WebApplicationContext wac; + BundleSystem bundleSystem; + private MockMvc mockMvc; private String sensorParserConfigUrl = "/api/v1/sensor/parser/config"; @@ -174,6 +182,14 @@ public class SensorParserConfigControllerIntegrationTest { @Before public void setup() throws Exception { this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + BundleSystem.reset(); + try(FileInputStream fis = new FileInputStream(new File("src/test/resources/zookeeper/bundle.properties"))) { + BundleProperties properties = BundleProperties.createBasicBundleProperties(fis, new HashMap<>()); + properties.setProperty(BundleProperties.BUNDLE_LIBRARY_DIRECTORY,"./target"); + properties.unSetProperty("bundle.library.directory.alt"); + bundleSystem = new BundleSystem.Builder().withBundleProperties(properties).build(); + ((SensorParserConfigServiceImpl)sensorParserConfigService).setBundleSystem(bundleSystem); + } } @Test @@ -194,14 +210,14 @@ public class SensorParserConfigControllerIntegrationTest { @Test public void test() throws Exception { cleanFileSystem(); - this.sensorParserConfigService.delete("broTest"); + this.sensorParserConfigService.delete("jsonTest"); this.sensorParserConfigService.delete("squidTest"); this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(10))) - .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser")) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.grok.GrokParser")) .andExpect(jsonPath("$.sensorTopic").value("squidTest")) .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest")) .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST")) @@ -216,7 +232,7 @@ public class SensorParserConfigControllerIntegrationTest { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(10))) - .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser")) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.grok.GrokParser")) .andExpect(jsonPath("$.sensorTopic").value("squidTest")) .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest")) .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST")) @@ -230,7 +246,7 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" + + .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.grok.GrokParser' &&" + "@.sensorTopic == 'squidTest' &&" + "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" + "@.parserConfig.patternLabel == 'SQUIDTEST' &&" + @@ -241,22 +257,22 @@ public class SensorParserConfigControllerIntegrationTest { "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" + "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists()); - this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(jsonMapJson)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(10))) - .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser")) - .andExpect(jsonPath("$.sensorTopic").value("broTest")) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.json.JSONMapParser")) + .andExpect(jsonPath("$.sensorTopic").value("jsonTest")) .andExpect(jsonPath("$.readMetadata").value("true")) .andExpect(jsonPath("$.mergeMetadata").value("true")) .andExpect(jsonPath("$.parserConfig").isEmpty()); - this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(jsonMapJson)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.*", hasSize(10))) - .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser")) - .andExpect(jsonPath("$.sensorTopic").value("broTest")) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.json.JSONMapParser")) + .andExpect(jsonPath("$.sensorTopic").value("jsonTest")) .andExpect(jsonPath("$.readMetadata").value("true")) .andExpect(jsonPath("$.mergeMetadata").value("true")) .andExpect(jsonPath("$.parserConfig").isEmpty()); @@ -264,7 +280,7 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" + + .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.grok.GrokParser' &&" + "@.sensorTopic == 'squidTest' &&" + "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" + "@.parserConfig.patternLabel == 'SQUIDTEST' &&" + @@ -274,8 +290,8 @@ public class SensorParserConfigControllerIntegrationTest { "@.fieldTransformations[0].output[1] == 'domain_without_subdomains' &&" + "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" + "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists()) - .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.bro.BasicBroParser' && " + - "@.sensorTopic == 'broTest')]").exists()); + .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.json.JSONMapParser' && " + + "@.sensorTopic == 'jsonTest')]").exists()); this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)).with(csrf())) .andExpect(status().isOk()); @@ -290,31 +306,31 @@ public class SensorParserConfigControllerIntegrationTest { .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist()) - .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").exists()); + .andExpect(jsonPath("$[?(@.sensorTopic == 'jsonTest')]").exists()); - this.mockMvc.perform(delete(sensorParserConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + this.mockMvc.perform(delete(sensorParserConfigUrl + "/jsonTest").with(httpBasic(user,password)).with(csrf())) .andExpect(status().isOk()); - this.mockMvc.perform(delete(sensorParserConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + this.mockMvc.perform(delete(sensorParserConfigUrl + "/jsonTest").with(httpBasic(user,password)).with(csrf())) .andExpect(status().isNotFound()); this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist()) - .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist()); + .andExpect(jsonPath("$[?(@.sensorTopic == 'jsonTest')]").doesNotExist()); this.mockMvc.perform(get(sensorParserConfigUrl + "/list/available").with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.Bro").value("org.apache.metron.parsers.bro.BasicBroParser")) - .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.GrokParser")); + .andExpect(jsonPath("$.JSONMap").value("org.apache.metron.parsers.json.JSONMapParser")) + .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.grok.GrokParser")); this.mockMvc.perform(get(sensorParserConfigUrl + "/reload/available").with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.Bro").value("org.apache.metron.parsers.bro.BasicBroParser")) - .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.GrokParser")); + .andExpect(jsonPath("$.JSONMap").value("org.apache.metron.parsers.json.JSONMapParser")) + .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.grok.GrokParser")); this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(parseRequest)) .andExpect(status().isOk()) @@ -345,9 +361,9 @@ public class SensorParserConfigControllerIntegrationTest { .andExpect(status().isInternalServerError()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.responseCode").value(500)) - .andExpect(jsonPath("$.message").value("java.lang.ClassNotFoundException: badClass")); + .andExpect(jsonPath("$.message").value("java.lang.IllegalStateException: The specified implementation class 'badClass' is not known.")); - this.sensorParserConfigService.delete("broTest"); + this.sensorParserConfigService.delete("jsonTest"); this.sensorParserConfigService.delete("squidTest"); } http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplNoMockTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplNoMockTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplNoMockTest.java new file mode 100644 index 0000000..e32c7c7 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplNoMockTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import java.nio.charset.StandardCharsets; +import javax.security.auth.Subject; +import oi.thekraken.grok.api.Grok; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.GrokValidation; +import org.apache.metron.rest.service.GrokService; +import org.apache.metron.rest.service.HdfsService; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.core.env.Environment; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; +import static org.powermock.api.mockito.PowerMockito.whenNew; + +public class GrokServiceImplNoMockTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private Environment environment; + private Grok grok; + private GrokService grokService; + + private Configuration configuration; + private HdfsService hdfsService; + private String testDir = "./target/hdfsUnitTest/"; + + @Before + public void setup() throws IOException { + configuration = new Configuration(); + hdfsService = new HdfsServiceImpl(configuration); + File file = new File(testDir); + if (!file.exists()) { + file.mkdirs(); + } + FileUtils.cleanDirectory(file); + environment = mock(Environment.class); + when(environment.getProperty(MetronRestConstants.HDFS_METRON_APPS_ROOT)).thenReturn(testDir); + grok = mock(Grok.class); + grokService = new GrokServiceImpl(environment, grok, new Configuration(),hdfsService); + } + + @After + public void teardown() throws IOException { + File file = new File(testDir); + FileUtils.cleanDirectory(file); + } + + @Test + public void saveStatementShouldSaveStatement() throws Exception { + Authentication authentication = mock(Authentication.class); + when(authentication.getName()).thenReturn("user1"); + SecurityContextHolder.getContext().setAuthentication(authentication); + String statement = "grok statement"; + grokService.saveStatement("/patterns/test/test", statement.getBytes(StandardCharsets.UTF_8)); + assertEquals(statement,grokService.getStatement("/patterns/test/test")); + + } + + @Test + public void getStatementFromClasspathShouldReturnStatement() throws Exception { + Authentication authentication = mock(Authentication.class); + when(authentication.getName()).thenReturn("user1"); + SecurityContextHolder.getContext().setAuthentication(authentication); + String expected = FileUtils.readFileToString(new File("../../metron-platform/metron-parsers/src/main/resources/patterns/common")); + assertEquals(expected, grokService.getStatement("/patterns/common")); + } + + @Test + public void getStatementFromClasspathShouldThrowRestException() throws Exception { + Authentication authentication = mock(Authentication.class); + when(authentication.getName()).thenReturn("user1"); + SecurityContextHolder.getContext().setAuthentication(authentication); + exception.expect(RestException.class); + exception.expectMessage("Could not find a statement at path /bad/path"); + + grokService.getStatement("/bad/path"); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java index 1935269..e6fdbd0 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java @@ -17,16 +17,22 @@ */ package org.apache.metron.rest.service.impl; +import javax.security.auth.Subject; import oi.thekraken.grok.api.Grok; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.GrokValidation; import org.apache.metron.rest.service.GrokService; +import org.apache.metron.rest.service.HdfsService; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.springframework.core.env.Environment; @@ -44,11 +50,12 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.powermock.api.mockito.PowerMockito.when; import static org.powermock.api.mockito.PowerMockito.whenNew; @RunWith(PowerMockRunner.class) -@PrepareForTest({GrokServiceImpl.class, FileWriter.class}) +@PrepareForTest({GrokServiceImpl.class, FileWriter.class, HdfsServiceImpl.class, FileSystem.class}) public class GrokServiceImplTest { @Rule public final ExpectedException exception = ExpectedException.none(); @@ -56,12 +63,13 @@ public class GrokServiceImplTest { private Environment environment; private Grok grok; private GrokService grokService; - + private HdfsService hdfsService; @Before public void setUp() throws Exception { environment = mock(Environment.class); grok = mock(Grok.class); - grokService = new GrokServiceImpl(environment, grok); + mockStatic(FileSystem.class); + grokService = new GrokServiceImpl(environment, grok, new Configuration(), hdfsService); } @Test @@ -203,9 +211,8 @@ public class GrokServiceImplTest { SecurityContextHolder.getContext().setAuthentication(authentication); when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target"); - grokService.saveTemporary(statement, "squid"); + File testFile = grokService.saveTemporary(statement, "squid"); - File testFile = new File("./target/user1/squid"); assertEquals(statement, FileUtils.readFileToString(testFile)); testFile.delete(); } @@ -232,18 +239,4 @@ public class GrokServiceImplTest { grokService.saveTemporary(null, "squid"); } - - @Test - public void getStatementFromClasspathShouldReturnStatement() throws Exception { - String expected = FileUtils.readFileToString(new File("../../metron-platform/metron-parsers/src/main/resources/patterns/squid")); - assertEquals(expected, grokService.getStatementFromClasspath("/patterns/squid")); - } - - @Test - public void getStatementFromClasspathShouldThrowRestException() throws Exception { - exception.expect(RestException.class); - exception.expectMessage("Could not find a statement at path /bad/path"); - - grokService.getStatementFromClasspath("/bad/path"); - } } http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java index d35a48c..1ded45e 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java @@ -18,21 +18,31 @@ package org.apache.metron.rest.service.impl; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.FileInputStream; +import java.nio.file.Paths; import org.adrianwalker.multilinestring.Multiline; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.DeleteBuilder; import org.apache.curator.framework.api.GetChildrenBuilder; import org.apache.curator.framework.api.GetDataBuilder; import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.metron.bundles.BundleSystem; +import org.apache.metron.bundles.bundle.Bundle; +import org.apache.metron.bundles.util.BundleProperties; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.rest.MetronRestConstants; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.ParseMessageRequest; import org.apache.metron.rest.service.GrokService; import org.apache.metron.rest.service.SensorParserConfigService; +import org.apache.metron.test.utils.ResourceCopier; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; +import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -62,11 +72,12 @@ public class SensorParserConfigServiceImplTest { ObjectMapper objectMapper; CuratorFramework curatorFramework; GrokService grokService; + BundleSystem bundleSystem; SensorParserConfigService sensorParserConfigService; /** { - "parserClassName": "org.apache.metron.parsers.GrokParser", + "parserClassName": "org.apache.metron.parsers.grok.GrokParser", "sensorTopic": "squid", "parserConfig": { "grokPath": "/patterns/squid", @@ -80,31 +91,53 @@ public class SensorParserConfigServiceImplTest { /** { - "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", - "sensorTopic":"bro", + "parserClassName":"org.apache.metron.parsers.json.JSONMapParser", + "sensorTopic":"jsonMap", "parserConfig": {} } */ @Multiline - public static String broJson; + public static String jsonMapJson; @Before public void setUp() throws Exception { + BundleSystem.reset(); objectMapper = mock(ObjectMapper.class); curatorFramework = mock(CuratorFramework.class); grokService = mock(GrokService.class); - sensorParserConfigService = new SensorParserConfigServiceImpl(objectMapper, curatorFramework, grokService); + environment = mock(Environment.class); + when(environment.getProperty(MetronRestConstants.HDFS_METRON_APPS_ROOT)).thenReturn("./target"); + try(FileInputStream fis = new FileInputStream(new File("src/test/resources/zookeeper/bundle.properties"))) { + BundleProperties properties = BundleProperties.createBasicBundleProperties(fis, new HashMap<>()); + properties.setProperty(BundleProperties.BUNDLE_LIBRARY_DIRECTORY,"./target"); + properties.unSetProperty("bundle.library.directory.alt"); + bundleSystem = new BundleSystem.Builder().withBundleProperties(properties).build(); + sensorParserConfigService = new SensorParserConfigServiceImpl(environment, objectMapper, curatorFramework, + grokService); + ((SensorParserConfigServiceImpl)sensorParserConfigService).setBundleSystem(bundleSystem); + } } + @After + public void tearDown() { + BundleSystem.reset(); + } + + @AfterClass + public static void afterClass() { + BundleSystem.reset(); + } + + @Test public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception { DeleteBuilder builder = mock(DeleteBuilder.class); when(curatorFramework.delete()).thenReturn(builder); - when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap")).thenThrow(KeeperException.NoNodeException.class); - assertFalse(sensorParserConfigService.delete("bro")); + assertFalse(sensorParserConfigService.delete("jsonMap")); } @Test @@ -114,9 +147,9 @@ public class SensorParserConfigServiceImplTest { DeleteBuilder builder = mock(DeleteBuilder.class); when(curatorFramework.delete()).thenReturn(builder); - when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap")).thenThrow(Exception.class); - assertFalse(sensorParserConfigService.delete("bro")); + assertFalse(sensorParserConfigService.delete("jsonMap")); } @Test @@ -124,9 +157,9 @@ public class SensorParserConfigServiceImplTest { DeleteBuilder builder = mock(DeleteBuilder.class); when(curatorFramework.delete()).thenReturn(builder); - when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(null); + when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap")).thenReturn(null); - assertTrue(sensorParserConfigService.delete("bro")); + assertTrue(sensorParserConfigService.delete("jsonMap")); verify(curatorFramework).delete(); } @@ -136,20 +169,20 @@ public class SensorParserConfigServiceImplTest { final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig(); GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); - when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap")).thenReturn(jsonMapJson.getBytes()); when(curatorFramework.getData()).thenReturn(getDataBuilder); - assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.findOne("bro")); + assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.findOne("jsonMap")); } @Test public void findOneShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception { GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); - when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap")).thenThrow(KeeperException.NoNodeException.class); when(curatorFramework.getData()).thenReturn(getDataBuilder); - assertNull(sensorParserConfigService.findOne("bro")); + assertNull(sensorParserConfigService.findOne("jsonMap")); } @Test @@ -157,11 +190,11 @@ public class SensorParserConfigServiceImplTest { exception.expect(RestException.class); GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); - when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap")).thenThrow(Exception.class); when(curatorFramework.getData()).thenReturn(getDataBuilder); - sensorParserConfigService.findOne("bro"); + sensorParserConfigService.findOne("jsonMap"); } @Test @@ -169,13 +202,13 @@ public class SensorParserConfigServiceImplTest { GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())) .thenReturn(new ArrayList() {{ - add("bro"); + add("jsonMap"); add("squid"); }}); when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); assertEquals(new ArrayList() {{ - add("bro"); + add("jsonMap"); add("squid"); }}, sensorParserConfigService.getAllTypes()); } @@ -205,7 +238,7 @@ public class SensorParserConfigServiceImplTest { GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())) .thenReturn(new ArrayList() {{ - add("bro"); + add("jsonMap"); add("squid"); }}); when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); @@ -213,7 +246,7 @@ public class SensorParserConfigServiceImplTest { final SensorParserConfig broSensorParserConfig = getTestBroSensorParserConfig(); final SensorParserConfig squidSensorParserConfig = getTestSquidSensorParserConfig(); GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); - when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap")).thenReturn(jsonMapJson.getBytes()); when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/squid")).thenReturn(squidJson.getBytes()); when(curatorFramework.getData()).thenReturn(getDataBuilder); @@ -228,12 +261,12 @@ public class SensorParserConfigServiceImplTest { exception.expect(RestException.class); SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); - when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class); + when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap", jsonMapJson.getBytes())).thenThrow(Exception.class); when(curatorFramework.setData()).thenReturn(setDataBuilder); final SensorParserConfig sensorParserConfig = new SensorParserConfig(); - sensorParserConfig.setSensorTopic("bro"); + sensorParserConfig.setSensorTopic("jsonMap"); sensorParserConfigService.save(sensorParserConfig); } @@ -241,22 +274,22 @@ public class SensorParserConfigServiceImplTest { public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception { final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig(); - when(objectMapper.writeValueAsString(sensorParserConfig)).thenReturn(broJson); + when(objectMapper.writeValueAsString(sensorParserConfig)).thenReturn(jsonMapJson); SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); - when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat()); + when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap", jsonMapJson.getBytes())).thenReturn(new Stat()); when(curatorFramework.setData()).thenReturn(setDataBuilder); assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.save(sensorParserConfig)); - verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/bro"), eq(broJson.getBytes())); + verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/jsonMap"), eq(jsonMapJson.getBytes())); } @Test public void reloadAvailableParsersShouldReturnParserClasses() throws Exception { Map<String, String> availableParsers = sensorParserConfigService.reloadAvailableParsers(); assertTrue(availableParsers.size() > 0); - assertEquals("org.apache.metron.parsers.GrokParser", availableParsers.get("Grok")); - assertEquals("org.apache.metron.parsers.bro.BasicBroParser", availableParsers.get("Bro")); + assertEquals("org.apache.metron.parsers.grok.GrokParser", availableParsers.get("Grok")); + assertEquals("org.apache.metron.parsers.json.JSONMapParser", availableParsers.get("JSONMap")); } @Test @@ -324,15 +357,15 @@ public class SensorParserConfigServiceImplTest { private SensorParserConfig getTestBroSensorParserConfig() { SensorParserConfig sensorParserConfig = new SensorParserConfig(); - sensorParserConfig.setSensorTopic("bro"); - sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser"); + sensorParserConfig.setSensorTopic("jsonMap"); + sensorParserConfig.setParserClassName("org.apache.metron.parsers.json.JSONMapParser"); return sensorParserConfig; } private SensorParserConfig getTestSquidSensorParserConfig() { SensorParserConfig sensorParserConfig = new SensorParserConfig(); sensorParserConfig.setSensorTopic("squid"); - sensorParserConfig.setParserClassName("org.apache.metron.parsers.GrokParser"); + sensorParserConfig.setParserClassName("org.apache.metron.parsers.grok.GrokParser"); sensorParserConfig.setParserConfig(new HashMap() {{ put("grokPath", "/patterns/squid"); put("patternLabel", "SQUID_DELIMITED"); http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/resources/metron-parser-test-assembly-0.4.0-archive.tar.gz ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/resources/metron-parser-test-assembly-0.4.0-archive.tar.gz b/metron-interface/metron-rest/src/test/resources/metron-parser-test-assembly-0.4.0-archive.tar.gz new file mode 100644 index 0000000..a4d7432 Binary files /dev/null and b/metron-interface/metron-rest/src/test/resources/metron-parser-test-assembly-0.4.0-archive.tar.gz differ http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/resources/zookeeper/bundle.properties ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/resources/zookeeper/bundle.properties b/metron-interface/metron-rest/src/test/resources/zookeeper/bundle.properties new file mode 100644 index 0000000..99e4bb9 --- /dev/null +++ b/metron-interface/metron-rest/src/test/resources/zookeeper/bundle.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +bundle.library.directory=./target/remote/metron/extension_lib/ +bundle.library.directory.alt=./target/remote/metron/extension_contrib_lib/ +bundle.archive.extension=bundle +bundle.meta.id.prefix=Bundle +bundle.extension.type.MessageParser=org.apache.metron.parsers.interfaces.MessageParser http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-interface/metron-rest/src/test/resources/zookeeper/global.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/resources/zookeeper/global.json b/metron-interface/metron-rest/src/test/resources/zookeeper/global.json index 396896f..9292f72 100644 --- a/metron-interface/metron-rest/src/test/resources/zookeeper/global.json +++ b/metron-interface/metron-rest/src/test/resources/zookeeper/global.json @@ -1,4 +1,29 @@ { + "es.clustername": "metron", + "es.ip": "localhost", + "es.port": 9300, + "es.date.format": "yyyy.MM.dd.HH", + + "solr.zookeeper": "localhost:2181", + "solr.collection": "metron", + "solr.numShards": 1, + "solr.replicationFactor": 1, + + "fieldValidations" : [ + { + "input" : [ "ip_src_addr", "ip_dst_addr"], + "validation" : "IP" + } + ], + + "profiler.client.period.duration": "15", + "profiler.client.period.duration.units": "MINUTES", + "profiler.client.hbase.table": "profiler", + "profiler.client.hbase.column.family": "P", + "profiler.client.salt.divisor": "1000", + "hbase.provider.impl": "org.apache.metron.hbase.HTableProvider", + + "geo.hdfs.file": "src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz", "update.hbase.table" : "updates", "update.hbase.cf" : "t" } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/README.md ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/README.md b/metron-maven-archetypes/README.md new file mode 100644 index 0000000..52515e6 --- /dev/null +++ b/metron-maven-archetypes/README.md @@ -0,0 +1,57 @@ +# Metron Maven Archetypes + +These are [Apache Maven Archetypes](http://maven.apache.org/archetype/index.html) for use in creating extension components for the Apache Metron system +Archetypes are helpful in creating maven based projects with the correct setup and dependency configurations for the target system, including providing sample implementations. + +### metron-parser-extension-archetype +This is an archetype for creating an Apache Metron Parser Extension + +#### Use + +Build and install the archetype +``` +$ mvn install +``` + +Create a directory to host your extension code + +``` +$ mkdir ~/src/metron-parser-nice-extension +$ cd ~/src/metron-parser-nice-extension +``` + +Use the archetype to create your project + +``` +$ mvn -U archetype:generate -DarchetypeCatalog=local +[Select the org.apache.metron:metron-parser-extension-archetype (Apache Parser Extension Archetype for Metron) entry] +``` + +Configure the project properties. Ending up with something like this: + +Confirm properties configuration: +* groupId: org.someorg +* artifactId: metron-parser-nice-extension +* version: 0.4.1 +* package: org.someorg.parsers +* metronVersion: 0.4.1 +* parserClassName: Nice +* parserName: nice + +This will produce a project: + + + + +#### Project description +##### metron-parser-nice-extension (The extension project) + +##### metron-parser-nice (The Parser Project) +This project contains the sample parser code, configuration, and tests + +##### metron-parser-nice-bundle (The Bundle Project ) +The project contains builds the Bundle file + +##### metron-parser-nice-assembly (The Assembly Project) +The project that builds a tar.gz assembly of the bundle and configuration. +This is the final, installable product. http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/README.md ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/README.md b/metron-maven-archetypes/metron-parser-extension-archetype/README.md new file mode 100644 index 0000000..6ab48e8 --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/README.md @@ -0,0 +1,83 @@ +# Metron Maven Parser Extension Archetype + +This archetype can be used to create a project to create, build and deploy one or more metron parsers +By filling out the parameters when creating the archetype, the produced project will be completely setup +to build. + + +### USAGE +Creating a parser and deploying it... and deleting it using the rest api in full dev + +## Preparation +In order to build the parser, we need to build and install the maven plugin to build the bundles, as well as the maven archetype itself. +At this time, since they are not published to apache maven, this will be a requirement. + +- Build the bundle plugin +``` +cd bundles-maven-plugin && mvn -q install && cd .. +``` +- Build the archetype +``` +cd metron-maven-archetypes/metron-parser-extension-archetype +mvn install +cd ../.. +``` + +### Build Metron +``` +cd metron +mvn clean install +``` + +## Create and build a new parser extension +- Make a directory and cd into it +- Create from archetype +``` +mvn -U archetype:generate -DarchetypeCatalog=local +``` + - choose the org.apache.metron:metron-maven-parser-extension-archetype (Apache Maven Parser Extension Archetype for Metron) option + - Fill out the information (information on the parameters is in the README for the archetype) +- cd into the created directory and build the parser extension +``` +mvn package +``` + +This will result in the tar.gz being created inside the {NAME}-parser-assembly project/target directory + +## start full_dev +- run vagrant up from metron-deployment/vagrant/full_dev_platform + +## add a slot to storm +- log into ambari on http://node1:8080 +- add a new slot to the storm config + +## Open swagger +- log into ambari on http://node1:8080 +- go into the metron service | quicklinks +- start swagger with user | password + +## Install your parser +In swagger, using the parser-extension-controller +- drop down the POST method +- use the file selector to select the created .tar.gz +- execute +The extension should now be installed: +- you should be able to use the other GET methods to review the generated configuration. + - the configuration will have the extensionID to use for the other {name} operations +- you should be able to view the installed files in hdfs/apps/metron/patterns/{parsername}, hdfs/apps/metron/extensions_alt_lib/ + +## Start the parser using rest +Still in swagger +- Use the Kafka Controller and the Storm Controller to create a {parserName} topic, and start a storm job for {parserName} +- In the storm UI you should see the storm instance for the parser +You should be able to verify using the rest api and the various controllers that the configurations are there and the topics exist and the storm jobs as well + + +## Uninstall your parser +In swagger, using the parser-extension-controller +- drop down the DELETE method +- fill in the name parameter with the extensionID from the config +- exectute + +When complete you should see that the kafka topic, storm job, the various configurations are all gone, along with the patterns and other things from hdfs + http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/pom.xml ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/pom.xml b/metron-maven-archetypes/metron-parser-extension-archetype/pom.xml new file mode 100644 index 0000000..10d9b8d --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/pom.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extension-archetype</artifactId> + <version>0.4.1</version> + <packaging>maven-archetype</packaging> + + <name>metron-parser-extension-archetype</name> + + <build> + <extensions> + <extension> + <groupId>org.apache.maven.archetype</groupId> + <artifactId>archetype-packaging</artifactId> + <version>3.0.1</version> + </extension> + </extensions> + + <pluginManagement> + <plugins> + <plugin> + <artifactId>maven-archetype-plugin</artifactId> + <version>3.0.1</version> + </plugin> + </plugins> + </pluginManagement> + </build> + + <description>Apache Maven Parser Extension Archetype for Metron</description> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml new file mode 100644 index 0000000..10afc04 --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -0,0 +1,81 @@ +<?xml version="1.0" encoding="UTF-8"?> +<archetype-descriptor xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0 http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd"; name="metron-parser-asa-extension" + xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <requiredProperties> + <requiredProperty key="metronVersion"> + </requiredProperty> + <requiredProperty key="parserName"> + <validationRegex> ^[a-z]+[A-Z,a-z]+$</validationRegex> + </requiredProperty> + <requiredProperty key="parserClassName"> + <validationRegex>^[A-Z].*$</validationRegex> + </requiredProperty> + </requiredProperties> + <modules> + <module id="metron-parser-__parserName__" dir="metron-parser-__parserName__" name="metron-parser-__parserName__"> + <fileSets> + <fileSet filtered="true" packaged="true" encoding="UTF-8"> + <directory>src/main/java</directory> + <includes> + <include>**/*.java</include> + </includes> + </fileSet> + <fileSet filtered="true" encoding="UTF-8"> + <directory>src/main/config</directory> + <includes> + <include>**/*.json</include> + <include>**/*.template</include> + </includes> + </fileSet> + <fileSet encoding="UTF-8"> + <directory>src/main/resources</directory> + <includes> + <include>**/*</include> + </includes> + </fileSet> + <fileSet filtered="true" packaged="true" encoding="UTF-8"> + <directory>src/test/java</directory> + <includes> + <include>**/*.java</include> + </includes> + </fileSet> + <fileSet filtered="true" encoding="UTF-8"> + <directory>src/test/resources</directory> + <includes> + <include>**/*.properties</include> + </includes> + </fileSet> + <fileSet filtered="true" encoding="UTF-8"> + <directory>src/test/resources</directory> + <includes> + <include>**/*.</include> + <include>**/*.config</include> + <include>**/*.gz</include> + <include>**/*.json</include> + <include>**/*.raw</include> + <include>**/*.parsed</include> + </includes> + </fileSet> + <fileSet filtered="true" encoding="UTF-8"> + <directory></directory> + <includes> + <include>README.md</include> + </includes> + </fileSet> + </fileSets> + </module> + <module id="metron-parser-__parserName__-bundle" dir="metron-parser-__parserName__-bundle" name="metron-parser-__parserName__-bundle"> + </module> + <module id="metron-parser-__parserName__-assembly" dir="metron-parser-__parserName__-assembly" name="metron-parser-parserName__-assembly"> + <fileSets> + <fileSet filtered="true" encoding="UTF-8"> + <directory>src/main/assembly</directory> + <includes> + <include>**/*.xml</include> + </includes> + </fileSet> + </fileSets> + </module> + </modules> +</archetype-descriptor> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/pom.xml b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/pom.xml new file mode 100644 index 0000000..6c749a7 --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/pom.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>${groupId}</groupId> + <artifactId>${rootArtifactId}</artifactId> + <version>${version}</version> + </parent> + + <groupId>${groupId}</groupId> + <artifactId>metron-parser-${parserName}-assembly</artifactId> + <version>${version}</version> + <packaging>pom</packaging> + <name>metron-parser-${parserName}-assembly</name> + + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptor>src/main/assembly/assembly.xml</descriptor> + </configuration> + <executions> + <execution> + <id>make-assembly</id> <!-- this is used for inheritance merges --> + <phase>package</phase> <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/src/main/assembly/assembly.xml b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/src/main/assembly/assembly.xml new file mode 100644 index 0000000..ae1ca96 --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-assembly/src/main/assembly/assembly.xml @@ -0,0 +1,56 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<assembly> + <id>archive</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${symbol_dollar}{project.basedir}/../metron-parser-${parserName}/src/main/config</directory> + <outputDirectory>/config</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0644</fileMode> + <lineEnding>unix</lineEnding> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${symbol_dollar}{project.basedir}/../metron-parser-${parserName}/src/main/resources/patterns</directory> + <outputDirectory>/patterns</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0644</fileMode> + <lineEnding>unix</lineEnding> + </fileSet> + <fileSet> + <directory>${symbol_dollar}{project.basedir}/../metron-parser-${parserName}-bundle/target</directory> + <includes> + <include>metron-parser-${parserName}-bundle-${symbol_dollar}{project.version}.bundle</include> + </includes> + <outputDirectory>/lib</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + </fileSet> + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-bundle/pom.xml b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-bundle/pom.xml new file mode 100644 index 0000000..1639eb7 --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__-bundle/pom.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>${groupId}</groupId> + <artifactId>${rootArtifactId}</artifactId> + <version>${version}</version> + </parent> + + <artifactId>metron-parser-${parserName}-bundle</artifactId> + <version>${version}</version> + <name>metron-parser-${parserName}-bundle</name> + <packaging>bundle</packaging> + <properties> + <maven.javadoc.skip>true</maven.javadoc.skip> + <source.skip>false</source.skip> + </properties> + + <dependencies> + <dependency> + <groupId>${groupId}</groupId> + <artifactId>metron-parser-${parserName}</artifactId> + <version>${version}</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/README.md ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/README.md b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/README.md new file mode 100644 index 0000000..df62fa9 --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/README.md @@ -0,0 +1,5 @@ +# ${parserClassName} Parser + +## Introduction + +This is the ${parserClassName} parser http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/pom.xml ---------------------------------------------------------------------- diff --git a/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/pom.xml b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/pom.xml new file mode 100644 index 0000000..93804eb --- /dev/null +++ b/metron-maven-archetypes/metron-parser-extension-archetype/src/main/resources/archetype-resources/metron-parser-__parserName__/pom.xml @@ -0,0 +1,83 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>${groupId}</groupId> + <artifactId>${rootArtifactId}</artifactId> + <version>${version}</version> + </parent> + <artifactId>metron-parser-${parserName}</artifactId> + <version>${version}</version> + <name>metron-parser-${parserName}</name> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>${metronVersion}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parsers</artifactId> + <version>${metronVersion}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.atteo.classindex</groupId> + <artifactId>classindex</artifactId> + <version>${global_classindex_version}</version> + <scope>provided</scope> + </dependency> + <!-- testing --> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions-testing</artifactId> + <version>${metronVersion}</version> + <type>pom</type> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>${global_jar_version}</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + <resource> + <directory>src/main/patterns</directory> + </resource> + <resource> + <directory>src/test/resources</directory> + </resource> + </resources> + </build> +</project>
