metron git commit: METRON-1145: Profiler mpack does not create kafka acl and hbase acls correctly in kerberized cluster closes apache/incubator-metron#726

2017-08-31 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master 38eebd59c -> 31d99f122


METRON-1145: Profiler mpack does not create kafka acl and hbase acls correctly 
in kerberized cluster closes apache/incubator-metron#726


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/31d99f12
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/31d99f12
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/31d99f12

Branch: refs/heads/master
Commit: 31d99f122e89d81590df84f66c2dc9fb336a03b8
Parents: 38eebd5
Author: cstella 
Authored: Thu Aug 31 15:48:01 2017 -0400
Committer: cstella 
Committed: Thu Aug 31 15:48:01 2017 -0400

--
 .../CURRENT/package/scripts/profiler_commands.py|  4 
 .../CURRENT/package/scripts/profiler_master.py  | 16 
 2 files changed, 20 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/31d99f12/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
index ddd66cb..4dfa371 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
@@ -98,6 +98,10 @@ class ProfilerCommands:
 Logger.info("Done creating HBase Tables")
 self.set_hbase_configured()
 
+def init_kafka_acls(self):
+Logger.info('Creating Kafka ACls for profiler')
+metron_service.init_kafka_acls(self.__params, [self.__profiler_topic], 
['profiler'])
+
 def set_hbase_acls(self):
 Logger.info("Setting HBase ACLs")
 if self.__params.security_enabled:

http://git-wip-us.apache.org/repos/asf/metron/blob/31d99f12/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
index 4946ab0..77c32f0 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
@@ -27,6 +27,7 @@ from resource_management.libraries.script import Script
 
 from metron_security import storm_security_setup
 import metron_service
+import metron_security
 from profiler_commands import ProfilerCommands
 
 
@@ -59,6 +60,9 @@ class Profiler(Script):
 commands.create_hbase_tables()
 if params.security_enabled and not commands.is_hbase_acl_configured():
 commands.set_hbase_acls()
+if params.security_enabled and not commands.is_acl_configured():
+commands.init_kafka_acls()
+commands.set_acl_configured()
 
 Logger.info("Calling security setup")
 storm_security_setup(params)
@@ -68,6 +72,18 @@ class Profiler(Script):
 env.set_params(params)
 self.configure(env)
 commands = ProfilerCommands(params)
+if params.security_enabled:
+metron_security.kinit(params.kinit_path_local,
+  params.metron_keytab_path,
+  params.metron_principal_name,
+  execute_user=params.metron_user)
+
+if params.security_enabled and not commands.is_hbase_acl_configured():
+commands.set_hbase_acls()
+if params.security_enabled and not commands.is_acl_configured():
+commands.init_kafka_acls()
+commands.set_acl_configured()
+
 commands.start_profiler_topology(env)
 
 def stop(self, env, upgrade_type=None):

metron git commit: METRON-1134: Allow parser command line options to be specified in the zookeeper parser config. closes apache/incubator-metron#717

2017-08-31 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master 8ef18d31a -> 38eebd59c


METRON-1134: Allow parser command line options to be specified in the zookeeper 
parser config. closes apache/incubator-metron#717


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/38eebd59
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/38eebd59
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/38eebd59

Branch: refs/heads/master
Commit: 38eebd59ce2884ead7f3797ff98f7308409e05dd
Parents: 8ef18d3
Author: cstella 
Authored: Thu Aug 31 15:46:29 2017 -0400
Committer: cstella 
Committed: Thu Aug 31 15:46:29 2017 -0400

--
 ...orParserConfigControllerIntegrationTest.java |  18 +-
 .../configuration/SensorParserConfig.java   | 208 -
 .../management/ConfigurationFunctionsTest.java  |  10 +-
 metron-platform/metron-parsers/README.md|  11 +
 .../parsers/topology/ParserTopologyBuilder.java |  87 ++--
 .../parsers/topology/ParserTopologyCLI.java | 218 +++---
 .../parsers/topology/config/ValueSupplier.java  |  30 ++
 .../components/ParserTopologyComponent.java |  30 +-
 .../parsers/topology/ParserTopologyCLITest.java | 425 ++-
 9 files changed, 934 insertions(+), 103 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/38eebd59/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
--
diff --git 
a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
 
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
index f5ea23d..66771eb 100644
--- 
a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
+++ 
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.rest.controller;
 
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.commons.io.FileUtils;
+import org.apache.metron.common.configuration.SensorParserConfig;
 import org.apache.metron.rest.MetronRestConstants;
 import org.apache.metron.rest.service.GrokService;
 import org.apache.metron.rest.service.SensorParserConfigService;
@@ -37,6 +38,7 @@ import org.springframework.web.context.WebApplicationContext;
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Method;
 
 import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
 import static org.hamcrest.Matchers.hasSize;
@@ -196,11 +198,17 @@ public class SensorParserConfigControllerIntegrationTest {
 cleanFileSystem();
 this.sensorParserConfigService.delete("broTest");
 this.sensorParserConfigService.delete("squidTest");
-
+Method[] method = SensorParserConfig.class.getMethods();
+int numFields = 0;
+for(Method m : method) {
+  if(m.getName().startsWith("set")) {
+numFields++;
+  }
+}
 this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, 
password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson))
 .andExpect(status().isCreated())
 
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-.andExpect(jsonPath("$.*", hasSize(10)))
+.andExpect(jsonPath("$.*", hasSize(numFields)))
 
.andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser"))
 .andExpect(jsonPath("$.sensorTopic").value("squidTest"))
 
.andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
@@ -215,7 +223,7 @@ public class SensorParserConfigControllerIntegrationTest {
 this.mockMvc.perform(get(sensorParserConfigUrl + 
"/squidTest").with(httpBasic(user,password)))
 .andExpect(status().isOk())
 
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
-.andExpect(jsonPath("$.*", hasSize(10)))
+.andExpect(jsonPath("$.*", hasSize(numFields)))
 
.andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser"))
 .andExpect(jsonPath("$.sensorTopic").value("squidTest"))
 
.andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
@@ -244,7 +252,7 @@ public class SensorParserConfigControllerIntegrationTest {

metron git commit: METRON-1138 Improve Error Message with Bad Profile Expression (nickwallen) closes apache/metron#721

2017-08-31 Thread nickallen 
Repository: metron
Updated Branches:
  refs/heads/master c73c7af1d -> 8ef18d31a


METRON-1138 Improve Error Message with Bad Profile Expression (nickwallen) 
closes apache/metron#721


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8ef18d31
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8ef18d31
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8ef18d31

Branch: refs/heads/master
Commit: 8ef18d31ab927acad7798c6c32252ca27c8e8b46
Parents: c73c7af
Author: nickwallen 
Authored: Thu Aug 31 10:51:53 2017 -0400
Committer: nickallen 
Committed: Thu Aug 31 10:51:53 2017 -0400

--
 .../metron/profiler/DefaultProfileBuilder.java  | 53 ++--
 .../profiler/DefaultProfileBuilderTest.java |  2 +-
 metron-analytics/metron-profiler/README.md  |  1 +
 3 files changed, 40 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/8ef18d31/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
index 1f352d0..2e34160 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
@@ -27,9 +27,11 @@ import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import org.apache.commons.collections4.ListUtils;
@@ -245,17 +247,28 @@ public class DefaultProfileBuilder implements 
ProfileBuilder, Serializable {
* @param expressionType The type of expression; init, update, result.  
Provides additional context if expression execution fails.
*/
   private void assign(Map expressions, Map 
transientState, String expressionType) {
-try {
 
-  // execute each of the 'update' expressions
-  MapUtils.emptyIfNull(expressions)
-  .forEach((var, expr) -> executor.assign(var, expr, 
transientState));
+// for each expression...
+for(Map.Entry entry : 
MapUtils.emptyIfNull(expressions).entrySet()) {
+  String var = entry.getKey();
+  String expr = entry.getValue();
+
+  try {
+// assign the result of the expression to the variable
+executor.assign(var, expr, transientState);
 
-} catch(ParseException e) {
+  } catch (Throwable e) {
 
-  // make it brilliantly clear that one of the 'update' expressions is bad
-  String msg = format("Bad '%s' expression: error=%s, profile=%s, 
entity=%s", expressionType, e.getMessage(), profileName, entity);
-  throw new ParseException(msg, e);
+// in-scope variables = persistent state maintained by the profiler + 
the transient state
+Set variablesInScope = new HashSet<>();
+variablesInScope.addAll(transientState.keySet());
+variablesInScope.addAll(executor.getState().keySet());
+
+String msg = format("Bad '%s' expression: error='%s', expr='%s', 
profile='%s', entity='%s', variables-available='%s'",
+expressionType, e.getMessage(), expr, profileName, entity, 
variablesInScope);
+LOG.error(msg, e);
+throw new ParseException(msg, e);
+  }
 }
   }
 
@@ -269,14 +282,24 @@ public class DefaultProfileBuilder implements 
ProfileBuilder, Serializable {
   private List execute(List expressions, Map 
transientState, String expressionType) {
 List results = new ArrayList<>();
 
-try {
-  ListUtils.emptyIfNull(expressions)
-  .forEach((expr) -> results.add(executor.execute(expr, 
transientState, Object.class)));
+for(String expr: ListUtils.emptyIfNull(expressions)) {
+  try {
+// execute an expression
+Object result = executor.execute(expr, transientState, Object.class);
+results.add(result);
+
+  } catch (Throwable e) {
 
-} catch (Throwable e) {
-  String msg = format("Bad '%s' expression: error=%s, profile=%s, 
entity=%s", expressionType, e.getMessage(), profileName, entity);
-  LOG.error(msg, e);
-  throw new ParseException(msg, e);
+// in-scope variables = persistent state maintained by the profiler + 
the

metron git commit: METRON-1132 Enhance Profiler Debug Functions (nickwallen) closes apache/metron#716

2017-08-31 Thread nickallen 
Repository: metron
Updated Branches:
  refs/heads/master 1a4f19c86 -> c73c7af1d


METRON-1132 Enhance Profiler Debug Functions (nickwallen) closes 
apache/metron#716


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c73c7af1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c73c7af1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c73c7af1

Branch: refs/heads/master
Commit: c73c7af1dc3cb6502517bc7ac1d37c631ec3dbe5
Parents: 1a4f19c
Author: nickwallen 
Authored: Thu Aug 31 10:38:21 2017 -0400
Committer: nickallen 
Committed: Thu Aug 31 10:38:21 2017 -0400

--
 .../metron-profiler-client/README.md| 21 +++--
 .../client/stellar/ProfilerFunctions.java   | 35 ++--
 .../client/stellar/ProfilerFunctionsTest.java   | 93 ++--
 .../metron/profiler/StandAloneProfiler.java | 40 +
 metron-analytics/metron-profiler/README.md  |  2 +
 5 files changed, 172 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/c73c7af1/metron-analytics/metron-profiler-client/README.md
--
diff --git a/metron-analytics/metron-profiler-client/README.md 
b/metron-analytics/metron-profiler-client/README.md
index a8b5a55..27aa3f4 100644
--- a/metron-analytics/metron-profiler-client/README.md
+++ b/metron-analytics/metron-profiler-client/README.md
@@ -418,8 +418,11 @@ Follow these steps in the Stellar REPL to see how it can 
be used to help create
 ```
 [Stellar]>>> profiler := PROFILER_INIT(conf)
 [Stellar]>>> profiler
-org.apache.metron.profiler.StandAloneProfiler@4f8ef473
+Profiler{1 profile(s), 0 messages(s), 0 route(s)}
 ```
+The profiler itself will show the number of profiles defined, the number 
of messages applied, and the number of routes taken.  
+
+A route is defined when a message is applied to a specific profile.  If a 
message is applied and not needed by any profile, then there are no routes.  If 
a message is needed by one profile, then one route has been defined.  If a 
message is needed by two profiles, then two routes have been defined.  
 
 1. Create a message to simulate the type of telemetry that you expect to be 
profiled.   As an example, in the editor copy/paste the JSON below.
 ```
@@ -436,14 +439,18 @@ Follow these steps in the Stellar REPL to see how it can 
be used to help create
 1. Apply some telemetry messages to your profiles.  The following applies the 
same message 3 times.
 ```
 [Stellar]>>> PROFILER_APPLY(message, profiler)
-org.apache.metron.profiler.StandAloneProfiler@4f8ef473
-
+Profiler{1 profile(s), 1 messages(s), 1 route(s)}
+```
+```
 [Stellar]>>> PROFILER_APPLY(message, profiler)
-org.apache.metron.profiler.StandAloneProfiler@4f8ef473
-
+Profiler{1 profile(s), 2 messages(s), 2 route(s)}
+```
+```
 [Stellar]>>> PROFILER_APPLY(message, profiler)
-org.apache.metron.profiler.StandAloneProfiler@4f8ef473
+Profiler{1 profile(s), 3 messages(s), 3 route(s)}
 ```
+
+It is also possible to apply multiple messages at once.  This is useful 
when testing against a larger set of data.  To do this, create a string that 
contains a JSON array of messages and pass that to the `PROFILER_APPLY` 
function.
 
 1. Flush the Profiler to see what has been calculated.  A flush is what occurs 
at the end of each 15 minute period in the Profiler.  The result is a list of 
profile measurements.  Each measurement is a map containing detailed 
information about the profile data that has been generated.
 ```
@@ -455,4 +462,4 @@ Follow these steps in the Stellar REPL to see how it can be 
used to help create
 
 This profile simply counts the number of messages by IP source address.  
Notice that the value is '3' for the entity '10.0.0.1' as we applied 3 messages 
with an 'ip_src_addr' of '10.0.0.1'.  There will always be one measurement for 
each [profile, entity] pair.
 
-1. If you are unhappy with the data that has been generated, then 'wash, rinse 
and repeat' this process.  After you are satisfied with the data being 
generated by the profile, then follow the [Getting 
Started](../metron-profiler#getting-started) guide to use the profile against 
your live, streaming data in a Metron cluster.
\ No newline at end of file
+1. If you are unhappy with the data that has been generated, then 'wash, rinse 
and repeat' this process.  Once you are happy with the profile that was 
created, follow the [Getting Started](../metron-profiler#getting-started) guide 
to use the profile against your live, streaming data in a Metron cluster.
\ No newline at end of file

metron git commit: METRON-1137 Build RPM for Metron MaaS as a part of rpm-docker packaging (anandsubbu via nickwallen) closes apache/metron#719

2017-08-31 Thread nickallen 
Repository: metron
Updated Branches:
  refs/heads/master 8ee9eb800 -> cd14cbeef


METRON-1137 Build RPM for Metron MaaS as a part of rpm-docker packaging 
(anandsubbu via nickwallen) closes apache/metron#719


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cd14cbee
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cd14cbee
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cd14cbee

Branch: refs/heads/master
Commit: cd14cbeefec6afc7aee9efcc288c9482d67d9657
Parents: 8ee9eb8
Author: anandsubbu 
Authored: Thu Aug 31 08:14:12 2017 -0400
Committer: nickallen 
Committed: Thu Aug 31 08:14:12 2017 -0400

--
 .../docker/rpm-docker/SPECS/metron.spec | 23 
 .../packaging/docker/rpm-docker/pom.xml |  6 +
 2 files changed, 29 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/cd14cbee/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec 
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 94c7e05..5a88877 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -53,6 +53,7 @@ Source8:metron-profiler-%{full_version}-archive.tar.gz
 Source9:metron-rest-%{full_version}-archive.tar.gz
 Source10:   metron-config-%{full_version}-archive.tar.gz
 Source11:   metron-management-%{full_version}-archive.tar.gz
+Source12:   metron-maas-service-%{full_version}-archive.tar.gz
 
 %description
 Apache Metron provides a scalable advanced security analytics framework
@@ -87,6 +88,7 @@ tar -xzf %{SOURCE8} -C %{buildroot}%{metron_home}
 tar -xzf %{SOURCE9} -C %{buildroot}%{metron_home}
 tar -xzf %{SOURCE10} -C %{buildroot}%{metron_home}
 tar -xzf %{SOURCE11} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE12} -C %{buildroot}%{metron_home}
 
 install %{buildroot}%{metron_home}/bin/metron-rest %{buildroot}/etc/init.d/
 install %{buildroot}%{metron_home}/bin/metron-management-ui 
%{buildroot}/etc/init.d/
@@ -426,7 +428,28 @@ chkconfig --del metron-management-ui
 
 # ~~
 
+%packagemaas-service
+Summary:Metron MaaS service
+Group:  Application/Internet
+Provides:   maas-service = %{version}
+
+%descriptionmaas-service
+This package install the Metron MaaS Service files %{metron_home}
+
+%files  maas-service
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/bin
+%{metron_home}/bin/maas_service.sh
+%{metron_home}/bin/maas_deploy.sh
+%attr(0644,root,root) 
%{metron_home}/lib/metron-maas-service-%{full_version}-uber.jar
+
+# ~~
+
 %changelog
+* Tue Aug 29 2017 Apache Metron  - 0.4.1
+- Add Metron MaaS service
 * Thu Jun 29 2017 Apache Metron  - 0.4.1
 - Add Metron Management jar 
 * Thu May 15 2017 Apache Metron  - 0.4.0

http://git-wip-us.apache.org/repos/asf/metron/blob/cd14cbee/metron-deployment/packaging/docker/rpm-docker/pom.xml
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml 
b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 749acd2..b6a9ce3 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -173,6 +173,12 @@
 *.tar.gz
 
 
+
+
${metron_dir}/metron-analytics/metron-maas-service/target/
+
+*.tar.gz
+
+