[25/43] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
new file mode 100644
index 000..4845b4f
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/main/java/org/apache/metron/parsers/ise/JavaCharStream.java
@@ -0,0 +1,633 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Generated By:JavaCC: Do not edit this line. JavaCharStream.java Version 5.0 
*/
+/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=false */
+package org.apache.metron.parsers.ise;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (with java-like unicode escape processing).
+ */
+
+class JavaCharStream
+{
+  /** Whether parser is static. */
+  public static final boolean staticFlag = false;
+
+  static final int hexval(char c) throws java.io.IOException {
+switch(c)
+{
+   case '0' :
+  return 0;
+   case '1' :
+  return 1;
+   case '2' :
+  return 2;
+   case '3' :
+  return 3;
+   case '4' :
+  return 4;
+   case '5' :
+  return 5;
+   case '6' :
+  return 6;
+   case '7' :
+  return 7;
+   case '8' :
+  return 8;
+   case '9' :
+  return 9;
+
+   case 'a' :
+   case 'A' :
+  return 10;
+   case 'b' :
+   case 'B' :
+  return 11;
+   case 'c' :
+   case 'C' :
+  return 12;
+   case 'd' :
+   case 'D' :
+  return 13;
+   case 'e' :
+   case 'E' :
+  return 14;
+   case 'f' :
+   case 'F' :
+  return 15;
+}
+
+throw new java.io.IOException(); // Should never come here
+  }
+
+/** Position in buffer. */
+  public int bufpos = -1;
+  int bufsize;
+  int available;
+  int tokenBegin;
+  protected int bufline[];
+  protected int bufcolumn[];
+
+  protected int column = 0;
+  protected int line = 1;
+
+  protected boolean prevCharIsCR = false;
+  protected boolean prevCharIsLF = false;
+
+  protected java.io.Reader inputStream;
+
+  protected char[] nextCharBuf;
+  protected char[] buffer;
+  protected int maxNextCharInd = 0;
+  protected int nextCharInd = -1;
+  protected int inBuf = 0;
+  protected int tabSize = 8;
+
+  protected void setTabSize(int i) { tabSize = i; }
+  protected int getTabSize(int i) { return tabSize; }
+
+  protected void ExpandBuff(boolean wrapAround)
+  {
+char[] newbuffer = new char[bufsize + 2048];
+int newbufline[] = new int[bufsize + 2048];
+int newbufcolumn[] = new int[bufsize + 2048];
+
+try
+{
+  if (wrapAround)
+  {
+System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - 
tokenBegin);
+System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos);
+buffer = newbuffer;
+
+System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - 
tokenBegin);
+System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+bufline = newbufline;
+
+System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - 
tokenBegin);
+System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, 
bufpos);
+bufcolumn = newbufcolumn;
+
+bufpos += (bufsize - tokenBegin);
+}
+else
+{
+System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - 
tokenBegin);
+buffer = newbuffer;
+
+System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - 
tokenBegin);
+bufline = newbufline;
+
+System.arraycopy(bufcolumn, 

[35/43] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
index 36863e3..a9446ce 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
@@ -24,6 +24,7 @@ import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.metron.common.Constants;
 import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.extensions.ParserExtensionConfig;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.StellarFunctions;
 import org.apache.metron.common.utils.JSONUtils;
@@ -70,6 +71,17 @@ public class ConfigurationsUtils {
 writeToZookeeper(GLOBAL.getZookeeperRoot(), globalConfig, client);
   }
 
+  public static void writeGlobalBundlePropertiesToZookeeper(byte[] 
bundleProperties, String zookeeperUrl) throws Exception{
+try(CuratorFramework client = getClient(zookeeperUrl)){
+  client.start();
+  writeGlobalBundlePropertiesToZookeeper(bundleProperties,client);
+}
+  }
+
+  public static void writeGlobalBundlePropertiesToZookeeper(byte[] 
bundleProperties, CuratorFramework client) throws Exception{
+writeToZookeeper(Constants.ZOOKEEPER_ROOT + "/bundle.properties", 
bundleProperties, client);
+  }
+
   public static void writeProfilerConfigToZookeeper(byte[] config, 
CuratorFramework client) throws Exception {
 PROFILER.deserialize(new String(config));
 writeToZookeeper(PROFILER.getZookeeperRoot(), config, client);
@@ -92,10 +104,31 @@ public class ConfigurationsUtils {
 writeToZookeeper(PARSER.getZookeeperRoot() + "/" + sensorType, configData, 
client);
   }
 
+  public static void writeParserExtensionConfigToZookeeper(String extensionID, 
byte[] configData, CuratorFramework client) throws Exception {
+ParserExtensionConfig c = (ParserExtensionConfig) 
PARSER_EXTENSION.deserialize(new String(configData));
+writeToZookeeper(PARSER_EXTENSION.getZookeeperRoot() + "/" + extensionID, 
configData, client);
+  }
+
+  public static void writeParserExtensionConfigToZookeeper(String extensionID, 
ParserExtensionConfig parserExtensionConfig, String zookeeperUrl) throws 
Exception {
+writeParserExtensionConfigToZookeeper(extensionID, 
JSONUtils.INSTANCE.toJSON(parserExtensionConfig), zookeeperUrl);
+  }
+
+  public static void writeParserExtensionConfigToZookeeper(String extensionID, 
byte[] configData, String zookeeperUrl) throws Exception {
+try(CuratorFramework client = getClient(zookeeperUrl)) {
+  client.start();
+  writeParserExtensionConfigToZookeeper(extensionID, configData, client);
+}
+  }
+
+  public static void deleteParsesrExtensionConfig(String extensionID, 
CuratorFramework client)throws Exception{
+deleteFromZookeeper(PARSER_EXTENSION.getZookeeperRoot() + "/" + 
extensionID, client);
+  }
+
   public static void writeSensorIndexingConfigToZookeeper(String sensorType, 
Map sensorIndexingConfig, String zookeeperUrl) throws Exception 
{
 writeSensorIndexingConfigToZookeeper(sensorType, 
JSONUtils.INSTANCE.toJSON(sensorIndexingConfig), zookeeperUrl);
   }
 
+
   public static void writeSensorIndexingConfigToZookeeper(String sensorType, 
byte[] configData, String zookeeperUrl) throws Exception {
 try(CuratorFramework client = getClient(zookeeperUrl)) {
   client.start();
@@ -143,6 +176,10 @@ public class ConfigurationsUtils {
 }
   }
 
+  public static void deleteFromZookeeper(String path, CuratorFramework client) 
throws Exception{
+client.delete().forPath(path);
+  }
+
   public static void updateConfigsFromZookeeper(Configurations configurations, 
CuratorFramework client) throws Exception {
 
configurations.updateGlobalConfig(readGlobalConfigBytesFromZookeeper(client));
   }
@@ -179,6 +216,11 @@ public class ConfigurationsUtils {
 return JSONUtils.INSTANCE.load(new 
ByteArrayInputStream(readFromZookeeper(PARSER.getZookeeperRoot() + "/" + 
sensorType, client)), SensorParserConfig.class);
   }
 
+  public static ParserExtensionConfig 
readParserExtensionConfigFromZookeeper(String extensionID, CuratorFramework 
client) throws Exception {
+return JSONUtils.INSTANCE.load(new 
ByteArrayInputStream(readFromZookeeper(PARSER_EXTENSION.getZookeeperRoot() + 
"/" + extensionID, client)), ParserExtensionConfig.class);
+  }
+
+
   public static byte[] 

[27/43] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/pom.xml
new file mode 100644
index 000..9731f36
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/pom.xml
@@ -0,0 +1,49 @@
+
+http://maven.apache.org/POM/4.0.0"; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+4.0.0
+
+
+org.apache.metron
+metron-parser-fireeye-extension
+0.4.1
+
+
+org.apache.metron
+metron-parser-fireeye-assembly
+0.4.1
+pom
+metron-parser-fireeye-assembly
+
+
+
+
+maven-assembly-plugin
+
+src/main/assembly/assembly.xml
+
+
+
+make-assembly 
+package 
+
+single
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/src/main/assembly/assembly.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/src/main/assembly/assembly.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/src/main/assembly/assembly.xml
new file mode 100644
index 000..cd2f99e
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-fireeye-extension/metron-parser-fireeye-assembly/src/main/assembly/assembly.xml
@@ -0,0 +1,53 @@
+http://maven.apache.org/POM/4.0.0"; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+4.0.0
+
+org.apache.metron
+metron-parser-yaf-extension
+0.4.1
+
+metron-parser-yaf
+0.4.1
+metron-parser-yaf
+jar
+
+UTF-8
+
UTF-8
+
+
+
+org.apache.metron
+metron-common
+${project.parent.version}
+provided
+
+
+org.apache.metron
+metron-parsers
+${project.parent.version}
+provided
+
+
+org.atteo.classindex
+classindex
+${global_classindex_version}
+provided
+
+
+
+org.apache.metron
+metron-parser-extensions-testing
+${project.parent.version}
+pom
+test
+
+
+
+
+
+org.apache.maven.plugins
+maven-jar-plugin
+${global_jar_version}
+
+
+
+test-jar
+
+
+
+
+
+
+
+src/main/resources
+
+
+src/main/patterns
+
+
+src/test/resources
+
+
+
+

[18/43] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-squid-extension/metron-parser-squid/src/main/resources/META-INF/LICENSE
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-squid-extension/metron-parser-squid/src/main/resources/META-INF/LICENSE
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-squid-extension/metron-parser-squid/src/main/resources/META-INF/LICENSE
new file mode 100644
index 000..849fa54
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-squid-extension/metron-parser-squid/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,602 @@
+Apache License
+   Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+  "License" shall mean the terms and conditions for use, reproduction,
+  and distribution as defined by Sections 1 through 9 of this document.
+
+  "Licensor" shall mean the copyright owner or entity authorized by
+  the copyright owner that is granting the License.
+
+  "Legal Entity" shall mean the union of the acting entity and all
+  other entities that control, are controlled by, or are under common
+  control with that entity. For the purposes of this definition,
+  "control" means (i) the power, direct or indirect, to cause the
+  direction or management of such entity, whether by contract or
+  otherwise, or (ii) ownership of fifty percent (50%) or more of the
+  outstanding shares, or (iii) beneficial ownership of such entity.
+
+  "You" (or "Your") shall mean an individual or Legal Entity
+  exercising permissions granted by this License.
+
+  "Source" form shall mean the preferred form for making modifications,
+  including but not limited to software source code, documentation
+  source, and configuration files.
+
+  "Object" form shall mean any form resulting from mechanical
+  transformation or translation of a Source form, including but
+  not limited to compiled object code, generated documentation,
+  and conversions to other media types.
+
+  "Work" shall mean the work of authorship, whether in Source or
+  Object form, made available under the License, as indicated by a
+  copyright notice that is included in or attached to the work
+  (an example is provided in the Appendix below).
+
+  "Derivative Works" shall mean any work, whether in Source or Object
+  form, that is based on (or derived from) the Work and for which the
+  editorial revisions, annotations, elaborations, or other modifications
+  represent, as a whole, an original work of authorship. For the purposes
+  of this License, Derivative Works shall not include works that remain
+  separable from, or merely link (or bind by name) to the interfaces of,
+  the Work and Derivative Works thereof.
+
+  "Contribution" shall mean any work of authorship, including
+  the original version of the Work and any modifications or additions
+  to that Work or Derivative Works thereof, that is intentionally
+  submitted to Licensor for inclusion in the Work by the copyright owner
+  or by an individual or Legal Entity authorized to submit on behalf of
+  the copyright owner. For the purposes of this definition, "submitted"
+  means any form of electronic, verbal, or written communication sent
+  to the Licensor or its representatives, including but not limited to
+  communication on electronic mailing lists, source code control systems,
+  and issue tracking systems that are managed by, or on behalf of, the
+  Licensor for the purpose of discussing and improving the Work, but
+  excluding communication that is conspicuously marked or otherwise
+  designated in writing by the copyright owner as "Not a Contribution."
+
+  "Contributor" shall mean Licensor and any individual or Legal Entity
+  on behalf of whom a Contribution has been received by Licensor and
+  subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  copyright license to reproduce, prepare Derivative Works of,
+  publicly display, publicly perform, sublicense, and 

[3/3] metron git commit: otto(o...@apache.org) Revert "otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master"

[otto]

otto(o...@apache.org)
Revert "otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers 
with master"

This reverts commit 4364665002756c1193cf834f3a2f53d774122760.


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d6a66eb2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d6a66eb2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d6a66eb2

Branch: refs/heads/feature/METRON-1136-extensions-parsers
Commit: d6a66eb26e71c4ffd5a673bd4791ce4c65112c3f
Parents: 4364665
Author: otto <o...@apache.org>
Authored: Wed Aug 30 11:26:43 2017 -0400
Committer: otto <o...@apache.org>
Committed: Wed Aug 30 11:29:21 2017 -0400

--
 .gitignore  |   2 +-
 metron-analytics/metron-maas-common/pom.xml |  11 -
 .../metron/maas/functions/MaaSFunctions.java| 324 ---
 metron-analytics/metron-maas-service/README.md  |  21 +-
 metron-analytics/metron-maas-service/pom.xml|  11 +
 .../metron/maas/functions/MaaSFunctions.java| 324 +++
 .../org/apache/metron/maas/service/Client.java  |  15 -
 .../apache/metron/maas/service/Constants.java   |   5 +-
 .../metron/maas/service/runner/Runner.java  |   9 +-
 .../metron/maas/submit/ModelSubmission.java |   5 -
 .../METRON/CURRENT/role_command_order.json  |   2 -
 .../configuration/metron-indexing-env.xml   |   2 +-
 .../configuration/metron-profiler-env.xml   | 155 -
 .../common-services/METRON/CURRENT/metainfo.xml |  43 ---
 .../package/scripts/params/params_linux.py  |  25 --
 .../package/scripts/params/status_params.py |  11 -
 .../package/scripts/profiler_commands.py| 194 ---
 .../CURRENT/package/scripts/profiler_master.py  |  94 --
 .../package/templates/profiler.properties.j2|  47 ---
 .../METRON/CURRENT/service_advisor.py   |   5 -
 .../METRON/CURRENT/themes/metron_theme.json | 198 +---
 .../roles/ambari_config/vars/single_node_vm.yml |   7 +-
 .../roles/ambari_config/vars/small_cluster.yml  |   5 +-
 metron-interface/metron-alerts/README.md|  22 +-
 .../metron-alerts/alerts-server-e2e.js  |  31 +-
 .../e2e/alerts-list/alerts-list.e2e-spec.ts |  17 +-
 .../configure-table/configure-table.e2e-spec.ts |  15 +-
 .../save-search/save-search.e2e-spec.ts |  11 -
 .../metron-alerts/e2e/login/login.e2e-spec.ts   |  44 ---
 .../metron-alerts/e2e/login/login.po.ts |  65 
 .../metron-alerts/e2e/utils/e2e_util.ts |  30 --
 .../metron-alerts/protractor.conf.js|  13 +-
 metron-interface/metron-alerts/proxy.conf.json  |   4 +-
 .../metron-alerts/scripts/alerts-server.js  |  12 +-
 .../scripts/start-server-for-e2e.sh |   3 +-
 .../metron-alerts/src/_variables.scss   |   3 -
 .../alert-details/alert-details.component.html  |   4 +-
 .../alert-details/alert-details.component.ts|  44 +--
 .../alert-details/alerts-details.routing.ts |   2 +-
 .../alerts-list/alerts-list.component.html  |   6 +-
 .../alerts/alerts-list/alerts-list.component.ts |  32 +-
 .../alerts/alerts-list/alerts-list.module.ts|   2 +-
 .../src/app/alerts/alerts-list/query-builder.ts |  18 +-
 .../metron-alerts/src/app/app-routing.module.ts |  12 +-
 .../metron-alerts/src/app/app.component.html|   5 +-
 .../metron-alerts/src/app/app.component.scss|  12 -
 .../metron-alerts/src/app/app.component.ts  |   8 -
 .../metron-alerts/src/app/app.module.ts |  14 +-
 .../src/app/login/login.component.html  |  28 --
 .../src/app/login/login.component.scss  |  55 
 .../src/app/login/login.component.spec.ts   |  65 
 .../src/app/login/login.component.ts|  43 ---
 .../metron-alerts/src/app/login/login.module.ts |  28 --
 .../src/app/login/login.routing.ts  |  25 --
 .../metron-alerts/src/app/model/alert-source.ts |  52 ---
 .../metron-alerts/src/app/model/alert.ts|  45 ++-
 .../src/app/model/search-request.ts |  12 +-
 .../metron-alerts/src/app/model/sort-field.ts   |  21 --
 .../src/app/service/alert.service.ts|  13 +-
 .../src/app/service/authentication.service.ts   |  91 --
 .../src/app/service/data-source.ts  |   5 +-
 .../service/elasticsearch-localstorage-impl.ts  |  11 +-
 .../src/app/service/rest-api-impl.ts|  46 ---
 .../metron-alerts/src/app/shared/auth-guard.ts  |  50 ---
 .../metron-alerts/src/app/shared/login-guard.ts |  40 ---
 .../src/app/utils/elasticsearch-utils.ts|   2 +-
 .../metron-alerts/src/app/utils/httpUtil.ts |   6 +-
 .../src/environments/environment.prod.ts|   3 +-
 .../transformation/StellarTransformation.java   |  11 -
 .../StellarTransformationTest.java  |  47 ---
 metron-platform/metron-parsers/README.md|  43

[1/3] metron git commit: otto(o...@apache.org) Revert "otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master"

[otto]

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1136-extensions-parsers 436466500 -> d6a66eb26


http://git-wip-us.apache.org/repos/asf/metron/blob/d6a66eb2/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index 0a3cbb0..12f8b5c 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -51,53 +51,6 @@ public class StellarTransformationTest {
 
   /** { "fieldTransformations" : [
 { "transformation" : "STELLAR"
-,"output" : [ "new_field", "new_field2", "old_field", "old_field2"]
-,"config" : {
-  "new_field" : "old_field"
- ,"new_field2" : "old_field2"
- ,"old_field" : "null"
- ,"old_field2" : "null"
-}
-}
-]
-  }
-   */
-  @Multiline
-  public static String configRename;
-
- @Test
- public void testStellarRename() throws Exception {
-
-   SensorParserConfig c = 
SensorParserConfig.fromBytes(Bytes.toBytes(configRename));
-   {
- JSONObject input = new JSONObject();
- input.put("old_field", "val");
- input.put("old_field2", "val2");
- for (FieldTransformer handler : c.getFieldTransformations()) {
-   handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
- }
- Assert.assertEquals(2, input.size());
- Assert.assertTrue(input.containsKey("new_field"));
- Assert.assertEquals("val", input.get("new_field"));
- Assert.assertEquals("val2", input.get("new_field2"));
- Assert.assertTrue(!input.containsKey("old_field"));
- Assert.assertTrue(!input.containsKey("old_field2"));
-   }
-   {
- JSONObject input = new JSONObject();
- input.put("old_field", "val");
- for (FieldTransformer handler : c.getFieldTransformations()) {
-   handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
- }
-
- Assert.assertEquals(1, input.size());
- Assert.assertTrue(input.containsKey("new_field"));
- Assert.assertEquals("val", input.get("new_field"));
-   }
- }
-
-  /** { "fieldTransformations" : [
-{ "transformation" : "STELLAR"
 ,"output" : [ "full_hostname", "domain_without_subdomains" ]
 ,"config" : {
   "full_hostname" : 
"URL_TO_HOST('http://1234567890123456789012345678901234567890123456789012345678901234567890/index.html')"

http://git-wip-us.apache.org/repos/asf/metron/blob/d6a66eb2/metron-platform/metron-parsers/README.md
--
diff --git a/metron-platform/metron-parsers/README.md 
b/metron-platform/metron-parsers/README.md
index ea4f1dd..3f8acbb 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -212,49 +212,6 @@ into `{ "protocol" : "TCP", "source.type" : "bro", ...}`
 * `STELLAR` : This transformation executes a set of transformations
   expressed as [Stellar Language](../metron-common) statements.
 
-### Assignment to `null`
-
-If, in your field transformation, you assign a field to `null`, the field will 
be removed.
-You can use this capability to rename variables.
-
-Consider this example:
-```
- "fieldTransformations" : [
- { "transformation" : "STELLAR"
- ,"output" : [ "new_field", "old_field"]
- ,"config" : {
-   "new_field" : "old_field"
-  ,"old_field" : "null"
- }
- }
- ]
-```
-This would set `new_field` to the value of `old_field` and remove `old_field`.
-
-### Warning: Transforming the same field twice
-
-Currently, the stellar expressions are expressed in the form of a map where 
the keys define
-the fields and the values define the Stellar expressions.  You order the 
expression evaluation
-in the `output` field.  A consequence of this choice to store the assignments 
as a map is that
-the same field cannot appear in the map as a key twice.
-
-For instance, the following will not function as expected:
-```
- "fieldTransformations" : [
- { "transformation" : "STELLAR"
- ,"output" : [ "new_field"]
- ,"config" : {
-   "new_field" : "TO_UPPER(field1)"
-  ,"new_field" : "TO_LOWER(new_field)"
- }
- }
- ]
-```
-
-In the above example, the last instance of `new_field` will win and 
`TO_LOWER(new_field)` will be evaluated
-while `TO_UPPER(field1)` will be skipped.
-
-### Example
 Consider the following sensor parser 

[2/3] metron git commit: otto(o...@apache.org) Revert "otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master"

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/d6a66eb2/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
--
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml 
b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
index 4c3cbce..482fafd 100644
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -31,7 +31,6 @@ hbase_slave: [HBASE_REGIONSERVER]
 es_master: [ES_MASTER]
 kibana_master: [KIBANA_MASTER]
 metron_indexing: [METRON_INDEXING]
-metron_profiler: [METRON_PROFILER]
 metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
 metron_parsers : [METRON_PARSERS]
 metron_rest: [METRON_REST]
@@ -39,8 +38,8 @@ metron_management_ui: [METRON_MANAGEMENT_UI]
 
 metron_components: >
   {{ hadoop_master | union(zookeeper_master) | union(storm_master) | 
union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
-  union(storm_slave) | union(kafka_broker) | union(hbase_slave) | 
union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
-  union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) 
| union(metron_management_ui) | union(es_master)  }}
+  union(storm_slave) | union(kafka_broker) | union(hbase_slave) | 
union(kibana_master) | union(metron_indexing) |
+  union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) 
| union(metron_management_ui) | union(es_master) }}
 
 cluster_name: "metron_cluster"
 blueprint_name: "metron_blueprint"
@@ -83,7 +82,7 @@ configurations:
   yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}'
   yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
   - storm-site:
-  supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704]"
+  supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
   storm.local.dir: '{{ storm_local_dir }}'
   topology.classpath: '{{ topology_classpath }}'
   - kafka-env:

http://git-wip-us.apache.org/repos/asf/metron/blob/d6a66eb2/metron-deployment/roles/ambari_config/vars/small_cluster.yml
--
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml 
b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
index 45b15f3..af22cb6 100644
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
@@ -31,7 +31,6 @@ es_master: [ES_MASTER]
 es_slave: [ES_SLAVE]
 kibana_master: [KIBANA_MASTER]
 metron_indexing: [METRON_INDEXING]
-metron_profiler: [METRON_PROFILER]
 metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
 metron_parsers : [METRON_PARSERS]
 metron_rest: [METRON_REST]
@@ -44,7 +43,7 @@ master_2_components: "{{ zookeeper_master | 
union(storm_master) | union(spark_ma
 master_2_host:
   - "{{groups.ambari_slave[1]}}"
 metron_components: >
-{{ metron_indexing | union(metron_profiler) | 
union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | 
union(metron_management_ui) | union(hadoop_slave) | union(storm_slave) |
+{{ metron_indexing | union(metron_enrichment_master) | 
union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | 
union(hadoop_slave) | union(storm_slave) |
 union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
 metron_host:
   - "{{ groups.metron[0] }}"
@@ -81,7 +80,7 @@ configurations:
   yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| 
default("/hadoop/yarn/log") }}'
   yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
   - storm-site:
-  supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704]"
+  supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
   storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
   topology.classpath: '{{ topology_classpath }}'
   - kafka-broker:

http://git-wip-us.apache.org/repos/asf/metron/blob/d6a66eb2/metron-interface/metron-alerts/README.md
--
diff --git a/metron-interface/metron-alerts/README.md 
b/metron-interface/metron-alerts/README.md
index db7f038..adba151 100644
--- a/metron-interface/metron-alerts/README.md
+++ b/metron-interface/metron-alerts/README.md
@@ -17,17 +17,15 @@
 
 ## Development Setup
 
-1. Install all the dependent node_modules using the following command
-```
-cd metron/metron-interface/metron-alerts
-npm install
-```
-1. UI can be run by using the following command
-```
-./scripts/start-dev.sh
-```
-1. You can view the GUI @http://localhost:4200 . The default credentials for 
login are admin/password
-
+Install all the dependent node_modules using the following command
+```
+cd metron/metron-interface/metron-alerts
+npm install
+```
+UI can be run by using the following command
+```
+./scripts/start-dev.sh
+```
 

[3/3] metron git commit: otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master

[otto]

otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/43646650
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/43646650
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/43646650

Branch: refs/heads/feature/METRON-1136-extensions-parsers
Commit: 4364665002756c1193cf834f3a2f53d774122760
Parents: ffcb91e
Author: ottofowler <>
Authored: Wed Aug 30 11:07:03 2017 -0400
Committer: otto <o...@apache.org>
Committed: Wed Aug 30 11:07:03 2017 -0400

--
 .gitignore  |   2 +-
 metron-analytics/metron-maas-common/pom.xml |  11 +
 .../metron/maas/functions/MaaSFunctions.java| 324 +++
 metron-analytics/metron-maas-service/README.md  |  21 +-
 metron-analytics/metron-maas-service/pom.xml|  11 -
 .../metron/maas/functions/MaaSFunctions.java| 324 ---
 .../org/apache/metron/maas/service/Client.java  |  15 +
 .../apache/metron/maas/service/Constants.java   |   5 +-
 .../metron/maas/service/runner/Runner.java  |   9 +-
 .../metron/maas/submit/ModelSubmission.java |   5 +
 .../METRON/CURRENT/role_command_order.json  |   2 +
 .../configuration/metron-indexing-env.xml   |   2 +-
 .../configuration/metron-profiler-env.xml   | 155 +
 .../common-services/METRON/CURRENT/metainfo.xml |  43 +++
 .../package/scripts/params/params_linux.py  |  25 ++
 .../package/scripts/params/status_params.py |  11 +
 .../package/scripts/profiler_commands.py| 194 +++
 .../CURRENT/package/scripts/profiler_master.py  |  94 ++
 .../package/templates/profiler.properties.j2|  47 +++
 .../METRON/CURRENT/service_advisor.py   |   5 +
 .../METRON/CURRENT/themes/metron_theme.json | 198 +++-
 .../roles/ambari_config/vars/single_node_vm.yml |   7 +-
 .../roles/ambari_config/vars/small_cluster.yml  |   5 +-
 metron-interface/metron-alerts/README.md|  22 +-
 .../metron-alerts/alerts-server-e2e.js  |  31 +-
 .../e2e/alerts-list/alerts-list.e2e-spec.ts |  17 +-
 .../configure-table/configure-table.e2e-spec.ts |  15 +-
 .../save-search/save-search.e2e-spec.ts |  11 +
 .../metron-alerts/e2e/login/login.e2e-spec.ts   |  44 +++
 .../metron-alerts/e2e/login/login.po.ts |  65 
 .../metron-alerts/e2e/utils/e2e_util.ts |  30 ++
 .../metron-alerts/protractor.conf.js|  13 +-
 metron-interface/metron-alerts/proxy.conf.json  |   4 +-
 .../metron-alerts/scripts/alerts-server.js  |  12 +-
 .../scripts/start-server-for-e2e.sh |   3 +-
 .../metron-alerts/src/_variables.scss   |   3 +
 .../alert-details/alert-details.component.html  |   4 +-
 .../alert-details/alert-details.component.ts|  44 ++-
 .../alert-details/alerts-details.routing.ts |   2 +-
 .../alerts-list/alerts-list.component.html  |   6 +-
 .../alerts/alerts-list/alerts-list.component.ts |  32 +-
 .../alerts/alerts-list/alerts-list.module.ts|   2 +-
 .../src/app/alerts/alerts-list/query-builder.ts |  18 +-
 .../metron-alerts/src/app/app-routing.module.ts |  12 +-
 .../metron-alerts/src/app/app.component.html|   5 +-
 .../metron-alerts/src/app/app.component.scss|  12 +
 .../metron-alerts/src/app/app.component.ts  |   8 +
 .../metron-alerts/src/app/app.module.ts |  14 +-
 .../src/app/login/login.component.html  |  28 ++
 .../src/app/login/login.component.scss  |  55 
 .../src/app/login/login.component.spec.ts   |  65 
 .../src/app/login/login.component.ts|  43 +++
 .../metron-alerts/src/app/login/login.module.ts |  28 ++
 .../src/app/login/login.routing.ts  |  25 ++
 .../metron-alerts/src/app/model/alert-source.ts |  52 +++
 .../metron-alerts/src/app/model/alert.ts|  45 +--
 .../src/app/model/search-request.ts |  12 +-
 .../metron-alerts/src/app/model/sort-field.ts   |  21 ++
 .../src/app/service/alert.service.ts|  13 +-
 .../src/app/service/authentication.service.ts   |  91 ++
 .../src/app/service/data-source.ts  |   5 +-
 .../service/elasticsearch-localstorage-impl.ts  |  11 +-
 .../src/app/service/rest-api-impl.ts|  46 +++
 .../metron-alerts/src/app/shared/auth-guard.ts  |  50 +++
 .../metron-alerts/src/app/shared/login-guard.ts |  40 +++
 .../src/app/utils/elasticsearch-utils.ts|   2 +-
 .../metron-alerts/src/app/utils/httpUtil.ts |   6 +-
 .../src/environments/environment.prod.ts|   3 +-
 .../transformation/StellarTransformation.java   |  11 +
 .../StellarTransformationTest.java  |  47 +++
 metron-platform/metron-parsers/README.md|  43 +++
 71 files changed, 2177 insertions(+), 5

[1/3] metron git commit: otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master

[otto]

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1136-extensions-parsers ffcb91ed0 -> 436466500


http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index 12f8b5c..0a3cbb0 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -51,6 +51,53 @@ public class StellarTransformationTest {
 
   /** { "fieldTransformations" : [
 { "transformation" : "STELLAR"
+,"output" : [ "new_field", "new_field2", "old_field", "old_field2"]
+,"config" : {
+  "new_field" : "old_field"
+ ,"new_field2" : "old_field2"
+ ,"old_field" : "null"
+ ,"old_field2" : "null"
+}
+}
+]
+  }
+   */
+  @Multiline
+  public static String configRename;
+
+ @Test
+ public void testStellarRename() throws Exception {
+
+   SensorParserConfig c = 
SensorParserConfig.fromBytes(Bytes.toBytes(configRename));
+   {
+ JSONObject input = new JSONObject();
+ input.put("old_field", "val");
+ input.put("old_field2", "val2");
+ for (FieldTransformer handler : c.getFieldTransformations()) {
+   handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+ }
+ Assert.assertEquals(2, input.size());
+ Assert.assertTrue(input.containsKey("new_field"));
+ Assert.assertEquals("val", input.get("new_field"));
+ Assert.assertEquals("val2", input.get("new_field2"));
+ Assert.assertTrue(!input.containsKey("old_field"));
+ Assert.assertTrue(!input.containsKey("old_field2"));
+   }
+   {
+ JSONObject input = new JSONObject();
+ input.put("old_field", "val");
+ for (FieldTransformer handler : c.getFieldTransformations()) {
+   handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+ }
+
+ Assert.assertEquals(1, input.size());
+ Assert.assertTrue(input.containsKey("new_field"));
+ Assert.assertEquals("val", input.get("new_field"));
+   }
+ }
+
+  /** { "fieldTransformations" : [
+{ "transformation" : "STELLAR"
 ,"output" : [ "full_hostname", "domain_without_subdomains" ]
 ,"config" : {
   "full_hostname" : 
"URL_TO_HOST('http://1234567890123456789012345678901234567890123456789012345678901234567890/index.html')"

http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-platform/metron-parsers/README.md
--
diff --git a/metron-platform/metron-parsers/README.md 
b/metron-platform/metron-parsers/README.md
index 3f8acbb..ea4f1dd 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -212,6 +212,49 @@ into `{ "protocol" : "TCP", "source.type" : "bro", ...}`
 * `STELLAR` : This transformation executes a set of transformations
   expressed as [Stellar Language](../metron-common) statements.
 
+### Assignment to `null`
+
+If, in your field transformation, you assign a field to `null`, the field will 
be removed.
+You can use this capability to rename variables.
+
+Consider this example:
+```
+ "fieldTransformations" : [
+ { "transformation" : "STELLAR"
+ ,"output" : [ "new_field", "old_field"]
+ ,"config" : {
+   "new_field" : "old_field"
+  ,"old_field" : "null"
+ }
+ }
+ ]
+```
+This would set `new_field` to the value of `old_field` and remove `old_field`.
+
+### Warning: Transforming the same field twice
+
+Currently, the stellar expressions are expressed in the form of a map where 
the keys define
+the fields and the values define the Stellar expressions.  You order the 
expression evaluation
+in the `output` field.  A consequence of this choice to store the assignments 
as a map is that
+the same field cannot appear in the map as a key twice.
+
+For instance, the following will not function as expected:
+```
+ "fieldTransformations" : [
+ { "transformation" : "STELLAR"
+ ,"output" : [ "new_field"]
+ ,"config" : {
+   "new_field" : "TO_UPPER(field1)"
+  ,"new_field" : "TO_LOWER(new_field)"
+ }
+ }
+ ]
+```
+
+In the above example, the last instance of `new_field` will win and 
`TO_LOWER(new_field)` will be evaluated
+while `TO_UPPER(field1)` will be skipped.
+
+### Example
 Consider the following sensor parser 

[2/3] metron git commit: otto(o...@apache.org) syncing feature/METRON-1136-extensions-parsers with master

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
--
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml 
b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
index 482fafd..4c3cbce 100644
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -31,6 +31,7 @@ hbase_slave: [HBASE_REGIONSERVER]
 es_master: [ES_MASTER]
 kibana_master: [KIBANA_MASTER]
 metron_indexing: [METRON_INDEXING]
+metron_profiler: [METRON_PROFILER]
 metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
 metron_parsers : [METRON_PARSERS]
 metron_rest: [METRON_REST]
@@ -38,8 +39,8 @@ metron_management_ui: [METRON_MANAGEMENT_UI]
 
 metron_components: >
   {{ hadoop_master | union(zookeeper_master) | union(storm_master) | 
union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
-  union(storm_slave) | union(kafka_broker) | union(hbase_slave) | 
union(kibana_master) | union(metron_indexing) |
-  union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) 
| union(metron_management_ui) | union(es_master) }}
+  union(storm_slave) | union(kafka_broker) | union(hbase_slave) | 
union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
+  union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) 
| union(metron_management_ui) | union(es_master)  }}
 
 cluster_name: "metron_cluster"
 blueprint_name: "metron_blueprint"
@@ -82,7 +83,7 @@ configurations:
   yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}'
   yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
   - storm-site:
-  supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
+  supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704]"
   storm.local.dir: '{{ storm_local_dir }}'
   topology.classpath: '{{ topology_classpath }}'
   - kafka-env:

http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-deployment/roles/ambari_config/vars/small_cluster.yml
--
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml 
b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
index af22cb6..45b15f3 100644
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
@@ -31,6 +31,7 @@ es_master: [ES_MASTER]
 es_slave: [ES_SLAVE]
 kibana_master: [KIBANA_MASTER]
 metron_indexing: [METRON_INDEXING]
+metron_profiler: [METRON_PROFILER]
 metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
 metron_parsers : [METRON_PARSERS]
 metron_rest: [METRON_REST]
@@ -43,7 +44,7 @@ master_2_components: "{{ zookeeper_master | 
union(storm_master) | union(spark_ma
 master_2_host:
   - "{{groups.ambari_slave[1]}}"
 metron_components: >
-{{ metron_indexing | union(metron_enrichment_master) | 
union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | 
union(hadoop_slave) | union(storm_slave) |
+{{ metron_indexing | union(metron_profiler) | 
union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | 
union(metron_management_ui) | union(hadoop_slave) | union(storm_slave) |
 union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
 metron_host:
   - "{{ groups.metron[0] }}"
@@ -80,7 +81,7 @@ configurations:
   yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| 
default("/hadoop/yarn/log") }}'
   yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
   - storm-site:
-  supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
+  supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704]"
   storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
   topology.classpath: '{{ topology_classpath }}'
   - kafka-broker:

http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-interface/metron-alerts/README.md
--
diff --git a/metron-interface/metron-alerts/README.md 
b/metron-interface/metron-alerts/README.md
index adba151..db7f038 100644
--- a/metron-interface/metron-alerts/README.md
+++ b/metron-interface/metron-alerts/README.md
@@ -17,15 +17,17 @@
 
 ## Development Setup
 
-Install all the dependent node_modules using the following command
-```
-cd metron/metron-interface/metron-alerts
-npm install
-```
-UI can be run by using the following command
-```
-./scripts/start-dev.sh
-```
+1. Install all the dependent node_modules using the following command
+```
+cd metron/metron-interface/metron-alerts
+npm install
+```
+1. UI can be run by using the following command
+```
+./scripts/start-dev.sh
+```
+1. You can view the GUI @http://localhost:4200 . The default credentials for 
login are admin/password
+
 

[38/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/src/main/java/org/apache/metron/bundles/BundleSystem.java
--
diff --git 
a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleSystem.java 
b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleSystem.java
new file mode 100644
index 000..7e93044
--- /dev/null
+++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleSystem.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.bundles;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.metron.bundles.bundle.Bundle;
+import org.apache.metron.bundles.util.BundleProperties;
+import org.apache.metron.bundles.util.FileSystemManagerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * High level interface to the Bundle System.  While you may want to use the 
lower level classes it
+ * is not required, as BundleSystem provides the base required interface for 
initializing the system
+ * and instantiating classes
+ */
+public class BundleSystem {
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  /**
+   * Builder for a BundleSystem. only {@link BundleProperties} are required. 
Beyond that, the
+   * BundleProperties, if they are the only parameter must have archive 
extension and bundle
+   * extension types properties present.
+   */
+  public static class Builder {
+
+private BundleProperties properties;
+private FileSystemManager fileSystemManager;
+private List extensionClasses = new LinkedList<>();
+private Bundle systemBundle;
+
+/**
+ * The BundleProperties to use.  Unless other builder parameters override 
options
+ * (withExtensionClasses ), they must have archive extension and bundle 
extensions types
+ * specified
+ *
+ * @param properties The BundleProperties
+ * @return Builder
+ */
+public Builder withBundleProperties(BundleProperties properties) {
+  this.properties = properties;
+  return this;
+}
+
+/**
+ * Provide a {@link FileSystemManager} to overide the default
+ *
+ * @param fileSystemManager override
+ * @return Builder
+ */
+public Builder withFileSystemManager(FileSystemManager fileSystemManager) {
+  this.fileSystemManager = fileSystemManager;
+  return this;
+}
+
+/**
+ * Provide Extension Classes.  If not provided with this override then the 
classes will be
+ * configured from the BundleProperties. If provided, the properties file 
will not be used for
+ * classes.
+ *
+ * @param extensionClasses override
+ * @return Builder
+ */
+public Builder withExtensionClasses(List extensionClasses) {
+  this.extensionClasses.addAll(extensionClasses);
+  return this;
+}
+
+/**
+ * Provide a SystemBundle.  If not provided with this override then the 
default SystemBundle
+ * will be created.
+ */
+public Builder withSystemBundle(Bundle systemBundle) {
+  this.systemBundle = systemBundle;
+  return this;
+}
+
+/**
+ * Builds a new BundleSystem.
+ *
+ * @return BundleSystem
+ * @throws NotInitializedException if any errors happen during build
+ */
+public BundleSystem build() throws NotInitializedException {
+  if (this.properties == null) {
+throw new IllegalArgumentException("BundleProperties are required");
+  }
+  try {
+if (this.fileSystemManager == null) {
+  this.fileSystemManager = FileSystemManagerFactory
+  .createFileSystemManager(new 
String[]{properties.getArchiveExtension()});
+}
+if (this.extensionClasses.isEmpty()) {
+  properties.getBundleExtensionTypes().forEach((x, y) -> {
+try {
+ 

[35/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-maven-plugin/src/main/java/org/apache/metron/maven/plugins/bundles/BundleProvidedDependenciesMojo.java
--
diff --git 
a/bundles-maven-plugin/src/main/java/org/apache/metron/maven/plugins/bundles/BundleProvidedDependenciesMojo.java
 
b/bundles-maven-plugin/src/main/java/org/apache/metron/maven/plugins/bundles/BundleProvidedDependenciesMojo.java
new file mode 100644
index 000..ca518b9
--- /dev/null
+++ 
b/bundles-maven-plugin/src/main/java/org/apache/metron/maven/plugins/bundles/BundleProvidedDependenciesMojo.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.maven.plugins.bundles;
+
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.handler.ArtifactHandler;
+import org.apache.maven.artifact.handler.manager.ArtifactHandlerManager;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+import org.apache.maven.plugins.annotations.Component;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.plugins.annotations.ResolutionScope;
+import org.apache.maven.project.DefaultProjectBuildingRequest;
+import org.apache.maven.project.MavenProject;
+import org.apache.maven.project.ProjectBuilder;
+import org.apache.maven.project.ProjectBuildingException;
+import org.apache.maven.project.ProjectBuildingRequest;
+import org.apache.maven.project.ProjectBuildingResult;
+import org.apache.maven.shared.dependency.tree.DependencyNode;
+import org.apache.maven.shared.dependency.tree.DependencyTreeBuilder;
+import org.apache.maven.shared.dependency.tree.DependencyTreeBuilderException;
+import org.apache.maven.shared.dependency.tree.traversal.DependencyNodeVisitor;
+import org.eclipse.aether.RepositorySystemSession;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Generates the listing of dependencies that is provided by the Bundle 
dependency of the current Bundle. This is important as artifacts that bundle 
dependencies will
+ * not project those dependences using the traditional maven dependency 
plugin. This plugin will override that setting in order to print the 
dependencies being
+ * inherited at runtime.
+ */
+@Mojo(name = "provided-bundle-dependencies", defaultPhase = 
LifecyclePhase.PACKAGE, threadSafe = false, requiresDependencyResolution = 
ResolutionScope.RUNTIME)
+public class BundleProvidedDependenciesMojo extends AbstractMojo {
+
+/**
+ * The Maven project.
+ */
+@Parameter(defaultValue = "${project}", readonly = true, required = true)
+private MavenProject project;
+
+/**
+ * The local artifact repository.
+ */
+@Parameter(defaultValue = "${localRepository}", readonly = true)
+private ArtifactRepository localRepository;
+
+/**
+ * The {@link RepositorySystemSession} used for obtaining the local and 
remote artifact repositories.
+ */
+@Parameter(defaultValue = "${repositorySystemSession}", readonly = true)
+private RepositorySystemSession repoSession;
+
+/**
+ * If specified, this parameter will cause the dependency tree to be 
written using the specified format. Currently supported format are: 
tree
+ * or pom.
+ */
+@Parameter(property = "mode", defaultValue = "tree")
+private String mode;
+
+/**
+ * The packageType we are using for dependencies, should be bundle, but may
+ * be changed in the configuration if the plugin is producing
+ * other archive extensions, this is a 'shared' configuration
+ * with the BundleMojo
+ */
+@Parameter(property = "packageType", required = false, defaultValue = 
"bundle")
+protected String packageType;
+
+/**
+ * The dependency tree builder to use for verbose output.
+ */
+@Component
+private 

[19/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/pom.xml
new file mode 100644
index 000..efa9b10
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/pom.xml
@@ -0,0 +1,36 @@
+
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+org.apache.metron
+metron-parser-extensions
+0.4.1
+
+org.apache.metron
+metron-parser-ise-extension
+metron-parser-ise-extension
+0.4.1
+pom
+
+
+ISE Parser Extension for Metron
+
+metron-parser-ise
+metron-parser-ise-bundle
+metron-parser-ise-assembly
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/pom.xml
new file mode 100644
index 000..b02cc85
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/pom.xml
@@ -0,0 +1,49 @@
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+
+org.apache.metron
+metron-parser-lancope-extension
+0.4.1
+
+
+org.apache.metron
+metron-parser-lancope-assembly
+0.4.1
+pom
+metron-parser-lancope-assembly
+
+
+
+
+maven-assembly-plugin
+
+src/main/assembly/assembly.xml
+
+
+
+make-assembly 
+package 
+
+single
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/src/main/assembly/assembly.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/src/main/assembly/assembly.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/src/main/assembly/assembly.xml
new file mode 100644
index 000..a287663
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-assembly/src/main/assembly/assembly.xml
@@ -0,0 +1,42 @@
+
+
+
+archive
+
+tar.gz
+
+false
+
+
+
${project.basedir}/../metron-parser-lancope/src/main/config
+/config
+true
+
+**/*.formatted
+**/*.filtered
+
+0644
+unix
+true
+
+
+
${project.basedir}/../metron-parser-lancope-bundle/target
+
+
metron-parser-lancope-bundle-${project.version}.bundle
+
+/lib
+true
+
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-bundle/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-bundle/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-bundle/pom.xml
new file mode 100644
index 000..b6d0a6d
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-lancope-extension/metron-parser-lancope-bundle/pom.xml
@@ -0,0 +1,41 @@
+
+http://maven.apache.org/POM/4.0.0; 

[31/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
index 36863e3..a9446ce 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
@@ -24,6 +24,7 @@ import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.metron.common.Constants;
 import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.extensions.ParserExtensionConfig;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.StellarFunctions;
 import org.apache.metron.common.utils.JSONUtils;
@@ -70,6 +71,17 @@ public class ConfigurationsUtils {
 writeToZookeeper(GLOBAL.getZookeeperRoot(), globalConfig, client);
   }
 
+  public static void writeGlobalBundlePropertiesToZookeeper(byte[] 
bundleProperties, String zookeeperUrl) throws Exception{
+try(CuratorFramework client = getClient(zookeeperUrl)){
+  client.start();
+  writeGlobalBundlePropertiesToZookeeper(bundleProperties,client);
+}
+  }
+
+  public static void writeGlobalBundlePropertiesToZookeeper(byte[] 
bundleProperties, CuratorFramework client) throws Exception{
+writeToZookeeper(Constants.ZOOKEEPER_ROOT + "/bundle.properties", 
bundleProperties, client);
+  }
+
   public static void writeProfilerConfigToZookeeper(byte[] config, 
CuratorFramework client) throws Exception {
 PROFILER.deserialize(new String(config));
 writeToZookeeper(PROFILER.getZookeeperRoot(), config, client);
@@ -92,10 +104,31 @@ public class ConfigurationsUtils {
 writeToZookeeper(PARSER.getZookeeperRoot() + "/" + sensorType, configData, 
client);
   }
 
+  public static void writeParserExtensionConfigToZookeeper(String extensionID, 
byte[] configData, CuratorFramework client) throws Exception {
+ParserExtensionConfig c = (ParserExtensionConfig) 
PARSER_EXTENSION.deserialize(new String(configData));
+writeToZookeeper(PARSER_EXTENSION.getZookeeperRoot() + "/" + extensionID, 
configData, client);
+  }
+
+  public static void writeParserExtensionConfigToZookeeper(String extensionID, 
ParserExtensionConfig parserExtensionConfig, String zookeeperUrl) throws 
Exception {
+writeParserExtensionConfigToZookeeper(extensionID, 
JSONUtils.INSTANCE.toJSON(parserExtensionConfig), zookeeperUrl);
+  }
+
+  public static void writeParserExtensionConfigToZookeeper(String extensionID, 
byte[] configData, String zookeeperUrl) throws Exception {
+try(CuratorFramework client = getClient(zookeeperUrl)) {
+  client.start();
+  writeParserExtensionConfigToZookeeper(extensionID, configData, client);
+}
+  }
+
+  public static void deleteParsesrExtensionConfig(String extensionID, 
CuratorFramework client)throws Exception{
+deleteFromZookeeper(PARSER_EXTENSION.getZookeeperRoot() + "/" + 
extensionID, client);
+  }
+
   public static void writeSensorIndexingConfigToZookeeper(String sensorType, 
Map sensorIndexingConfig, String zookeeperUrl) throws Exception 
{
 writeSensorIndexingConfigToZookeeper(sensorType, 
JSONUtils.INSTANCE.toJSON(sensorIndexingConfig), zookeeperUrl);
   }
 
+
   public static void writeSensorIndexingConfigToZookeeper(String sensorType, 
byte[] configData, String zookeeperUrl) throws Exception {
 try(CuratorFramework client = getClient(zookeeperUrl)) {
   client.start();
@@ -143,6 +176,10 @@ public class ConfigurationsUtils {
 }
   }
 
+  public static void deleteFromZookeeper(String path, CuratorFramework client) 
throws Exception{
+client.delete().forPath(path);
+  }
+
   public static void updateConfigsFromZookeeper(Configurations configurations, 
CuratorFramework client) throws Exception {
 
configurations.updateGlobalConfig(readGlobalConfigBytesFromZookeeper(client));
   }
@@ -179,6 +216,11 @@ public class ConfigurationsUtils {
 return JSONUtils.INSTANCE.load(new 
ByteArrayInputStream(readFromZookeeper(PARSER.getZookeeperRoot() + "/" + 
sensorType, client)), SensorParserConfig.class);
   }
 
+  public static ParserExtensionConfig 
readParserExtensionConfigFromZookeeper(String extensionID, CuratorFramework 
client) throws Exception {
+return JSONUtils.INSTANCE.load(new 
ByteArrayInputStream(readFromZookeeper(PARSER_EXTENSION.getZookeeperRoot() + 
"/" + extensionID, client)), ParserExtensionConfig.class);
+  }
+
+
   public static byte[] 

[08/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
--
diff --git 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
 
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
index af90e14..82cb8cb 100644
--- 
a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
+++ 
b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ConfigurationFunctions.java
@@ -71,6 +71,7 @@ public class ConfigurationFunctions {
 }
 CuratorFramework client = (CuratorFramework) clientOpt.get();
 TreeCache cache = new TreeCache(client, Constants.ZOOKEEPER_TOPOLOGY_ROOT);
+TreeCache exCache = new TreeCache(client, 
Constants.ZOOKEEPER_EXTENSIONS_ROOT);
 TreeCacheListener listener = new TreeCacheListener() {
   @Override
   public void childEvent(CuratorFramework client, TreeCacheEvent event) 
throws Exception {
@@ -91,6 +92,9 @@ public class ConfigurationFunctions {
   } else if 
(path.startsWith(ConfigurationType.INDEXING.getZookeeperRoot())) {
 Map sensorMap = (Map)configMap.get(ConfigurationType.INDEXING);
 sensorMap.put(sensor, new String(data));
+  } else if 
(path.startsWith(ConfigurationType.PARSER_EXTENSION.getZookeeperRoot())) {
+Map parserExtensionMap = 
(Map)configMap.get(ConfigurationType.PARSER_EXTENSION);
+parserExtensionMap.put(sensor,new String(data));
   }
 }
 else if(event.getType().equals(TreeCacheEvent.Type.NODE_REMOVED)) {
@@ -99,6 +103,9 @@ public class ConfigurationFunctions {
   if (path.startsWith(ConfigurationType.PARSER.getZookeeperRoot())) {
 Map sensorMap = (Map)configMap.get(ConfigurationType.PARSER);
 sensorMap.remove(sensor);
+  }else if 
(path.startsWith(ConfigurationType.PARSER_EXTENSION.getZookeeperRoot())) {
+Map parserExtensionMap = 
(Map)configMap.get(ConfigurationType.PARSER_EXTENSION);
+parserExtensionMap.remove(sensor);
   }
   else if 
(path.startsWith(ConfigurationType.ENRICHMENT.getZookeeperRoot())) {
 Map sensorMap = (Map)configMap.get(ConfigurationType.ENRICHMENT);
@@ -117,8 +124,32 @@ public class ConfigurationFunctions {
 }
   }
 };
+TreeCacheListener exListener = new TreeCacheListener() {
+  @Override
+  public void childEvent(CuratorFramework client, TreeCacheEvent event) 
throws Exception {
+if (event.getType().equals(TreeCacheEvent.Type.NODE_ADDED) || 
event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
+  String path = event.getData().getPath();
+  byte[] data = event.getData().getData();
+  String sensor = Iterables.getLast(Splitter.on("/").split(path), 
null);
+   if 
(path.startsWith(ConfigurationType.PARSER_EXTENSION.getZookeeperRoot())) {
+Map parserExtensionMap = 
(Map)configMap.get(ConfigurationType.PARSER_EXTENSION);
+parserExtensionMap.put(sensor,new String(data));
+  }
+}
+else if(event.getType().equals(TreeCacheEvent.Type.NODE_REMOVED)) {
+  String path = event.getData().getPath();
+  String sensor = Iterables.getLast(Splitter.on("/").split(path), 
null);
+  if 
(path.startsWith(ConfigurationType.PARSER_EXTENSION.getZookeeperRoot())) {
+Map parserExtensionMap = 
(Map)configMap.get(ConfigurationType.PARSER_EXTENSION);
+parserExtensionMap.remove(sensor);
+  }
+}
+  }
+};
 cache.getListenable().addListener(listener);
 cache.start();
+exCache.getListenable().addListener(exListener);
+exCache.start();
 for(ConfigurationType ct : ConfigurationType.values()) {
   switch(ct) {
 case GLOBAL:
@@ -146,17 +177,28 @@ public class ConfigurationFunctions {
 }
   }
   break;
+case PARSER_EXTENSION:
+{
+  List extensionIds = 
client.getChildren().forPath(ct.getZookeeperRoot());
+  Map parserExtensionMap = 
(Map)configMap.get(ct);
+  for (String extensionId : extensionIds){
+parserExtensionMap.put(extensionId, new 
String(ConfigurationsUtils.readFromZookeeper(ct.getZookeeperRoot() + "/" + 
extensionId,client)));
+  }
+}
+break;
   }
 }
 context.addCapability("treeCache", () -> cache);
+context.addCapability("exTreeCache",() -> exCache);
   

[04/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
deleted file mode 100644
index 7a5d2e6..000
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/fireeye/BasicFireEyeParserTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.fireeye;
-
-import java.util.Map;
-import java.util.Map.Entry;
-import java.time.Year;
-import java.time.ZonedDateTime;
-import java.time.ZoneOffset;
-
-import org.apache.metron.parsers.AbstractParserConfigTest;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BasicFireEyeParserTest extends AbstractParserConfigTest {
-
-  @Before
-  public void setUp() throws Exception {
-inputStrings = 
super.readTestDataFromFile("src/test/resources/logData/FireEyeParserTest.txt");
-parser = new BasicFireEyeParser();
-  }
-
-  @SuppressWarnings({"rawtypes"})
-  @Test
-  public void testParse() throws ParseException {
-for (String inputString : inputStrings) {
-  JSONObject parsed = parser.parse(inputString.getBytes()).get(0);
-  Assert.assertNotNull(parsed);
-
-  JSONParser parser = new JSONParser();
-
-  Map json = (Map) parser.parse(parsed.toJSONString());
-
-  Assert.assertNotNull(json);
-  Assert.assertFalse(json.isEmpty());
-
-  for (Object o : json.entrySet()) {
-Entry entry = (Entry) o;
-String key = (String) entry.getKey();
-String value = json.get(key).toString();
-Assert.assertNotNull(value);
-  }
-}
-  }
-
-  private final static String fireeyeMessage = "<164>Mar 19 05:24:39 
10.220.15.15 fenotify-851983.alert: 
CEF:0|FireEye|CMS|7.2.1.244420|DM|domain-match|1|rt=Feb 09 2015 12:28:26 UTC 
dvc=10.201.78.57 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=dev001srv02.example.com proto=udp cs5Label=cncHost cs5=mfdclk001.org 
dvchost=DEVFEYE1 spt=54527 dvc=10.100.25.16 smac=00:00:0c:07:ac:00 
cn1Label=vlan cn1=0 externalId=851983 cs4Label=link 
cs4=https://DEVCMS01.example.com/event_stream/events_for_bot?ev_id\\=851983 
dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Trojan.Generic.DNS";
-
-  @SuppressWarnings("rawtypes")
-  @Test
-  public void testTimestampParsing() throws ParseException {
-JSONObject parsed = parser.parse(fireeyeMessage.getBytes()).get(0);
-JSONParser parser = new JSONParser();
-Map json = (Map) parser.parse(parsed.toJSONString());
-long expectedTimestamp = 
ZonedDateTime.of(Year.now(ZoneOffset.UTC).getValue(), 3, 19, 5, 24, 39, 0, 
ZoneOffset.UTC).toInstant().toEpochMilli();
-Assert.assertEquals(expectedTimestamp, json.get("timestamp"));
-  }
-}

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/grok/GrokBuilderTest.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/grok/GrokBuilderTest.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/grok/GrokBuilderTest.java
new file mode 100644
index 000..57e206f
--- /dev/null
+++ 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/grok/GrokBuilderTest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements.  See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership.  The ASF licenses this file to you under the Apache 
License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the 
License.  You 

[39/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

METRON-1136 Metron Extensions System and Parser Extensions Feature Branch 
(ottobackwards) closes apache/metron#720


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ffcb91ed
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ffcb91ed
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ffcb91ed

Branch: refs/heads/feature/METRON-1136-extensions-parsers
Commit: ffcb91ed09c36d25008cfe57f06a9568d278f2f4
Parents: c8e84fa
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Wed Aug 30 11:03:32 2017 -0400
Committer: otto <o...@apache.org>
Committed: Wed Aug 30 11:03:32 2017 -0400

--
 .gitignore  |1 +
 .travis.yml |1 +
 README.md   |   11 +-
 bundles-lib/README.md   |  171 +++
 bundles-lib/pom.xml |  185 +++
 .../metron/bundles/BundleClassLoaders.java  |  353 ++
 .../apache/metron/bundles/BundleCloseable.java  |   86 ++
 .../metron/bundles/BundleManifestEntry.java |   48 +
 .../org/apache/metron/bundles/BundleMapper.java |  191 +++
 .../org/apache/metron/bundles/BundleSystem.java |  202 +++
 .../bundles/BundleThreadContextClassLoader.java |  211 
 .../apache/metron/bundles/ExtensionManager.java |  534 
 .../apache/metron/bundles/ExtensionMapping.java |  156 +++
 .../metron/bundles/InstanceClassLoader.java |  161 +++
 .../metron/bundles/NotInitializedException.java |   38 +
 .../metron/bundles/VFSBundleClassLoader.java|  520 
 .../bundles/VFSBundleClassLoaderResource.java   |  110 ++
 .../behavior/RequiresInstanceClassLoading.java  |   37 +
 .../apache/metron/bundles/bundle/Bundle.java|   48 +
 .../bundles/bundle/BundleCoordinates.java   |   96 ++
 .../metron/bundles/bundle/BundleDetails.java|  205 +++
 .../metron/bundles/util/BundleProperties.java   |  247 
 .../metron/bundles/util/BundleSelector.java |   48 +
 .../apache/metron/bundles/util/BundleUtil.java  |  145 +++
 .../metron/bundles/util/DummyFileObject.java|  230 
 .../bundles/util/FileSystemManagerFactory.java  |   87 ++
 .../apache/metron/bundles/util/FileUtils.java   |   35 +
 .../bundles/util/ImmutableCollectionUtils.java  |   65 +
 .../apache/metron/bundles/util/StringUtils.java |  118 ++
 .../org/apache/metron/bundles/AbstractFoo.java  |   24 +
 .../metron/bundles/BundleClassLoadersTest.java  |  110 ++
 .../apache/metron/bundles/BundleMapperTest.java |  180 +++
 .../apache/metron/bundles/BundleSystemTest.java |   56 +
 .../BundleThreadContextClassLoaderTest.java |  142 +++
 .../apache/metron/bundles/BundleUtilTest.java   |  125 ++
 .../bundles/ExtensionClassInitializerTest.java  |   33 +
 .../metron/bundles/ExtensionManagerTest.java|   82 ++
 .../BundleMapperIntegrationTest.java|  146 +++
 .../util/ImmutableCollectionUtilsTest.java  |   85 ++
 .../metron/bundles/util/ResourceCopier.java |   60 +
 .../apache/metron/bundles/util/TestUtil.java|   55 +
 .../parsers/interfaces/MessageParser.java   |   22 +
 .../BundleMapper/conf/bundle.properties |   21 +
 .../lib/metron-parser-bar-bundle-0.4.1.bundle   |  Bin 0 -> 21972 bytes
 .../lib2/metron-parser-foo-bundle-0.4.1.bundle  |  Bin 0 -> 21983 bytes
 .../org.apache.metron.bundles.AbstractFoo   |   16 +
 .../src/test/resources/bundle.properties|   22 +
 .../bundle-with-versioning/META-INF/MANIFEST.MF |   15 +
 .../META-INF/MANIFEST.MF|   12 +
 .../META-INF/MANIFEST.MF|7 +
 bundles-maven-plugin/LICENSE|  202 +++
 bundles-maven-plugin/NOTICE |5 +
 bundles-maven-plugin/README.md  |   89 ++
 bundles-maven-plugin/pom.xml|  328 +
 .../maven/plugins/bundles/BundleMojo.java   |  743 +++
 .../bundles/BundleProvidedDependenciesMojo.java |  328 +
 .../resources/META-INF/plexus/components.xml|   52 +
 metron-analytics/metron-maas-service/README.md  |2 +-
 .../METRON/CURRENT/configuration/metron-env.xml |   42 +
 .../common-services/METRON/CURRENT/metainfo.xml |   39 +
 .../CURRENT/package/scripts/metron_service.py   |   43 +-
 .../package/scripts/params/params_linux.py  |8 +
 .../package/scripts/params/status_params.py |9 +
 .../CURRENT/package/scripts/parser_commands.py  |   62 +-
 .../package/templates/bundle.properties.j2  |   21 +
 .../CURRENT/package/templates/global.json.j2|3 +-
 .../METRON/CURRENT/package/templates/metron.j2  |1 +
 .../docker/rpm-docker/SPECS/metron.spec |  498 +++-
 .../packaging/docker/rpm-docker/pom.xml |   79 ++
 .../sensor-config-single-parser.e2e-spec.ts |2 +-
 .../sensor-field-schema.component.spec.ts   | 

[25/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed
new file mode 100644
index 000..b1d3102
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed
@@ -0,0 +1,27 @@
+{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"CTo78A11g7CYbbOHvj","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP
 | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 
uri:\/ tags:[] uid:CTo78A11g7CYbbOHvj resp_mime_types:[\"text\\\/html\"] 
trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 
response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) 
libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 
ts:1402307733.473 id.resp_h:72.163.4.161 
resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0
 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 
libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15l
 
xUn5ngPfd"],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"}
+{"TTLs":[3600.0,289.0,14.0],"qclass_name":"C_INTERNET","bro_timestamp":"1402308259.609","qtype_name":"","ip_dst_port":53,"qtype":28,"rejected":false,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"trans_id":62418,"uid":"CuJT272SKaJSuqO0Ia","protocol":"dns","original_string":"DNS
 | AA:true TTLs:[3600.0,289.0,14.0] qclass_name:C_INTERNET id.orig_p:33976 
qtype_name: qtype:28 rejected:false id.resp_p:53 query:www.cisco.com 
answers:[\"www.cisco.com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"]
 trans_id:62418 rcode:0 rcode_name:NOERROR TC:false RA:true 
uid:CuJT272SKaJSuqO0Ia RD:true proto:udp id.orig_h:10.122.196.204 Z:0 qclass:1 
ts:1402308259.609 
id.resp_h:144.254.71.184","ip_dst_addr":"144.254.71.184","Z":0,"ip_src_addr":"10.122.196.204","qclass":1,"timestamp":1402308259609,"AA":true,"query":"www.cisco.com","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","RD":true,"ip_src_port":33976,"proto":"udp","gui
 d":"this-is-random-uuid-will-be-36-chars"}
+{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP
 | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 
uri:\/ tags:[] uid:KIRAN resp_mime_types:[\"text\\\/html\"] trans_depth:1 
host:www.cisco.com status_msg:OK id.orig_h:10.122.196.204 
response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) 
libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 
ts:1402307733.473 id.resp_h:72.163.4.161 
resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"10.122.196.204","user_agent":"curl\/7.22.0
 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 
libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"],"timestamp":14
 02307733473,"guid":"this-is-random-uuid-will-be-36-chars"}
+{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN12312312","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP
 | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 
uri:\/ tags:[] uid:KIRAN12312312 resp_mime_types:[\"text\\\/html\"] 
trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 
response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) 
libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 
ts:1402307733.473 id.resp_h:72.163.4.161 
resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0
 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 
libidn\/1.23 

[03/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
--
diff --git 
a/metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
 
b/metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
deleted file mode 100644
index f3be97a..000
--- 
a/metron-platform/metron-parsers/src/test/resources/logData/FireEyeParserTest.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-<164>Mar 19 05:24:39 10.220.15.15 fenotify-851983.alert: 
CEF:0|FireEye|CMS|7.2.1.244420|DM|domain-match|1|rt=Feb 09 2015 12:28:26 UTC 
dvc=10.201.78.57 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=dev001srv02.example.com proto=udp cs5Label=cncHost cs5=mfdclk001.org 
dvchost=DEVFEYE1 spt=54527 dvc=10.100.25.16 smac=00:00:0c:07:ac:00 
cn1Label=vlan cn1=0 externalId=851983 cs4Label=link 
cs4=https://DEVCMS01.example.com/event_stream/events_for_bot?ev_id\\=851983 
dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Trojan.Generic.DNS
-<164>Mar 19 05:24:39 10.220.15.15 fenotify-851987.alert: 
CEF:0|FireEye|CMS|7.2.1.244420|DM|domain-match|1|rt=Feb 09 2015 12:33:41 UTC 
dvc=10.201.78.113 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=dev001srv02.example.com proto=udp cs5Label=cncHost cs5=mfdclk001.org 
dvchost=DEVFEYE1 spt=51218 dvc=10.100.25.16 smac=00:00:0c:07:ac:00 
cn1Label=vlan cn1=0 externalId=851987 cs4Label=link 
cs4=https://DEVCMS01.example.com/event_stream/events_for_bot?ev_id\\=851987 
dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Trojan.Generic.DNS
-<164>Mar 19 05:24:39 10.220.15.15 fenotify-3483808.2.alert: 1::~~User-Agent: 
WinHttpClient::~~Host: www.microads.me::~~Connection: Keep-Alive::~~::~~GET 
/files/microads/update/InjectScript.js HTTP/1.1::~~User-Agent: 
WinHttpClient::~~Host: www.microads.me::~~Connection: Keep-Alive::~~::~~GET 
/files/microads/update/InjectScript.js HTTP/1.1::~~User-Agent: 
WinHttpClient::~~Host: www.microads.me::~~Connection: Keep-Alive::~~::~~GET 
/files/microads/update/InjectScript.js HTTP/1.1::~~User-Agent: 
WinHttpClient::~~Host: www.microads.me::~~Connection: Keep-Alive::~~::~~GET 
/files/microads/update/InjectScript.js HTTP/1.1::~~User-Agent: 
WinHttpClient::~~Host: www.microads.me::~~Connection: Keep-Alive::~~::~~GET 
/files/microads/update/InjectScript.js HTTP/1.1::~~User-Agent: 
WinHttpClient::~~Host: www.microads.me::~~Connection: Keep-Alive::~~::~~GET 
/files/microads/update/InjectScript.js HTTP/1.1::~~User-Agent: 
WinHttpClient::~~Host: www.microads.me::~~Connection: Keep-Alive::~~::~~GET 
/files/mic
 roads/update/InjectScript.js HTTP
-<164>Mar 19 05:24:39 10.220.15.15 fenotify-793972.2.alert: Control: 
no-cache::~~::~~ dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Exploit.Kit.Magnitude
-<161>Apr  1 05:24:39 10.220.15.15 fenotify-864461.alert: 
CEF:0|FireEye|CMS|7.5.1.318703|DM|domain-match|1|rt=Mar 19 2015 12:23:47 UTC 
src=10.191.193.20 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=abc123.example.com proto=udp spt=60903 cs5Label=cncHost cs5=mfdclk001.org 
dvchost=ABC123 dvc=10.190.1.16 smac=00:00:0c:07:ac:c8 cn1Label=vlan cn1=0 
externalId=864461 cs4Label=link 
cs4=https:\/\/ABC123.example.com\/event_stream\/events_for_bot?ev_id\\=864461 
act=notified dmac=88:43:e1:95:13:29 cs1Label=sname cs1=Trojan.Generic.DNS
-fireeye[-]: <161>Mar 19 05:24:39 10.220.15.15 fenotify-864461.alert: 
CEF:0|FireEye|CMS|7.5.1.318703|DM|domain-match|1|rt=Mar 19 2015 12:23:47 UTC 
src=10.191.193.20 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=abc123.example.com proto=udp spt=60903 cs5Label=cncHost cs5=mfdclk001.org 
dvchost=ABC123 dvc=10.190.1.16 smac=00:00:0c:07:ac:c8 cn1Label=vlan cn1=0 
externalId=864461 cs4Label=link 
cs4=https:\/\/ABC123.example.com\/event_stream\/events_for_bot?ev_id\\=864461 
act=notified dmac=88:43:e1:95:13:29 cs1Label=sname cs1=Trojan.Generic.DNS
-fireeye[-]: <161>Apr  1 02:49:49 10.220.15.15 fenotify-900702.alert: 
CEF:0|FireEye|CMS|7.5.1.318703|DM|domain-match|1|rt=Apr 01 2015 09:49:14 UTC 
src=10.1.97.20 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=abcd0060xzy03.example.com proto=udp spt=63100 cs5Label=cncHost 
cs5=mfdclk001.org dvchost=DEV1FEYE1 dvc=10.220.15.16 smac=00:00:0c:07:ac:00 
cn1Label=vlan cn1=0 externalId=900702 cs4Label=link 
cs4=https://ABCD0040CMS01.example.com/event_stream/events_for_bot?ev_id\=900702 
act=notified dmac=00:1d:a2:af:32:a1 cs1Label=sname cs1=Trojan.Generic.DNS
-<161>Apr 11 05:24:39 10.220.15.15 fenotify-864461.alert: 
CEF:0|FireEye|CMS|7.5.1.318703|DM|domain-match|1|rt=Mar 19 2015 12:23:47 UTC 
src=10.191.193.20 cn3Label=cncPort cn3=53 cn2Label=sid cn2=80494706 
shost=abc123.example.com proto=udp spt=60903 cs5Label=cncHost cs5=mfdclk001.org 
dvchost=ABC123 dvc=10.190.1.16 smac=00:00:0c:07:ac:c8 cn1Label=vlan cn1=0 
externalId=864461 cs4Label=link 
cs4=https:\/\/ABC123.example.com\/event_stream\/events_for_bot?ev_id\\=864461 
act=notified 

[37/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/src/main/java/org/apache/metron/bundles/bundle/BundleDetails.java
--
diff --git 
a/bundles-lib/src/main/java/org/apache/metron/bundles/bundle/BundleDetails.java 
b/bundles-lib/src/main/java/org/apache/metron/bundles/bundle/BundleDetails.java
new file mode 100644
index 000..7813775
--- /dev/null
+++ 
b/bundles-lib/src/main/java/org/apache/metron/bundles/bundle/BundleDetails.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bundles.bundle;
+
+import org.apache.commons.vfs2.FileObject;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import org.apache.commons.vfs2.FileSystemException;
+
+/**
+ * Metadata about a bundle. the coordinates and bundleFile properties are 
required
+ */
+public class BundleDetails {
+
+  private final FileObject bundleFile;
+
+  private final BundleCoordinates coordinates;
+  private final BundleCoordinates dependencyCoordinates;
+
+  private final String buildTag;
+  private final String buildRevision;
+  private final String buildBranch;
+  private final String buildTimestamp;
+  private final String buildJdk;
+  private final String builtBy;
+
+  private BundleDetails(final Builder builder) {
+this.bundleFile = builder.bundleFile;
+this.coordinates = builder.coordinates;
+this.dependencyCoordinates = builder.dependencyCoordinates;
+
+this.buildTag = builder.buildTag;
+this.buildRevision = builder.buildRevision;
+this.buildBranch = builder.buildBranch;
+this.buildTimestamp = builder.buildTimestamp;
+this.buildJdk = builder.buildJdk;
+this.builtBy = builder.builtBy;
+
+if (this.coordinates == null) {
+  if (this.bundleFile == null) {
+throw new IllegalStateException("Coordinate cannot be null");
+  } else {
+throw new IllegalStateException(
+"Coordinate cannot be null for " + this.bundleFile.getName());
+  }
+}
+
+if (this.bundleFile == null) {
+  throw new IllegalStateException("bundleFile cannot be null for " + 
this.coordinates
+  .getId());
+}
+  }
+
+  public FileObject getBundleFile() {
+return bundleFile;
+  }
+
+  public BundleCoordinates getCoordinates() {
+return coordinates;
+  }
+
+  public BundleCoordinates getDependencyCoordinates() {
+return dependencyCoordinates;
+  }
+
+  public String getBuildTag() {
+return buildTag;
+  }
+
+  public String getBuildRevision() {
+return buildRevision;
+  }
+
+  public String getBuildBranch() {
+return buildBranch;
+  }
+
+  public String getBuildTimestamp() {
+return buildTimestamp;
+  }
+
+  public String getBuildJdk() {
+return buildJdk;
+  }
+
+  public String getBuiltBy() {
+return builtBy;
+  }
+
+  @Override
+  public String toString() {
+return coordinates.toString();
+  }
+
+  public Date getBuildTimestampDate() {
+if (buildTimestamp != null && !buildTimestamp.isEmpty()) {
+  try {
+SimpleDateFormat buildTimestampFormat = new 
SimpleDateFormat("-MM-dd'T'HH:mm:ss'Z'");
+Date buildTimestampDate = buildTimestampFormat.parse(buildTimestamp);
+return buildTimestampDate;
+  } catch (ParseException parseEx) {
+return null;
+  }
+} else {
+  return null;
+}
+  }
+
+  /**
+   * Builder for BundleDetails. The withCoordinates and withBundleFile 
properties are required
+   */
+  public static class Builder {
+
+private FileObject bundleFile;
+
+private BundleCoordinates coordinates;
+private BundleCoordinates dependencyCoordinates;
+
+private String buildTag;
+private String buildRevision;
+private String buildBranch;
+private String buildTimestamp;
+private String buildJdk;
+private String builtBy;
+
+public Builder withBundleFile(final FileObject bundleFile) {
+  this.bundleFile = bundleFile;
+  return this;
+}
+
+public Builder withCoordinates(final BundleCoordinates coordinates) {
+  this.coordinates = coordinates;
+  return this;
+}
+
+public Builder 

[28/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml
new file mode 100644
index 000..616bb24
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml
@@ -0,0 +1,36 @@
+
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+org.apache.metron
+metron-parser-extensions
+0.4.1
+
+org.apache.metron
+metron-parser-asa-extension
+metron-parser-asa-extension
+0.4.1
+pom
+
+
+ASA Parser Extension for Metron
+
+metron-parser-asa
+metron-parser-asa-bundle
+metron-parser-asa-assembly
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml
new file mode 100644
index 000..0a83e12
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml
@@ -0,0 +1,49 @@
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+
+org.apache.metron
+metron-parser-bro-extension
+0.4.1
+
+
+org.apache.metron
+metron-parser-bro-assembly
+0.4.1
+pom
+metron-parser-bro-assembly
+
+
+
+
+maven-assembly-plugin
+
+src/main/assembly/assembly.xml
+
+
+
+make-assembly 
+package 
+
+single
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml
new file mode 100644
index 000..d292a2d
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml
@@ -0,0 +1,42 @@
+
+
+
+archive
+
+tar.gz
+
+false
+
+
+
${project.basedir}/../metron-parser-bro/src/main/config
+/config
+true
+
+**/*.formatted
+**/*.filtered
+
+0644
+unix
+true
+
+
+
${project.basedir}/../metron-parser-bro-bundle/target
+
+
metron-parser-bro-bundle-${project.version}.bundle
+
+/lib
+true
+
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml
new file mode 100644
index 000..26d8b6e
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml
@@ -0,0 +1,41 @@
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 

[09/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed
--
diff --git 
a/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed
 
b/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed
deleted file mode 100644
index b1d3102..000
--- 
a/metron-platform/metron-integration-test/src/main/sample/data/bro/parsed/BroExampleParsed
+++ /dev/null
@@ -1,27 +0,0 @@
-{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"CTo78A11g7CYbbOHvj","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP
 | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 
uri:\/ tags:[] uid:CTo78A11g7CYbbOHvj resp_mime_types:[\"text\\\/html\"] 
trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 
response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) 
libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 
ts:1402307733.473 id.resp_h:72.163.4.161 
resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0
 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 
libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15l
 
xUn5ngPfd"],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"}
-{"TTLs":[3600.0,289.0,14.0],"qclass_name":"C_INTERNET","bro_timestamp":"1402308259.609","qtype_name":"","ip_dst_port":53,"qtype":28,"rejected":false,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"trans_id":62418,"uid":"CuJT272SKaJSuqO0Ia","protocol":"dns","original_string":"DNS
 | AA:true TTLs:[3600.0,289.0,14.0] qclass_name:C_INTERNET id.orig_p:33976 
qtype_name: qtype:28 rejected:false id.resp_p:53 query:www.cisco.com 
answers:[\"www.cisco.com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"]
 trans_id:62418 rcode:0 rcode_name:NOERROR TC:false RA:true 
uid:CuJT272SKaJSuqO0Ia RD:true proto:udp id.orig_h:10.122.196.204 Z:0 qclass:1 
ts:1402308259.609 
id.resp_h:144.254.71.184","ip_dst_addr":"144.254.71.184","Z":0,"ip_src_addr":"10.122.196.204","qclass":1,"timestamp":1402308259609,"AA":true,"query":"www.cisco.com","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","RD":true,"ip_src_port":33976,"proto":"udp","gui
 d":"this-is-random-uuid-will-be-36-chars"}
-{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP
 | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 
uri:\/ tags:[] uid:KIRAN resp_mime_types:[\"text\\\/html\"] trans_depth:1 
host:www.cisco.com status_msg:OK id.orig_h:10.122.196.204 
response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) 
libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 
ts:1402307733.473 id.resp_h:72.163.4.161 
resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"10.122.196.204","user_agent":"curl\/7.22.0
 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 
libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"],"timestamp":14
 02307733473,"guid":"this-is-random-uuid-will-be-36-chars"}
-{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN12312312","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP
 | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 
uri:\/ tags:[] uid:KIRAN12312312 resp_mime_types:[\"text\\\/html\"] 
trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 
response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) 
libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 
ts:1402307733.473 id.resp_h:72.163.4.161 
resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0
 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 
libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"
 ],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"}

[30/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
new file mode 100644
index 000..8bc254c
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.asa;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.lang.invoke.MethodHandles;
+import java.time.Clock;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.metron.common.Constants;
+import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.ParseException;
+import org.apache.metron.parsers.grok.GrokBuilder;
+import org.apache.metron.parsers.utils.SyslogUtils;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class BasicAsaParser extends BasicParser {
+
+  protected static final Logger LOG = 
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  protected Clock deviceClock;
+  private String syslogPattern = "CISCO_TAGGED_SYSLOG";
+
+  private Grok syslogGrok;
+  private Map parserConfiguration;
+
+  private static final Map patternMap = ImmutableMap. builder()
+  .put("ASA-2-106001", "CISCOFW106001")
+  .put("ASA-2-106006", "CISCOFW106006_106007_106010")
+  .put("ASA-2-106007", "CISCOFW106006_106007_106010")
+  .put("ASA-2-106010", "CISCOFW106006_106007_106010")
+  .put("ASA-3-106014", "CISCOFW106014")
+  .put("ASA-6-106015", "CISCOFW106015")
+  .put("ASA-1-106021", "CISCOFW106021")
+  .put("ASA-4-106023", "CISCOFW106023")
+  .put("ASA-5-106100", "CISCOFW106100")
+  .put("ASA-6-110002", "CISCOFW110002")
+  .put("ASA-6-302010", "CISCOFW302010")
+  .put("ASA-6-302013", "CISCOFW302013_302014_302015_302016")
+  .put("ASA-6-302014", "CISCOFW302013_302014_302015_302016")
+  .put("ASA-6-302015", "CISCOFW302013_302014_302015_302016")
+  .put("ASA-6-302016", "CISCOFW302013_302014_302015_302016")
+  .put("ASA-6-302020", "CISCOFW302020_302021")
+  .put("ASA-6-302021", "CISCOFW302020_302021")
+  .put("ASA-6-305011", "CISCOFW305011")
+  .put("ASA-3-313001", "CISCOFW313001_313004_313008")
+  .put("ASA-3-313004", "CISCOFW313001_313004_313008")
+  .put("ASA-3-313008", "CISCOFW313001_313004_313008")
+  .put("ASA-4-313005", "CISCOFW313005")
+  .put("ASA-4-402117", "CISCOFW402117")
+  .put("ASA-4-402119", "CISCOFW402119")
+  .put("ASA-4-419001", "CISCOFW419001")
+  .put("ASA-4-419002", "CISCOFW419002")
+  .put("ASA-4-54", "CISCOFW54")
+  .put("ASA-6-602303", "CISCOFW602303_602304")
+  .put("ASA-6-602304", "CISCOFW602303_602304")
+  .put("ASA-7-710001", "CISCOFW710001_710002_710003_710005_710006")
+  .put("ASA-7-710002", 

[36/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/src/test/java/org/apache/metron/bundles/integration/BundleMapperIntegrationTest.java
--
diff --git 
a/bundles-lib/src/test/java/org/apache/metron/bundles/integration/BundleMapperIntegrationTest.java
 
b/bundles-lib/src/test/java/org/apache/metron/bundles/integration/BundleMapperIntegrationTest.java
new file mode 100644
index 000..977afa7
--- /dev/null
+++ 
b/bundles-lib/src/test/java/org/apache/metron/bundles/integration/BundleMapperIntegrationTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bundles.integration;
+
+import static org.apache.metron.bundles.util.TestUtil.loadSpecifiedProperties;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.metron.bundles.BundleClassLoaders;
+import org.apache.metron.bundles.BundleMapper;
+import org.apache.metron.bundles.ExtensionManager;
+import org.apache.metron.bundles.ExtensionMapping;
+import org.apache.metron.bundles.bundle.Bundle;
+import org.apache.metron.bundles.util.BundleProperties;
+import org.apache.metron.bundles.util.FileSystemManagerFactory;
+import org.apache.metron.integration.components.MRComponent;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class BundleMapperIntegrationTest {
+
+  static final Map EMPTY_MAP = new HashMap();
+  static MRComponent component;
+  static Configuration configuration;
+  static FileSystem fileSystem;
+
+
+  @AfterClass
+  public static void after() {
+ExtensionManager.reset();
+BundleClassLoaders.reset();
+  }
+
+  @After
+  public void afterTest() {
+ExtensionManager.reset();
+BundleClassLoaders.reset();
+  }
+  @BeforeClass
+  public static void setup() {
+ExtensionManager.reset();
+BundleClassLoaders.reset();
+component = new MRComponent().withBasePath("target/hdfs");
+component.start();
+configuration = component.getConfiguration();
+
+try {
+  fileSystem = FileSystem.newInstance(configuration);
+  fileSystem.mkdirs(new Path("/work/"),
+  new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, 
FsAction.READ_WRITE));
+  fileSystem.copyFromLocalFile(new 
Path("./src/test/resources/bundle.properties"),
+  new Path("/work/"));
+  fileSystem
+  .copyFromLocalFile(new 
Path("./src/test/resources/BundleMapper/lib/"), new Path("/"));
+  fileSystem
+  .copyFromLocalFile(new 
Path("./src/test/resources/BundleMapper/lib2/"), new Path("/"));
+  RemoteIterator files = fileSystem.listFiles(new 
Path("/"), true);
+  System.out.println("==(BEFORE)==");
+  while (files.hasNext()) {
+LocatedFileStatus fileStat = files.next();
+System.out.println(fileStat.getPath().toString());
+  }
+} catch (IOException e) {
+  throw new RuntimeException("Unable to start cluster", e);
+}
+  }
+
+  @AfterClass
+  public static void teardown() {
+try {
+  RemoteIterator files = fileSystem.listFiles(new 
Path("/"), true);
+  System.out.println("==(AFTER)==");
+  while (files.hasNext()) {
+LocatedFileStatus fileStat = files.next();
+System.out.println(fileStat.getPath().toString());
+  }
+} catch (Exception e) {
+}
+component.stop();
+BundleClassLoaders.reset();
+  }
+
+  @Test
+  public void 

[16/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-snort-extension/metron-parser-snort/src/main/resources/META-INF/LICENSE
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-snort-extension/metron-parser-snort/src/main/resources/META-INF/LICENSE
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-snort-extension/metron-parser-snort/src/main/resources/META-INF/LICENSE
new file mode 100644
index 000..849fa54
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-snort-extension/metron-parser-snort/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,602 @@
+Apache License
+   Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+  "License" shall mean the terms and conditions for use, reproduction,
+  and distribution as defined by Sections 1 through 9 of this document.
+
+  "Licensor" shall mean the copyright owner or entity authorized by
+  the copyright owner that is granting the License.
+
+  "Legal Entity" shall mean the union of the acting entity and all
+  other entities that control, are controlled by, or are under common
+  control with that entity. For the purposes of this definition,
+  "control" means (i) the power, direct or indirect, to cause the
+  direction or management of such entity, whether by contract or
+  otherwise, or (ii) ownership of fifty percent (50%) or more of the
+  outstanding shares, or (iii) beneficial ownership of such entity.
+
+  "You" (or "Your") shall mean an individual or Legal Entity
+  exercising permissions granted by this License.
+
+  "Source" form shall mean the preferred form for making modifications,
+  including but not limited to software source code, documentation
+  source, and configuration files.
+
+  "Object" form shall mean any form resulting from mechanical
+  transformation or translation of a Source form, including but
+  not limited to compiled object code, generated documentation,
+  and conversions to other media types.
+
+  "Work" shall mean the work of authorship, whether in Source or
+  Object form, made available under the License, as indicated by a
+  copyright notice that is included in or attached to the work
+  (an example is provided in the Appendix below).
+
+  "Derivative Works" shall mean any work, whether in Source or Object
+  form, that is based on (or derived from) the Work and for which the
+  editorial revisions, annotations, elaborations, or other modifications
+  represent, as a whole, an original work of authorship. For the purposes
+  of this License, Derivative Works shall not include works that remain
+  separable from, or merely link (or bind by name) to the interfaces of,
+  the Work and Derivative Works thereof.
+
+  "Contribution" shall mean any work of authorship, including
+  the original version of the Work and any modifications or additions
+  to that Work or Derivative Works thereof, that is intentionally
+  submitted to Licensor for inclusion in the Work by the copyright owner
+  or by an individual or Legal Entity authorized to submit on behalf of
+  the copyright owner. For the purposes of this definition, "submitted"
+  means any form of electronic, verbal, or written communication sent
+  to the Licensor or its representatives, including but not limited to
+  communication on electronic mailing lists, source code control systems,
+  and issue tracking systems that are managed by, or on behalf of, the
+  Licensor for the purpose of discussing and improving the Work, but
+  excluding communication that is conspicuously marked or otherwise
+  designated in writing by the copyright owner as "Not a Contribution."
+
+  "Contributor" shall mean Licensor and any individual or Legal Entity
+  on behalf of whom a Contribution has been received by Licensor and
+  subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  copyright license to reproduce, prepare Derivative Works of,
+  publicly display, publicly perform, sublicense, and distribute the
+  Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  (except 

[06/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
--
diff --git 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
 
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
deleted file mode 100644
index 2ccc23a..000
--- 
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 
*/
-/* JavaCCOptions: */
-package org.apache.metron.parsers.ise;
-
-/** Token Manager Error. */
-class TokenMgrError extends Error
-{
-
-  /**
-   * The version identifier for this Serializable class.
-   * Increment only if the serialized form of the
-   * class changes.
-   */
-  private static final long serialVersionUID = 1L;
-
-  /*
-   * Ordinals for various reasons why an Error of this type can be thrown.
-   */
-
-  /**
-   * Lexical error occurred.
-   */
-  static final int LEXICAL_ERROR = 0;
-
-  /**
-   * An attempt was made to create a second instance of a static token manager.
-   */
-  static final int STATIC_LEXER_ERROR = 1;
-
-  /**
-   * Tried to change to an invalid lexical state.
-   */
-  static final int INVALID_LEXICAL_STATE = 2;
-
-  /**
-   * Detected (and bailed out of) an infinite loop in the token manager.
-   */
-  static final int LOOP_DETECTED = 3;
-
-  /**
-   * Indicates the reason why the exception is thrown. It will have
-   * one of the above 4 values.
-   */
-  int errorCode;
-
-  /**
-   * Replaces unprintable characters by their escaped (or unicode escaped)
-   * equivalents in the given string
-   */
-  protected static final String addEscapes(String str) {
-StringBuffer retval = new StringBuffer();
-char ch;
-for (int i = 0; i < str.length(); i++) {
-  switch (str.charAt(i))
-  {
-case 0 :
-  continue;
-case '\b':
-  retval.append("\\b");
-  continue;
-case '\t':
-  retval.append("\\t");
-  continue;
-case '\n':
-  retval.append("\\n");
-  continue;
-case '\f':
-  retval.append("\\f");
-  continue;
-case '\r':
-  retval.append("\\r");
-  continue;
-case '\"':
-  retval.append("\\\"");
-  continue;
-case '\'':
-  retval.append("\\\'");
-  continue;
-case '\\':
-  retval.append("");
-  continue;
-default:
-  if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
-String s = "" + Integer.toString(ch, 16);
-retval.append("\\u" + s.substring(s.length() - 4, s.length()));
-  } else {
-retval.append(ch);
-  }
-  continue;
-  }
-}
-return retval.toString();
-  }
-
-  /**
-   * Returns a detailed message for the Error when it is thrown by the
-   * token manager to indicate a lexical error.
-   * Parameters :
-   *EOFSeen : indicates if EOF caused the lexical error
-   *curLexState : lexical state in which this error occurred
-   *errorLine   : line number when the error occurred
-   *errorColumn : column number when the error occurred
-   *errorAfter  : prefix that was seen before this error occurred
-   *curchar : the offending character
-   * Note: You can customize the lexical error message by modifying this 
method.
-   */
-  protected static String LexicalError(boolean EOFSeen, int lexState, int 
errorLine, int errorColumn, String errorAfter, char curChar) {
-return("Lexical error at line " +
-  errorLine + ", column " +
-  errorColumn + ".  Encountered: " 

[34/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
--
diff --git 
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
 
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
index 6c4eab1..e89bd8a 100644
--- 
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
+++ 
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
@@ -97,7 +97,7 @@ class MockSensorParserConfigService extends 
SensorParserConfigService {
 return Observable.create(observer => {
   observer.next({
 'Bro': 'org.apache.metron.parsers.bro.BasicBroParser',
-'Grok': 'org.apache.metron.parsers.GrokParser'
+'Grok': 'org.apache.metron.parsers.grok.GrokParser'
   });
   observer.complete();
 });
@@ -251,6 +251,20 @@ class MockGrokValidationService extends 
GrokValidationService {
 this.contents = contents;
   }
 
+  public save(path: string, contents: string): Observable<{}> {
+if (this.contents === null) {
+  let error = new RestError();
+  error.message = 'HDFS post Error';
+  return Observable.throw(error);
+}
+this.path = path;
+this.contents = contents;
+return Observable.create(observer => {
+  observer.next(this.contents);
+  observer.complete();
+});
+  }
+
   public list(): Observable {
 return Observable.create(observer => {
   observer.next({
@@ -451,7 +465,7 @@ describe('Component: SensorParserConfig', () => {
   let router: MockRouter;
 
   let squidSensorParserConfig: any = {
-'parserClassName': 'org.apache.metron.parsers.GrokParser',
+'parserClassName': 'org.apache.metron.parsers.grok.GrokParser',
 'sensorTopic': 'squid',
 'parserConfig': {
   'grokPath': '/apps/metron/patterns/squid',
@@ -586,7 +600,7 @@ describe('Component: SensorParserConfig', () => {
 component.getAvailableParsers();
 expect(component.availableParsers).toEqual({
   'Bro': 'org.apache.metron.parsers.bro.BasicBroParser',
-  'Grok': 'org.apache.metron.parsers.GrokParser'
+  'Grok': 'org.apache.metron.parsers.grok.GrokParser'
 });
 expect(component.availableParserNames).toEqual(['Bro', 'Grok']);
 
@@ -597,7 +611,7 @@ describe('Component: SensorParserConfig', () => {
 component.init('new');
 
 let expectedSensorParserConfig = new SensorParserConfig();
-expectedSensorParserConfig.parserClassName = 
'org.apache.metron.parsers.GrokParser';
+expectedSensorParserConfig.parserClassName = 
'org.apache.metron.parsers.grok.GrokParser';
 expect(component.sensorParserConfig).toEqual(expectedSensorParserConfig);
 expect(component.sensorEnrichmentConfig).toEqual(new 
SensorEnrichmentConfig());
 expect(component.indexingConfigurations).toEqual(new 
IndexingConfigurations());
@@ -610,7 +624,7 @@ describe('Component: SensorParserConfig', () => {
 Object.assign(new SensorEnrichmentConfig(), 
squidSensorEnrichmentConfig));
 sensorIndexingConfigService.setSensorIndexingConfig('squid',
 Object.assign(new IndexingConfigurations(), 
squidIndexingConfigurations));
-hdfsService.setContents('/apps/metron/patterns/squid', 'SQUID_DELIMITED 
grok statement');
+grokValidationService.setContents('/apps/metron/patterns/squid', 
'SQUID_DELIMITED grok statement');
 
 component.init('squid');
 expect(component.sensorParserConfig).toEqual(Object.assign(new 
SensorParserConfig(), squidSensorParserConfig));
@@ -688,7 +702,7 @@ describe('Component: SensorParserConfig', () => {
 expect(component.hidePane).not.toHaveBeenCalled();
 expect(component.isConfigValid).toHaveBeenCalled();
 
-component.sensorParserConfig.parserClassName = 
'org.apache.metron.parsers.GrokParser';
+component.sensorParserConfig.parserClassName = 
'org.apache.metron.parsers.grok.GrokParser';
 component.onParserTypeChange();
 expect(component.parserClassValid).toEqual(true);
 expect(component.hidePane).not.toHaveBeenCalled();
@@ -724,7 +738,7 @@ describe('Component: SensorParserConfig', () => {
 component.isConfigValid();
 expect(component.configValid).toEqual(true);
 
-component.sensorParserConfig.parserClassName = 
'org.apache.metron.parsers.GrokParser';
+component.sensorParserConfig.parserClassName = 
'org.apache.metron.parsers.grok.GrokParser';
 component.isConfigValid();
 expect(component.configValid).toEqual(false);
 
@@ -774,11 +788,11 @@ describe('Component: SensorParserConfig', () => {
 
 component.onSaveGrokStatement('grok statement');
 expect(component.grokStatement).toEqual('grok statement');
-

[20/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/test/resources/logData/IseParserTest.txt
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/test/resources/logData/IseParserTest.txt
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/test/resources/logData/IseParserTest.txt
new file mode 100644
index 000..54cf4ff
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-ise-extension/metron-parser-ise/src/test/resources/logData/IseParserTest.txt
@@ -0,0 +1,308 @@
+Aug  6 17:26:31 10.34.84.145 Aug  7 00:45:43 stage-pdp01 CISE_Profiler 
024855 1 0 2014-08-07 00:45:43.741 -07:00 288542 80002 INFO  Profiler: 
Profiler EndPoint profiling event occurred, ConfigVersionId=113, 
EndpointCertainityMetric=10, EndpointIPAddress=10.56.111.14, 
EndpointMacAddress=3C:97:0E:C3:F8:F1, EndpointMatchedPolicy=Nortel-Device, 
EndpointNADAddress=10.56.72.127, EndpointOUI=Wistron 
InfoComm(Kunshan)Co.\,Ltd., EndpointPolicy=Nortel-Device, 
EndpointProperty=StaticAssignment=false\,PostureApplicable=Yes\,PolicyVersion=402\,IdentityGroupID=0c1d9270-68a6-11e1-bc72-0050568e013c\,Total
 Certainty 
Factor=10\,BYODRegistration=Unknown\,FeedService=false\,EndPointPolicyID=49054ed0-68a6-11e1-bc72-0050568e013c\,FirstCollection=1407397543718\,MatchedPolicyID=49054ed0-68a6-11e1-bc72-0050568e013c\,TimeToProfile=19\,StaticGroupAssignment=false\,NmapSubnetScanID=0\,DeviceRegistrationStatus=NotRegistered\,PortalUser=,
 EndpointSourceEvent=SNMPQuery Probe, EndpointIdentityGroup=Profile
 d, ProfilerServer=stage-pdp01.cisco.com,
+Aug  6 17:26:31 10.34.84.145 Aug  7 00:45:43 stage-pdp01 CISE_Profiler 
024856 1 0 2014-08-07 00:45:43.786 -07:00 288543 80002 INFO  Profiler: 
Profiler EndPoint profiling event occurred, ConfigVersionId=113, 
EndpointCertainityMetric=10, EndpointIPAddress=10.56.111.14, 
EndpointMacAddress=3C:97:0E:C3:F8:F1, EndpointMatchedPolicy=Nortel-Device, 
EndpointNADAddress=10.56.72.127, EndpointOUI=Wistron 
InfoComm(Kunshan)Co.\,Ltd., EndpointPolicy=Nortel-Device, 
EndpointProperty=StaticAssignment=false\,PostureApplicable=Yes\,BYODRegistration=Unknown\,EndPointPolicyID=49054ed0-68a6-11e1-bc72-0050568e013c\,FirstCollection=1407397543718\,TimeToProfile=19\,LastNmapScanTime=0\,StaticGroupAssignment=false\,DeviceRegistrationStatus=NotRegistered\,UpdateTime=0\,PolicyVersion=402\,IdentityGroupID=0c1d9270-68a6-11e1-bc72-0050568e013c\,Total
 Certainty 
Factor=10\,FeedService=false\,MatchedPolicyID=49054ed0-68a6-11e1-bc72-0050568e013c\,NmapScanCount=0\,NmapSubnetScanID=0\,PortalUser=,
 EndpointSourceE
 vent=SNMPQuery Probe, EndpointIdentityGroup=Profiled, 
ProfilerServer=stage-pdp01.cisco.com,
+Aug  6 20:00:52 10.42.7.64 Aug  7 03:20:05 npf-sjca-pdp02 CISE_Profiler 
373185 1 0 2014-08-07 03:20:05.549 -07:00 0011310202 80002 INFO  Profiler: 
Profiler EndPoint profiling event occurred, ConfigVersionId=241, 
EndpointCertainityMetric=90, EndpointIPAddress=10.56.129.142, 
EndpointMacAddress=3C:A9:F4:46:75:CC, 
EndpointMatchedPolicy=Windows7-Workstation, EndpointNADAddress=10.56.129.4, 
EndpointOUI=Intel Corporate, EndpointPolicy=Windows7-Workstation, 
EndpointProperty=StaticAssignment=false\,Calling-Station-ID=3c-a9-f4-46-75-cc\,Device
 Identifier=\,PostureApplicable=Yes\,dhcp-class-identifier=MSFT 
5.0\,host-name=EXAMPLE\,BYODRegistration=Unknown\,EndPointPolicyID=615ed410-68a6-11e1-bc72-0050568e013c\,FirstCollection=1406112353750\,TimeToProfile=11\,Framed-IP-Address=10.56.129.142\,LastNmapScanTime=0\,StaticGroupAssignment=false\,DeviceRegistrationStatus=NotRegistered\,NAS-Port-Type=Wireless
 - IEEE 
802.11\,RegistrationTimeStamp=0\,UpdateTime=1407394245820\,PolicyVersion=403\,Identi
 tyGroupID=5cb39b80-68a6-11e1-bc72-0050568e013c\,Total Certainty 
Factor=90\,FeedService=false\,MatchedPolicyID=615ed410-68a6-11e1-bc72-0050568e013c\,DestinationIPAddress=10.42.7.64\,CreateTime=1394526689397\,NmapScanCount=0\,NmapSubnetScanID=0\,AAA-Server=npf-sjca-pdp02\,PortalUser=,
 EndpointSourceEvent=RADIUS Probe, EndpointUserAgent=Mozilla/5.0 (Windows NT 
6.1\\\; WOW64\\ rv:30.0) Gecko/20100101 Firefox/30.0, 
EndpointIdentityGroup=Workstation, ProfilerServer=npf.example.com,
+Aug  6 21:00:48 10.42.7.64 Aug  7 04:20:00 npf-sjca-pdp02 CISE_Profiler 
373902 1 0 2014-08-07 04:20:00.983 -07:00 0011322557 80002 INFO  Profiler: 
Profiler EndPoint profiling event occurred, ConfigVersionId=241, 
EndpointCertainityMetric=30, EndpointIPAddress=10.56.129.142, 
EndpointMacAddress=3C:A9:F4:46:75:CC, 
EndpointMatchedPolicy=Microsoft-Workstation, EndpointNADAddress=10.56.129.4, 
EndpointOUI=Intel Corporate, EndpointPolicy=Microsoft-Workstation, 

[18/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/pom.xml
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/pom.xml
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/pom.xml
new file mode 100644
index 000..b36e831
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/pom.xml
@@ -0,0 +1,83 @@
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+org.apache.metron
+metron-parser-logstash-extension
+0.4.1
+
+metron-parser-logstash
+0.4.1
+metron-parser-logstash
+jar
+
+UTF-8
+
UTF-8
+
+
+
+org.apache.metron
+metron-common
+${project.parent.version}
+provided
+
+
+org.apache.metron
+metron-parsers
+${project.parent.version}
+provided
+
+
+org.atteo.classindex
+classindex
+${global_classindex_version}
+provided
+
+
+
+org.apache.metron
+metron-parser-extensions-testing
+${project.parent.version}
+pom
+test
+
+
+
+
+
+org.apache.maven.plugins
+maven-jar-plugin
+${global_jar_version}
+
+
+
+test-jar
+
+
+
+
+
+
+
+src/main/resources
+
+
+src/main/patterns
+
+
+src/test/resources
+
+
+
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/enrichments/logstash.json
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/enrichments/logstash.json
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/enrichments/logstash.json
new file mode 100644
index 000..2cd0667
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/enrichments/logstash.json
@@ -0,0 +1,20 @@
+{
+"enrichment" : {
+"fieldMap": {
+"geo": [
+"ip_dst_addr",
+"ip_src_addr"
+]
+}
+},
+"threatIntel": {
+"fieldMap": {
+"hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+},
+"fieldToTypeMap": {
+"ip_src_addr" : ["malicious_ip"],
+"ip_dst_addr" : ["malicious_ip"]
+}
+}
+}
+

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/indexing/logstash.json
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/indexing/logstash.json
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/indexing/logstash.json
new file mode 100644
index 000..beb012e
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-logstash-extension/metron-parser-logstash/src/main/config/zookeeper/indexing/logstash.json
@@ -0,0 +1,18 @@
+{
+  "hdfs" : {
+"index": "logstash",
+"batchSize": 5,
+"enabled" : true
+  },
+  "elasticsearch" : {
+"index": "logstash",
+"batchSize": 5,
+"enabled" : true
+  },
+  "solr" : {
+"index": "logstash",
+"batchSize": 5,
+"enabled" : true
+  }
+}
+

[29/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/test/resources/data/parsed/test.parsed
--
diff --git 
a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/test/resources/data/parsed/test.parsed
 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/test/resources/data/parsed/test.parsed
new file mode 100755
index 000..bbf4cd0
--- /dev/null
+++ 
b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/metron-parser-asa/src/test/resources/data/parsed/test.parsed
@@ -0,0 +1,128 @@
+{"syslog_host":"10.22.8.216","original_string":"<167>Jan  5 08:52:35 
10.22.8.216 %ASA-7-609001: Built local-host 
inside:10.22.8.205","ciscotag":"ASA-7-609001","syslog_facility":"local4","syslog_severity":"debug","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.216","protocol":"icmp","original_string":"<166>Jan  5 
08:52:35 10.22.8.216 %ASA-6-302021: Teardown ICMP connection for faddr 
10.22.8.74\/0(LOCAL\\user.name) gaddr 10.22.8.205\/0 laddr 
10.22.8.205\/0","ip_dst_addr":"10.22.8.74","ciscotag":"ASA-6-302021","syslog_facility":"local4","action":"teardown","ip_src_addr":"10.22.8.205","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.216","original_string":"<167>Jan  5 08:52:35 
10.22.8.216 %ASA-7-609002: Teardown local-host inside:10.22.8.205 duration 
0:00:00","ciscotag":"ASA-7-609002","syslog_facility":"local4","syslog_severity":"debug","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.201","protocol":"tcp","original_string":"<142>Jan  5 
08:52:35 10.22.8.201 %ASA-6-302014: Teardown TCP connection 488167725 for 
Outside_VPN:147.111.72.16\/26436 to DMZ-Inside:10.22.8.53\/443 duration 0:00:00 
bytes 9687 TCP 
FINs","ip_dst_addr":"10.22.8.53","ip_src_port":26436,"ip_dst_port":443,"ciscotag":"ASA-6-302014","syslog_facility":"local1","action":"teardown","ip_src_addr":"147.111.72.16","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.216","protocol":"tcp","original_string":"<166>Jan  5 
08:52:35 10.22.8.216 %ASA-6-302014: Teardown TCP connection 212805593 for 
outside:10.22.8.223\/59614(LOCAL\\user.name) to inside:10.22.8.78\/8102 
duration 0:00:07 bytes 3433 TCP FINs 
(user.name)","ip_dst_addr":"10.22.8.78","ip_src_port":59614,"ip_dst_port":8102,"ciscotag":"ASA-6-302014","syslog_facility":"local4","action":"teardown","ip_src_addr":"10.22.8.223","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.212","protocol":"tcp","original_string":"<174>Jan  5 
14:52:35 10.22.8.212 %ASA-6-302013: Built inbound TCP connection 76245503 for 
outside:10.22.8.233\/54209 (10.22.8.233\/54209) to inside:198.111.72.238\/443 
(198.111.72.238\/443) 
(user.name)","ip_dst_addr":"198.111.72.238","ip_src_port":54209,"ip_dst_port":443,"ciscotag":"ASA-6-302013","syslog_facility":"local5","action":"built","ip_src_addr":"10.22.8.233","syslog_severity":"info","timestamp":145200000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.216","protocol":"tcp","original_string":"<166>Jan  5 
08:52:35 10.22.8.216 %ASA-6-302013: Built inbound TCP connection 212806031 for 
outside:10.22.8.17\/58633 (10.22.8.17\/58633)(LOCAL\\user.name) to 
inside:10.22.8.12\/389 (10.22.8.12\/389) 
(user.name)","ip_dst_addr":"10.22.8.12","ip_src_port":58633,"ip_dst_port":389,"ciscotag":"ASA-6-302013","syslog_facility":"local4","action":"built","ip_src_addr":"10.22.8.17","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.201","protocol":"tcp","original_string":"<142>Jan  5 
08:52:35 10.22.8.201 %ASA-6-302014: Teardown TCP connection 488168292 for 
DMZ-Inside:10.22.8.51\/51231 to Inside-Trunk:10.22.8.174\/40004 duration 
0:00:00 bytes 2103 TCP 
FINs","ip_dst_addr":"10.22.8.174","ip_src_port":51231,"ip_dst_port":40004,"ciscotag":"ASA-6-302014","syslog_facility":"local1","action":"teardown","ip_src_addr":"10.22.8.51","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
+{"syslog_host":"10.22.8.201","protocol":"tcp","original_string":"<142>Jan  5 
08:52:35 10.22.8.201 %ASA-6-106015: Deny TCP (no connection) from 
186.111.72.11\/80 to 204.111.72.226\/45019 flags SYN ACK  on interface 

[01/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1136-extensions-parsers c8e84fa3b -> ffcb91ed0


http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
--
diff --git 
a/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
 
b/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
deleted file mode 100644
index 0e4bf74..000
--- 
a/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt
+++ /dev/null
@@ -1 +0,0 @@
-{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 
2014-07-17T15:58:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index The host's 
concern index has either exceeded the CI threshold or rapidly increased. 
Observed 36.55M points. Policy maximum allows up to 20M 
points.","@version":"1","@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt
--
diff --git 
a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt
 
b/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt
deleted file mode 100644
index c58bcc8..000
--- 
a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-<11>Jan  5 05:38:59 PAN1.exampleCustomer.com 1,2015/01/05 
05:38:58,0006C110285,THREAT,vulnerability,1,2015/01/05 
05:38:58,10.0.0.115,216.0.10.198,0.0.0.0,0.0.0.0,EX-Allow,example\\user.name,,web-browsing,vsys1,internal,external,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05
 
05:38:58,12031,1,54180,80,0,0,0x80004000,tcp,reset-both,\"ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9\",HTTP:
 IIS Denial Of Service 
Attempt(40019),any,high,client-to-server,347368099,0x0,10.0.0.0-10.255.255.255,US,0,,1200568889751109656,,
-<14>Jan  5 12:51:34 PAN1.exampleCustomer.com 1,2015/01/05 
12:51:33,0011C103117,TRAFFIC,end,1,2015/01/05 
12:51:33,10.0.0.39,10.1.0.163,0.0.0.0,0.0.0.0,EX-Allow,,example\\user.name,ms-ds-smb,vsys1,v_external,v_internal,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05
 12:51:33,33760927,1,52688,445,0,0,0x401a,tcp,allow,2229,1287,942,10,2015/01/05 
12:51:01,30,any,0,17754932062,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,4
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
--
diff --git 
a/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
 
b/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
deleted file mode 100644
index af257aa..000
--- 
a/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-SFIMS: [Primary Detection Engine (a7213248-6423-11e3-8537-fac6a92b7d9d)][MTD 
Access Control] Connection Type: Start, User: Unknown, Client: Unknown, 
Application Protocol: Unknown, Web App: Unknown, Firewall Rule Name: MTD Access 
Control, Firewall Rule Action: Allow, Firewall Rule Reasons: Unknown, URL 
Category: Unknown, URL_Reputation: Risk unknown, URL: Unknown, Interface 
Ingress: s1p1, Interface Egress: N/A, Security Zone Ingress: Unknown, Security 
Zone Egress: N/A, Security Intelligence Matching IP: None, Security 
Intelligence Category: None, {TCP} 72.163.0.129:60517 -> 10.1.128.236:443
-snort: [1:3192:2] WEB-CLIENT Windows Media Player directory traversal via 
Content-Disposition attempt [Classification: Attempted User Privilege Gain] 
[Priority: 1] {TCP} 46.149.110.103:80 -> 192.168.56.102:1073
-SFIMS: Correlation Event: Open Soc Log Forwarding/Opensoc Log Forwarding at 
Thu Oct 23 04:55:39 2014 UTC: [1:19123:7] \"MALWARE-CNC Dropper 
Win.Trojan.Cefyns.A variant outbound connection\" [Impact: Unknown] From 
\"172.19.50.7\" at Thu Oct 23 04:55:38 2014 UTC [Classification: A Network 
Trojan was Detected] [Priority: 1] {tcp} 139.230.245.23:52078->72.52.4.91:80
\ No newline at end of file

[05/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
--
diff --git 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
 
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
deleted file mode 100644
index 12c39ca..000
--- 
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/asa/BasicAsaParserTest.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.asa;
-
-import org.apache.log4j.Level;
-import org.apache.metron.test.utils.UnitTestHelper;
-import org.json.simple.JSONObject;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.time.*;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.startsWith;
-import static org.junit.Assert.*;
-
-public class BasicAsaParserTest {
-
-private static BasicAsaParser asaParser;
-
-@BeforeClass
-public static void setUpOnce() throws Exception {
-Map parserConfig = new HashMap<>();
-asaParser = new BasicAsaParser();
-asaParser.configure(parserConfig);
-asaParser.init();
-}
-
-@Test
-public void testConfigureDefault() {
-Map parserConfig = new HashMap<>();
-BasicAsaParser testParser = new BasicAsaParser();
-testParser.configure(parserConfig);
-testParser.init();
-assertTrue(testParser.deviceClock.getZone().equals(ZoneOffset.UTC));
-}
-
-@Test
-public void testConfigureTimeZoneOffset() {
-Map parserConfig = new HashMap<>();
-parserConfig.put("deviceTimeZone", "UTC-05:00");
-BasicAsaParser testParser = new BasicAsaParser();
-testParser.configure(parserConfig);
-testParser.init();
-ZonedDateTime deviceTime = 
ZonedDateTime.ofInstant(Instant.ofEpochSecond(1475323200), 
testParser.deviceClock.getZone());
-ZonedDateTime referenceTime = 
ZonedDateTime.ofInstant(Instant.ofEpochSecond(1475323200), 
ZoneOffset.ofHours(-5));
-assertTrue(deviceTime.isEqual(referenceTime));
-}
-
-@Test
-public void testConfigureTimeZoneText() {
-Map parserConfig = new HashMap<>();
-parserConfig.put("deviceTimeZone", "America/New_York");
-BasicAsaParser testParser = new BasicAsaParser();
-testParser.configure(parserConfig);
-testParser.init();
-ZonedDateTime deviceTime = 
ZonedDateTime.ofInstant(Instant.ofEpochSecond(1475323200), 
testParser.deviceClock.getZone());
-ZonedDateTime referenceTime = 
ZonedDateTime.ofInstant(Instant.ofEpochSecond(1475323200), 
ZoneOffset.ofHours(-5));
-assertTrue(deviceTime.isEqual(referenceTime));
-}
-
-@Test
-public void testCISCOFW106023() {
-String rawMessage = "<164>Aug 05 2016 01:01:34: %ASA-4-106023: Deny 
tcp src Inside:10.30.9.121/54580 dst Outside:192.168.135.51/42028 by 
access-group \"Inside_access_in\" [0x962df600, 0x0]";
-JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
-assertEquals(asaJson.get("original_string"), rawMessage);
-assertTrue(asaJson.get("ip_src_addr").equals("10.30.9.121"));
-assertTrue(asaJson.get("ip_dst_addr").equals("192.168.135.51"));
-assertTrue(asaJson.get("ip_src_port").equals(54580));
-assertTrue(asaJson.get("ip_dst_port").equals(42028));
-assertTrue((long) asaJson.get("timestamp") == 1470358894000L);
-}
-
-@Test
-public void testCISCOFW106006() {
-String rawMessage = "<162>Aug 05 2016 01:02:25: %ASA-2-106006: Deny 
inbound UDP from 10.25.177.164/63279 to 10.2.52.71/161 on interface Inside";
-JSONObject asaJson = asaParser.parse(rawMessage.getBytes()).get(0);
-assertEquals(asaJson.get("original_string"), rawMessage);
-

[10/39] metron git commit: METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720

[otto]

http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-integration-test/src/main/sample/data/asa/parsed/asa_parsed
--
diff --git 
a/metron-platform/metron-integration-test/src/main/sample/data/asa/parsed/asa_parsed
 
b/metron-platform/metron-integration-test/src/main/sample/data/asa/parsed/asa_parsed
deleted file mode 100755
index bbf4cd0..000
--- 
a/metron-platform/metron-integration-test/src/main/sample/data/asa/parsed/asa_parsed
+++ /dev/null
@@ -1,128 +0,0 @@
-{"syslog_host":"10.22.8.216","original_string":"<167>Jan  5 08:52:35 
10.22.8.216 %ASA-7-609001: Built local-host 
inside:10.22.8.205","ciscotag":"ASA-7-609001","syslog_facility":"local4","syslog_severity":"debug","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.216","protocol":"icmp","original_string":"<166>Jan  5 
08:52:35 10.22.8.216 %ASA-6-302021: Teardown ICMP connection for faddr 
10.22.8.74\/0(LOCAL\\user.name) gaddr 10.22.8.205\/0 laddr 
10.22.8.205\/0","ip_dst_addr":"10.22.8.74","ciscotag":"ASA-6-302021","syslog_facility":"local4","action":"teardown","ip_src_addr":"10.22.8.205","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.216","original_string":"<167>Jan  5 08:52:35 
10.22.8.216 %ASA-7-609002: Teardown local-host inside:10.22.8.205 duration 
0:00:00","ciscotag":"ASA-7-609002","syslog_facility":"local4","syslog_severity":"debug","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.201","protocol":"tcp","original_string":"<142>Jan  5 
08:52:35 10.22.8.201 %ASA-6-302014: Teardown TCP connection 488167725 for 
Outside_VPN:147.111.72.16\/26436 to DMZ-Inside:10.22.8.53\/443 duration 0:00:00 
bytes 9687 TCP 
FINs","ip_dst_addr":"10.22.8.53","ip_src_port":26436,"ip_dst_port":443,"ciscotag":"ASA-6-302014","syslog_facility":"local1","action":"teardown","ip_src_addr":"147.111.72.16","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.216","protocol":"tcp","original_string":"<166>Jan  5 
08:52:35 10.22.8.216 %ASA-6-302014: Teardown TCP connection 212805593 for 
outside:10.22.8.223\/59614(LOCAL\\user.name) to inside:10.22.8.78\/8102 
duration 0:00:07 bytes 3433 TCP FINs 
(user.name)","ip_dst_addr":"10.22.8.78","ip_src_port":59614,"ip_dst_port":8102,"ciscotag":"ASA-6-302014","syslog_facility":"local4","action":"teardown","ip_src_addr":"10.22.8.223","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.212","protocol":"tcp","original_string":"<174>Jan  5 
14:52:35 10.22.8.212 %ASA-6-302013: Built inbound TCP connection 76245503 for 
outside:10.22.8.233\/54209 (10.22.8.233\/54209) to inside:198.111.72.238\/443 
(198.111.72.238\/443) 
(user.name)","ip_dst_addr":"198.111.72.238","ip_src_port":54209,"ip_dst_port":443,"ciscotag":"ASA-6-302013","syslog_facility":"local5","action":"built","ip_src_addr":"10.22.8.233","syslog_severity":"info","timestamp":145200000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.216","protocol":"tcp","original_string":"<166>Jan  5 
08:52:35 10.22.8.216 %ASA-6-302013: Built inbound TCP connection 212806031 for 
outside:10.22.8.17\/58633 (10.22.8.17\/58633)(LOCAL\\user.name) to 
inside:10.22.8.12\/389 (10.22.8.12\/389) 
(user.name)","ip_dst_addr":"10.22.8.12","ip_src_port":58633,"ip_dst_port":389,"ciscotag":"ASA-6-302013","syslog_facility":"local4","action":"built","ip_src_addr":"10.22.8.17","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.201","protocol":"tcp","original_string":"<142>Jan  5 
08:52:35 10.22.8.201 %ASA-6-302014: Teardown TCP connection 488168292 for 
DMZ-Inside:10.22.8.51\/51231 to Inside-Trunk:10.22.8.174\/40004 duration 
0:00:00 bytes 2103 TCP 
FINs","ip_dst_addr":"10.22.8.174","ip_src_port":51231,"ip_dst_port":40004,"ciscotag":"ASA-6-302014","syslog_facility":"local1","action":"teardown","ip_src_addr":"10.22.8.51","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}
-{"syslog_host":"10.22.8.201","protocol":"tcp","original_string":"<142>Jan  5 
08:52:35 10.22.8.201 %ASA-6-106015: Deny TCP (no connection) from 
186.111.72.11\/80 to 204.111.72.226\/45019 flags SYN ACK  on interface 
Outside_VPN","ip_dst_addr":"204.111.72.226","ip_src_port":80,"ip_dst_port":45019,"ciscotag":"ASA-6-106015","syslog_facility":"local1","action":"deny","ip_src_addr":"186.111.72.11","syslog_severity":"info","timestamp":1451983955000,"source.type":"asa","guid":"this-is-random-uuid-will-be-36-chars"}

[metron] Git Push Summary

[otto]

Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1136-extensions-parsers [created] c8e84fa3b

[2/2] metron git commit: fix dependencies csv for zjsonpatch pre-commit

[otto]

fix dependencies csv for zjsonpatch pre-commit


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c8e84fa3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c8e84fa3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c8e84fa3

Branch: refs/heads/master
Commit: c8e84fa3be89901013168d15df38b8a58265148a
Parents: d5dbfc2
Author: otto <o...@apache.org>
Authored: Sat Aug 26 11:20:32 2017 -0400
Committer: otto <o...@apache.org>
Committed: Sat Aug 26 11:20:32 2017 -0400

--
 dependencies_with_url.csv | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/c8e84fa3/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index fac1164..d09fed6 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -18,6 +18,7 @@ com.esotericsoftware.minlog:minlog:jar:1.2:compile,New BSD 
License,http://code.g
 com.esotericsoftware.minlog:minlog:jar:1.3.0:compile,New BSD 
License,http://code.google.com/p/minlog/
 com.esotericsoftware:minlog:jar:1.3.0:compile,New BSD 
License,http://code.google.com/p/minlog/
 com.esotericsoftware:reflectasm:jar:1.10.1:compile,New BSD 
License,http://code.google.com/p/minlog/
+com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.1:compile,Apache v2, 
https://github.com/flipkart-incubator/zjsonpatch
 com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/

[1/2] metron git commit: METRON-1061 Add Fuzzy String Scoring to Stellar (ottobackwards) closes apache/metron#667

[otto]

Repository: metron
Updated Branches:
  refs/heads/master a2bae0bce -> c8e84fa3b


METRON-1061 Add Fuzzy String Scoring to Stellar (ottobackwards) closes 
apache/metron#667


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d5dbfc20
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d5dbfc20
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d5dbfc20

Branch: refs/heads/master
Commit: d5dbfc20cb42708c27e9a02c5f7eacac98604745
Parents: a2bae0b
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Sat Aug 26 10:46:01 2017 -0400
Committer: otto <o...@apache.org>
Committed: Sat Aug 26 10:46:01 2017 -0400

--
 dependencies_with_url.csv   |   1 +
 metron-stellar/stellar-common/README.md |  14 +++
 metron-stellar/stellar-common/pom.xml   |   5 +
 .../stellar/dsl/functions/TextFunctions.java| 112 +++
 .../dsl/functions/TextFunctionsTest.java| 101 +
 5 files changed, 233 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/d5dbfc20/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 83078ad..fac1164 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -177,6 +177,7 @@ 
commons-logging:commons-logging:jar:1.1.3:compile,ASLv2,http://commons.apache.or
 
commons-logging:commons-logging:jar:1.2:compile,ASLv2,http://commons.apache.org/proper/commons-logging/
 commons-net:commons-net:jar:3.1:compile,ASLv2,http://commons.apache.org/net/
 commons-net:commons-net:jar:3.1:provided,ASLv2,http://commons.apache.org/net/
+commons-text:commons-text:jar:1.1:compile,ASLv2,http://commons.apache.org/proper/commons-text/
 
commons-validator:commons-validator:jar:1.4.0:compile,ASLv2,http://commons.apache.org/validator/
 
commons-validator:commons-validator:jar:1.5.1:compile,ASLv2,http://commons.apache.org/proper/commons-validator/
 
commons-validator:commons-validator:jar:1.6:compile,ASLv2,http://commons.apache.org/proper/commons-validator/

http://git-wip-us.apache.org/repos/asf/metron/blob/d5dbfc20/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md 
b/metron-stellar/stellar-common/README.md
index a25c831..8746e60 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -131,6 +131,8 @@ In the core language functions, we support basic functional 
programming primitiv
 | [ `FILL_RIGHT`](#fill_right) 
  |
 | [ `FILTER`](#filter) 
  |
 | [ `FLOOR`](#floor)   
|
+| [ `FUZZY_LANGS`](#fuzzy_langs)   
|
+| [ `FUZZY_SCORE`](#fuzzy_score)   
|
 | [ `FORMAT`](#format) 
  |
 | [ `GEO_GET`](#geo_get)   
  |
 | [ `GET`](#get)   
  |
@@ -412,6 +414,18 @@ In the core language functions, we support basic 
functional programming primitiv
 * format - string
 * arguments... - object(s)
   * Returns: A formatted string.
+  
+### `FUZZY_LANGS`
+  * Description: Returns a list of IETF BCP 47 available to the system, such 
as en, fr, de.
+  * Returns: A list of IEF BGP 47 language tag strings
+
+### `FUZZY_SCORE`
+  * Description: Returns the Fuzzy Score which indicates the similarity score 
between two strings. One point is given for every matched character. Subsequent 
matches yield two bonus points. A higher score indicates a higher similarity.
+  * Input:
+* string - The full term that should be matched against.
+* string - The query that will be matched against a term.
+* string - The IETF BCP 47 language code to use.
+  * Returns: An Integer representing the score.
 
 ### `GEO_GET`
   * Description: Look up an IPV4 address and returns geographic information 
about it

http://git-wip-us.apache.org/repos/asf/metron/blob/d5dbfc20/metron-stellar/stellar-common/pom.xml
--
diff --git a/metron-stellar/stellar-common/pom.xml 
b/metron-stellar/stellar-common/pom.xml
index 2f4cb6e..5945bbd 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -97,6 +97,11 @@
  

metron git commit: METRON-711 StellarShell assigns variables even if an exception was thrown in the statement. (ottobackwards) closes apache/metron#686

[otto]

Repository: metron
Updated Branches:
  refs/heads/master 5a6370a63 -> 73dc16e7a


METRON-711 StellarShell assigns variables even if an exception was thrown in 
the statement. (ottobackwards) closes apache/metron#686


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73dc16e7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73dc16e7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73dc16e7

Branch: refs/heads/master
Commit: 73dc16e7a40edb38de6657074c22860647441148
Parents: 5a6370a
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Thu Aug 17 01:17:44 2017 -0400
Committer: otto <o...@apache.org>
Committed: Thu Aug 17 01:17:44 2017 -0400

--
 .../stellar/common/shell/StellarShell.java  | 40 
 1 file changed, 15 insertions(+), 25 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/73dc16e7/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
index b8b31a3..0d2f0c3 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
@@ -264,12 +264,21 @@ public class StellarShell extends AeshConsoleCallback 
implements Completion {
 stellarExpression = stellarExpression.trim();
   }
 }
-Object result = executeStellar(stellarExpression);
-if(result != null && variable == null) {
-  writeLine(result.toString());
-}
-if(variable != null) {
-  executor.assign(variable, stellarExpression, result);
+
+try {
+  Object result = executor.execute(stellarExpression);
+  if (result != null && variable == null) {
+writeLine(result.toString());
+  }
+  if (variable != null) {
+executor.assign(variable, stellarExpression, result);
+  }
+} catch (Throwable t) {
+  if(variable != null) {
+writeLine(String.format("%s ERROR: Variable %s not assigned", 
ERROR_PROMPT, variable));
+  }
+  writeLine(ERROR_PROMPT + t.getMessage());
+  t.printStackTrace();
 }
   }
 
@@ -352,25 +361,6 @@ public class StellarShell extends AeshConsoleCallback 
implements Completion {
 return StringUtils.startsWith(expression, DOC_PREFIX);
   }
 
-  /**
-   * Executes a Stellar expression.
-   * @param expression The expression to execute.
-   * @return The result of the expression.
-   */
-  private Object executeStellar(String expression) {
-Object result = null;
-
-try {
-  result = executor.execute(expression);
-
-} catch(Throwable t) {
-  writeLine(ERROR_PROMPT + t.getMessage());
-  t.printStackTrace();
-}
-
-return result;
-  }
-
   private void write(String out) {
 System.out.print(out);
   }

[1/2] metron git commit: METRON-379 Stellar - No Error When Adding Variables That Do Not Exist (ottobackwards) closes apache/metron#675

[otto]

Repository: metron
Updated Branches:
  refs/heads/master b2375a1f1 -> c08cd07f3


http://git-wip-us.apache.org/repos/asf/metron/blob/c08cd07f/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
--
diff --git 
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
 
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index 06edb12..d6c3713 100644
--- 
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++ 
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.metron.stellar.common.StellarProcessor;
 import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.DefaultVariableResolver;
 import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.Stellar;
 import org.apache.metron.stellar.dsl.StellarFunction;
@@ -37,6 +38,7 @@ import org.junit.rules.ExpectedException;
 import java.util.*;
 
 import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
+import static 
org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate;
 import static 
org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate;
 
 @SuppressWarnings("ALL")
@@ -146,6 +148,32 @@ public class BasicStellarTest {
 }
   }
 
+  @Test(expected = ParseException.class)
+  public void testMissingVariablesWithParse() {
+String query = "someVar";
+run(query,new HashMap<>());
+  }
+
+  @Test
+  public void testValidateDoesNotThrow(){
+String query = "someVar";
+validate(query);
+  }
+
+  @Test
+  public void testContextActivityTypeReset(){
+String query = "someVar";
+Context context = Context.EMPTY_CONTEXT();
+
+validate(query,context);
+Assert.assertNull(context.getActivityType());
+
+run(query,ImmutableMap.of("someVar","someValue"),context);
+Assert.assertNull(context.getActivityType());
+
+
+  }
+
   @Test
   public void testIfThenElseBug1() {
 String query = "50 + (true == true ? 10 : 20)";
@@ -411,10 +439,10 @@ public class BasicStellarTest {
 Collection c = new ArrayList();
 Assert.assertEquals(0,run(query,ImmutableMap.of("foo",c)));
   }
-  @Test
+  @Test(expected = ParseException.class)
   public void testNoVarLength(){
 String query = "LENGTH(foo)";
-Assert.assertEquals(0,run(query,ImmutableMap.of()));
+run(query,ImmutableMap.of());
   }
 
   @Test
@@ -547,14 +575,14 @@ public class BasicStellarTest {
   put("empty", "");
   put("spaced", "metron is great");
 }};
-Assert.assertFalse(runPredicate("not('casey' == foo and true)", v -> 
variableMap.get(v)));
-Assert.assertTrue(runPredicate("not(not('casey' == foo and true))", v -> 
variableMap.get(v)));
-Assert.assertTrue(runPredicate("('casey' == foo) && ( false != true )", v 
-> variableMap.get(v)));
-Assert.assertFalse(runPredicate("('casey' == foo) and (FALSE == TRUE)", v 
-> variableMap.get(v)));
-Assert.assertFalse(runPredicate("'casey' == foo and FALSE", v -> 
variableMap.get(v)));
-Assert.assertTrue(runPredicate("'casey' == foo and true", v -> 
variableMap.get(v)));
-Assert.assertTrue(runPredicate("true", v -> variableMap.get(v)));
-Assert.assertTrue(runPredicate("TRUE", v -> variableMap.get(v)));
+Assert.assertFalse(runPredicate("not('casey' == foo and true)", new 
DefaultVariableResolver(v -> variableMap.get(v),v -> 
variableMap.containsKey(v;
+Assert.assertTrue(runPredicate("not(not('casey' == foo and true))", new 
DefaultVariableResolver(v -> variableMap.get(v),v -> 
variableMap.containsKey(v;
+Assert.assertTrue(runPredicate("('casey' == foo) && ( false != true )", 
new DefaultVariableResolver(v -> variableMap.get(v),v -> 
variableMap.containsKey(v;
+Assert.assertFalse(runPredicate("('casey' == foo) and (FALSE == TRUE)", 
new DefaultVariableResolver(v -> variableMap.get(v),v -> 
variableMap.containsKey(v;
+Assert.assertFalse(runPredicate("'casey' == foo and FALSE", new 
DefaultVariableResolver(v -> variableMap.get(v),v -> 
variableMap.containsKey(v;
+Assert.assertTrue(runPredicate("'casey' == foo and true", new 
DefaultVariableResolver(v -> variableMap.get(v),v -> 
variableMap.containsKey(v;
+Assert.assertTrue(runPredicate("true", new DefaultVariableResolver(v -> 
variableMap.get(v),v -> variableMap.containsKey(v;
+Assert.assertTrue(runPredicate("TRUE", new DefaultVariableResolver(v -> 
variableMap.get(v),v -> variableMap.containsKey(v;
   }
 
   @Test
@@ -563,16 +591,16 @@ public class BasicStellarTest {
   put("foo", "casey");
   put("empty", 

[2/2] metron git commit: METRON-379 Stellar - No Error When Adding Variables That Do Not Exist (ottobackwards) closes apache/metron#675

[otto]

METRON-379 Stellar - No Error When Adding Variables That Do Not Exist 
(ottobackwards) closes apache/metron#675


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c08cd07f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c08cd07f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c08cd07f

Branch: refs/heads/master
Commit: c08cd07f36cd9bf2608a586a209bf809130a069a
Parents: b2375a1
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Thu Aug 3 10:24:42 2017 -0400
Committer: otto <o...@apache.org>
Committed: Thu Aug 3 10:24:42 2017 -0400

--
 .../client/stellar/WindowLookbackTest.java  |   5 +-
 .../metron/statistics/BinFunctionsTest.java |   3 +-
 .../StellarStatisticsFunctionsTest.java |   3 +-
 .../outlier/MedianAbsoluteDeviationTest.java|   3 +-
 .../stellar/GeoEnrichmentFunctionsTest.java |   3 +-
 .../SimpleHBaseEnrichmentFunctionsTest.java |   3 +-
 .../EnrichmentConfigFunctionsTest.java  |   3 +-
 .../management/IndexingConfigFunctionsTest.java |  18 +-
 .../KafkaFunctionsIntegrationTest.java  |   3 +-
 .../management/ParserConfigFunctionsTest.java   |   6 +-
 .../metron/management/ShellFunctionsTest.java   |   4 +-
 .../management/ThreatTriageFunctionsTest.java   |   8 +-
 .../metron/pcap/filter/PcapFieldResolver.java   |   5 +
 .../stellar/common/BaseStellarProcessor.java|  19 +-
 .../metron/stellar/common/LambdaExpression.java |   5 +-
 .../metron/stellar/common/StellarCompiler.java  |   8 +-
 .../common/utils/StellarProcessorUtils.java |  12 +-
 .../org/apache/metron/stellar/dsl/Context.java  |  17 ++
 .../stellar/dsl/DefaultVariableResolver.java|  44 
 .../metron/stellar/dsl/MapVariableResolver.java |  13 +-
 .../metron/stellar/dsl/VariableResolver.java|   2 +
 ...larComparisonExpressionWithOperatorTest.java | 234 +-
 .../shell/StellarShellOptionsValidatorTest.java |  68 +++---
 .../stellar/common/utils/BloomFilterTest.java   |  10 +-
 .../stellar/dsl/functions/BasicStellarTest.java | 235 +++
 .../dsl/functions/DateFunctionsTest.java|  39 ++-
 .../dsl/functions/EncodingFunctionsTest.java|  21 +-
 .../dsl/functions/FunctionalFunctionsTest.java  |  30 ++-
 .../dsl/functions/MathFunctionsTest.java|   5 +-
 .../dsl/functions/RegExFunctionsTest.java   |  21 +-
 .../dsl/functions/StringFunctionsTest.java  |  44 +++-
 31 files changed, 569 insertions(+), 325 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/c08cd07f/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
--
diff --git 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
index 41d2074..fd6d122 100644
--- 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
+++ 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
@@ -22,6 +22,7 @@ package org.apache.metron.profiler.client.stellar;
 import com.google.common.collect.ImmutableMap;
 import org.apache.commons.lang3.Range;
 import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.DefaultVariableResolver;
 import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
 import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
@@ -71,7 +72,7 @@ public class WindowLookbackTest {
 Map<String, Object> variables = new HashMap<>();
 StellarProcessor stellar = new StellarProcessor();
 List periods = (List)stellar.parse( 
stellarStatement
-, k -> 
variables.get(k)
+, new 
DefaultVariableResolver(k -> variables.get(k),k -> variables.containsKey(k))
 , resolver
 , context
 );
@@ -135,7 +136,7 @@ public class WindowLookbackTest {
 }
 StellarProcessor stellar = new StellarProcessor();
 List periods = (List)stellar.parse( 
stellarStatement
-, k -> 
variables.get(k)
+ 

metron git commit: METRON-1011 Stellar ENRICHMENT functions should enforce submission of 4 parameters (jasper-k via ottobackwards) closes apache/metron#628

[otto]

Repository: metron
Updated Branches:
  refs/heads/master ec959d20e -> 14bcc0389


METRON-1011 Stellar ENRICHMENT functions should enforce submission of 4 
parameters (jasper-k via ottobackwards) closes apache/metron#628


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14bcc038
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14bcc038
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14bcc038

Branch: refs/heads/master
Commit: 14bcc0389431879b1d97ae53f8a8a6e7079bed46
Parents: ec959d2
Author: jasper-k <jknu...@hortonworks.com>
Authored: Tue Aug 1 11:54:19 2017 -0400
Committer: otto <o...@apache.org>
Committed: Tue Aug 1 11:54:19 2017 -0400

--
 .../enrichment/stellar/SimpleHBaseEnrichmentFunctions.java   | 8 
 .../stellar/SimpleHBaseEnrichmentFunctionsTest.java  | 8 
 2 files changed, 12 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/14bcc038/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
index 1368631..f8cdf36 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctions.java
@@ -140,8 +140,8 @@ public class SimpleHBaseEnrichmentFunctions {
   if(!initialized) {
 return false;
   }
-  if(args.size() < 2) {
-throw new IllegalStateException("Requires at least an enrichment type 
and indicator");
+  if(args.size() != 4) {
+throw new IllegalStateException("All parameters are mandatory, submit 
'enrichment type', 'indicator', 'nosql_table' and 'column_family'");
   }
   int i = 0;
   String enrichmentType = (String) args.get(i++);
@@ -217,8 +217,8 @@ public class SimpleHBaseEnrichmentFunctions {
   if(!initialized) {
 return false;
   }
-  if(args.size() < 2) {
-throw new IllegalStateException("Requires at least an enrichment type 
and indicator");
+  if(args.size() != 4) {
+throw new IllegalStateException("All parameters are mandatory, submit 
'enrichment type', 'indicator', 'nosql_table' and 'column_family'");
   }
   int i = 0;
   String enrichmentType = (String) args.get(i++);

http://git-wip-us.apache.org/repos/asf/metron/blob/14bcc038/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
index b7668f3..7ed0320 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
+++ 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/SimpleHBaseEnrichmentFunctionsTest.java
@@ -22,8 +22,10 @@ import com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.StellarFunctions;
 import org.apache.metron.stellar.common.StellarProcessor;
+
 import org.apache.metron.enrichment.converter.EnrichmentHelper;
 import org.apache.metron.enrichment.converter.EnrichmentKey;
 import org.apache.metron.enrichment.converter.EnrichmentValue;
@@ -125,4 +127,10 @@ public class SimpleHBaseEnrichmentFunctionsTest {
 Map<String, Object> out = (Map<String, Object>) result;
 Assert.assertTrue(out.isEmpty());
   }
+
+  @Test(expected = ParseException.class)
+  public void testProvidedParameters() throws Exception {
+String stellar = "ENRICHMENT_GET('et', indicator)";
+Object result = run(stellar, ImmutableMap.of("indicator", "indicator7"));
+  }
 }

metron git commit: METRON-982 REST STORM Support for supervisor summary (ottobackwards) closes apache/metron#607

[otto]

Repository: metron
Updated Branches:
  refs/heads/master ae50723a5 -> ba46fa734


METRON-982 REST STORM Support for supervisor summary (ottobackwards) closes 
apache/metron#607


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ba46fa73
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ba46fa73
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ba46fa73

Branch: refs/heads/master
Commit: ba46fa734d868c78b3e75c86210d947ed7621f0e
Parents: ae50723
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Fri Jul 28 11:24:32 2017 -0400
Committer: otto <o...@apache.org>
Committed: Fri Jul 28 11:24:32 2017 -0400

--
 .../metron/rest/model/SupervisorStatus.java | 144 +++
 .../metron/rest/model/SupervisorSummary.java|  53 +++
 metron-interface/metron-rest/README.md  |   6 +
 .../apache/metron/rest/MetronRestConstants.java |   1 +
 .../metron/rest/controller/StormController.java |   8 ++
 .../metron/rest/service/StormStatusService.java |   3 +
 .../service/impl/StormStatusServiceImpl.java|  63 
 .../StormControllerIntegrationTest.java |  13 ++
 .../metron/rest/mock/MockStormRestTemplate.java |  11 +-
 9 files changed, 274 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/ba46fa73/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SupervisorStatus.java
--
diff --git 
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SupervisorStatus.java
 
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SupervisorStatus.java
new file mode 100644
index 000..a624984
--- /dev/null
+++ 
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SupervisorStatus.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more 
contributor license
+ * agreements.  See the NOTICE file distributed with this work for additional 
information regarding
+ * copyright ownership.  The ASF licenses this file to you under the Apache 
License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the 
License.  You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software 
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 
KIND, either express
+ * or implied. See the License for the specific language governing permissions 
and limitations under
+ * the License.
+ */
+
+package org.apache.metron.rest.model;
+
+public class SupervisorStatus {
+  /*
+  /api/v1/supervisor/summary(GET)
+
+  returns all supervisors summary
+  Response Fields:
+  Field Value Description
+  id String Supervisor's id
+  host String Supervisor's host name
+  upTime String Shows how long the supervisor is running
+  slotsTotal Integer Total number of available worker slots for this supervisor
+  slotsUsed Integer
+  Number of worker slots used on this supervisor
+
+  Sample Response:
+  json
+
+  {
+"supervisors": [{
+"id":"0b879808-2a26-442b-8f7d-23101e0c3696", "host":"10.11.1.7",
+"upTime":"5m 58s", "slotsTotal":4, "slotsUsed":3
+  } ]}
+
+  */
+
+  private String id;
+  private String host;
+  private String uptime;
+  private int slotsTotal;
+  private int slotsUsed;
+
+  public SupervisorStatus() {
+  }
+
+  /**
+   * Creates a new SupervisorStatus.
+   *
+   * @param id Supervisor ID
+   * @param host Supervisor Host
+   * @param upTime the uptime
+   * @param slotsTotal The number of slots total
+   * @param slotsUsed The number of slots used
+   */
+  public SupervisorStatus(String id, String host, String upTime, int 
slotsTotal, int slotsUsed) {
+this.id = id;
+this.host = host;
+this.uptime = upTime;
+this.slotsTotal = slotsTotal;
+this.slotsUsed = slotsUsed;
+  }
+
+  public String getId() {
+return id;
+  }
+
+  public void setId(String id) {
+this.id = id;
+  }
+
+  public String getHost() {
+return host;
+  }
+
+  public void setHost(String host) {
+this.host = host;
+  }
+
+  public String getUptime() {
+return uptime;
+  }
+
+  public void setUptime(String upTime) {
+this.uptime = upTime;
+  }
+
+  public int getSlotsTotal() {
+return slotsTotal;
+  }
+
+  public void setSlotsTotal(int slotsTotal) {
+this.slotsTotal = slotsTotal;
+  }
+
+  public int getSlotsUsed() {
+return slotsUsed;
+  }
+
+  public void setSlotsUsed(int slotsUsed) {
+

metron git commit: METRON-984 Create STELLAR Decoding Functions (ottobackwards) closes apache/metron#642

[otto]

Repository: metron
Updated Branches:
  refs/heads/master 9355a0465 -> e206f2508


METRON-984 Create STELLAR Decoding Functions (ottobackwards) closes 
apache/metron#642


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e206f250
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e206f250
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e206f250

Branch: refs/heads/master
Commit: e206f2508ef7e7d798510df76ccfeb38b9530e89
Parents: 9355a04
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Thu Jul 27 23:37:28 2017 -0400
Committer: otto <o...@apache.org>
Committed: Thu Jul 27 23:37:28 2017 -0400

--
 metron-stellar/stellar-common/README.md |  52 +-
 metron-stellar/stellar-common/pom.xml   |   5 +
 .../stellar/common/encoding/Encodings.java  | 158 +++
 .../dsl/functions/EncodingFunctions.java| 157 ++
 .../stellar/common/encoding/EncodingsTest.java  | 104 
 .../stellar/dsl/functions/BasicStellarTest.java |   2 -
 .../dsl/functions/EncodingFunctionsTest.java| 125 +++
 7 files changed, 594 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e206f250/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md 
b/metron-stellar/stellar-common/README.md
index 0ffb096..2b5d4b6 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -116,15 +116,17 @@ In the core language functions, we support basic 
functional programming primitiv
 | [ `DAY_OF_MONTH`](#day_of_month) 
  |
 | [ `DAY_OF_WEEK`](#day_of_week)   
  |
 | [ `DAY_OF_YEAR`](#day_of_year)   
  |
+| [ `DECODE`](#decode) 
  |
 | [ `DOMAIN_REMOVE_SUBDOMAINS`](#domain_remove_subdomains) 
  |
 | [ `DOMAIN_REMOVE_TLD`](#domain_remove_tld)   
  |
 | [ `DOMAIN_TO_TLD`](#domain_to_tld)   
  |
+| [ `ENCODE`](#encode) 
  |
 | [ `ENDS_WITH`](#ends_with)   
  |
 | [ `ENRICHMENT_EXISTS`](#enrichment_exists)   
  |
 | [ `ENRICHMENT_GET`](#enrichment_get) 
  |
 | [ `FILL_LEFT`](#fill_left)   
  |
 | [ `FILL_RIGHT`](#fill_right) 
  |
-| [ `FILTER`](#filter) 
|
+| [ `FILTER`](#filter) 
  |
 | [ `FORMAT`](#format) 
  |
 | [ 
`HLLP_CARDINALITY`](../../metron-analytics/metron-statistics#hllp_cardinality)  
 |
 | [ `HLLP_INIT`](../../metron-analytics/metron-statistics#hllp_init)   
  |
@@ -134,8 +136,10 @@ In the core language functions, we support basic 
functional programming primitiv
 | [ `GET`](#get)   
  |
 | [ `GET_FIRST`](#get_first)   
  |
 | [ `GET_LAST`](#get_last) 
  |
+| [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings)   
|
 | [ `IN_SUBNET`](#in_subnet)   
  |
 | [ `IS_DATE`](#is_date)   
  |
+| [ `IS_ENCODING`](#is_encoding)   
  |
 | [ `IS_DOMAIN`](#is_domain)   
  |
 | [ `IS_EMAIL`](#is_email) 
  |
 | [ `IS_EMPTY`](#is_empty) 
  |
@@ -148,10 +152,10 @@ In the core language functions, we support basic 
functional progr

metron git commit: METRON-1058 address checkstyle warnings for UnusedImports in metron-stellar (dbist via ottobackwards) closes apache/metron#663

[otto]

Repository: metron
Updated Branches:
  refs/heads/master 240b69f8c -> 28a974037


METRON-1058 address checkstyle warnings for UnusedImports in metron-stellar 
(dbist via ottobackwards) closes apache/metron#663


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/28a97403
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/28a97403
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/28a97403

Branch: refs/heads/master
Commit: 28a974037e016b21902a176cb0e0e36b65ddb24a
Parents: 240b69f
Author: dbist <generi...@gmail.com>
Authored: Tue Jul 25 12:03:49 2017 -0400
Committer: otto <o...@apache.org>
Committed: Tue Jul 25 12:03:49 2017 -0400

--
 .../org/apache/metron/stellar/common/BaseStellarProcessor.java | 1 -
 .../java/org/apache/metron/stellar/common/StellarCompiler.java | 1 -
 .../src/main/java/org/apache/metron/stellar/dsl/Stellar.java   | 2 --
 3 files changed, 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/28a97403/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java
index 2802cdd..323c9d1 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/BaseStellarProcessor.java
@@ -24,7 +24,6 @@ import com.google.common.cache.CacheLoader;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.TokenStream;
 
 import java.util.Set;

http://git-wip-us.apache.org/repos/asf/metron/blob/28a97403/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
index 8f2b9c0..fb3da9f 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
@@ -17,7 +17,6 @@
  */
 package org.apache.metron.stellar.common;
 
-import org.antlr.v4.runtime.ParserRuleContext;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.Token;

http://git-wip-us.apache.org/repos/asf/metron/blob/28a97403/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Stellar.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Stellar.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Stellar.java
index 865e6d6..8afcad3 100644
--- 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Stellar.java
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/Stellar.java
@@ -19,10 +19,8 @@ package org.apache.metron.stellar.dsl;
 
 import org.atteo.classindex.IndexAnnotated;
 
-import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
 
 @Retention(RetentionPolicy.RUNTIME)
 @IndexAnnotated

metron git commit: METRON-933 STELLAR support for setting fields to regex match captures (ottobackwards) closes apache/metron#638

[otto]

Repository: metron
Updated Branches:
  refs/heads/master dcec5a7cf -> cf165ff30


METRON-933 STELLAR support for setting fields to regex match captures 
(ottobackwards) closes apache/metron#638


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cf165ff3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cf165ff3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cf165ff3

Branch: refs/heads/master
Commit: cf165ff306764641a2c7991ff93cf48228d7910d
Parents: dcec5a7
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Tue Jul 11 15:11:12 2017 -0400
Committer: otto <o...@apache.org>
Committed: Tue Jul 11 15:11:12 2017 -0400

--
 metron-stellar/stellar-common/README.md |   9 ++
 .../stellar/common/utils/PatternCache.java  |  37 +++
 .../stellar/dsl/functions/RegExFunctions.java   | 103 +++
 .../stellar/dsl/functions/StringFunctions.java  |  23 -
 .../dsl/functions/RegExFunctionsTest.java   |  70 +
 5 files changed, 219 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/cf165ff3/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md 
b/metron-stellar/stellar-common/README.md
index 2d2ffdd..8a2699d 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -161,6 +161,7 @@ In the core language functions, we support basic functional 
programming primitiv
 | [ `PROTOCOL_TO_NAME`](#protocol_to_name) 
  |
 | [ `REDUCE`](#reduce) 
  |
 | [ `REGEXP_MATCH`](#regexp_match) 
  |
+| [ `REGEXP_GROUP_VAL`](#regexp_group_val) 
  |
 | [ `SPLIT`](#split)   
  |
 | [ `STARTS_WITH`](#starts_with)   
  |
 | [ `STATS_ADD`](../../metron-analytics/metron-statistics#stats_add)   
  |
@@ -571,6 +572,14 @@ In the core language functions, we support basic 
functional programming primitiv
 * string - The string to test
 * pattern - The proposed regex pattern
   * Returns: True if the regex pattern matches the string and false if 
otherwise.
+  
+### `REGEXP_GROUP_VAL`
+  * Description: Returns the value of a group in a regex against a string
+  * Input:
+* string - The string to test
+* pattern - The proposed regex pattern
+* group - The integer that selects what group to select, starting at 1
+  * Returns: The value of the group, or null if not matched or no group at 
index.
 
 ### `STRING_ENTROPY`
   * Description: Computes the base-2 shannon entropy of a string.

http://git-wip-us.apache.org/repos/asf/metron/blob/cf165ff3/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java
--
diff --git 
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java
 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java
new file mode 100644
index 000..f0fcee9
--- /dev/null
+++ 
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/PatternCache.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.common.utils;
+
+import java.util.HashMap;
+import java.util.regex.Pattern;
+
+public enum PatternCache {
+  INSTANCE;
+
+  private static final ThreadLocal<HashMap<String,Pattern>> _cache = 
ThreadLocal.withInitial(() ->
+  new HashMap<>());
+
+  public Patter

metron git commit: METRON-906 Rest service storm configuration does not allow for proper URLs (justinleet via ottobackwards) closes apache/metron#602

[otto]

Repository: metron
Updated Branches:
  refs/heads/master 30d0e2a6f -> f523c1795


METRON-906 Rest service storm configuration does not allow for proper URLs 
(justinleet via ottobackwards) closes apache/metron#602


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f523c179
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f523c179
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f523c179

Branch: refs/heads/master
Commit: f523c179585bea3a61f99c694d2752fc6ceb7a3b
Parents: 30d0e2a
Author: justinleet <justinjl...@gmail.com>
Authored: Fri Jun 16 10:16:27 2017 -0400
Committer: otto <o...@apache.org>
Committed: Fri Jun 16 10:16:27 2017 -0400

--
 .../METRON/CURRENT/configuration/metron-env.xml |  2 +-
 .../METRON/CURRENT/service_advisor.py   |  6 +-
 .../roles/ambari_config/vars/single_node_vm.yml |  2 +-
 .../roles/ambari_config/vars/small_cluster.yml  |  2 +-
 .../service/impl/StormStatusServiceImpl.java| 19 +++--
 .../src/main/resources/application-vagrant.yml  |  2 +-
 .../metron/rest/mock/MockStormRestTemplate.java | 13 +++-
 .../impl/StormStatusServiceImplTest.java| 73 +---
 8 files changed, 83 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/f523c179/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 3e5a405..00c48a6 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -224,7 +224,7 @@
 
 storm_rest_addr
 Storm Rest Server Address
-URL of Storm UI (storm.ui.hostname:8744)
+URL of Storm UI (http://storm.ui.hostname:8744). If no 
protocol is provided, http is assumed.
 
 
 

http://git-wip-us.apache.org/repos/asf/metron/blob/f523c179/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
--
diff --git 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index 7da06f5..4a95e63 100644
--- 
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ 
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -108,7 +108,11 @@ class 
METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
 if "storm-site" in services["configurations"]:
 stormUIServerHost = self.getComponentHostNames(services, "STORM", 
"STORM_UI_SERVER")[0]
 stormUIServerPort = 
services["configurations"]["storm-site"]["properties"]["ui.port"]
-stormUIServerURL = stormUIServerHost + ":" + stormUIServerPort
+stormUIProtocol = "http://;
+if "ui.https.port" in 
services["configurations"]["storm-site"]["properties"]:
+stormUIServerPort = 
services["configurations"]["storm-site"]["properties"]["ui.https.port"]
+stormUIProtocol = "https://;
+stormUIServerURL = stormUIProtocol + stormUIServerHost + ":" + 
stormUIServerPort
 putMetronEnvProperty = self.putProperty(configurations, 
"metron-env", services)
 putMetronEnvProperty("storm_rest_addr",stormUIServerURL)
 

http://git-wip-us.apache.org/repos/asf/metron/blob/f523c179/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
--
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml 
b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
index 7e54db3..6a36fac 100644
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.ym

incubator-metron git commit: METRON-915 platform_info script should have node and npm version (ottobackwards) closes apache/incubator-metron#562

[otto]

Repository: incubator-metron
Updated Branches:
  refs/heads/master 29e7c70df -> 494643c38


METRON-915 platform_info script should have node and npm version 
(ottobackwards) closes apache/incubator-metron#562


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/494643c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/494643c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/494643c3

Branch: refs/heads/master
Commit: 494643c38a64834a05ab3c6c2615aab6e9450a4e
Parents: 29e7c70
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Wed May 3 08:47:09 2017 -0400
Committer: otto <o...@apache.org>
Committed: Wed May 3 08:47:09 2017 -0400

--
 metron-deployment/scripts/platform-info.sh | 11 +++
 1 file changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/494643c3/metron-deployment/scripts/platform-info.sh
--
diff --git a/metron-deployment/scripts/platform-info.sh 
b/metron-deployment/scripts/platform-info.sh
index f84d822..a1dce34 100755
--- a/metron-deployment/scripts/platform-info.sh
+++ b/metron-deployment/scripts/platform-info.sh
@@ -63,6 +63,17 @@ mvn --version
 echo "--"
 docker --version
 
+# node
+echo "--"
+echo "node"
+node --version
+
+# npm
+echo "--"
+echo "npm"
+npm --version
+
+
 # operating system
 echo "--"
 uname -a

incubator-metron git commit: METRON-892 platform_info.sh should have the version of docker (ottobackwards) closes apache/incubator-metron#551

[otto]

Repository: incubator-metron
Updated Branches:
  refs/heads/master 6e1e322ef -> 58fa1f69c


METRON-892 platform_info.sh should have the version of docker (ottobackwards) 
closes apache/incubator-metron#551


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/58fa1f69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/58fa1f69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/58fa1f69

Branch: refs/heads/master
Commit: 58fa1f69cf14436b54414f660ff11c2c48c99c6f
Parents: 6e1e322
Author: ottobackwards <ottobackwa...@gmail.com>
Authored: Wed Apr 26 10:03:42 2017 -0400
Committer: otto <o...@apache.org>
Committed: Wed Apr 26 10:03:42 2017 -0400

--
 metron-deployment/scripts/platform-info.sh | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/58fa1f69/metron-deployment/scripts/platform-info.sh
--
diff --git a/metron-deployment/scripts/platform-info.sh 
b/metron-deployment/scripts/platform-info.sh
index 79ebcac..f84d822 100755
--- a/metron-deployment/scripts/platform-info.sh
+++ b/metron-deployment/scripts/platform-info.sh
@@ -59,6 +59,10 @@ python --version 2>&1
 echo "--"
 mvn --version
 
+# docker
+echo "--"
+docker --version
+
 # operating system
 echo "--"
 uname -a
@@ -73,7 +77,7 @@ case "${OSTYPE}" in
 cat /proc/cpuinfo | grep -i '^processor' | wc -l | awk '{print "Total 
Physical Processors: " $0}'
 cat /proc/cpuinfo | grep -i cores | cut -d: -f2 | awk '{corecount+=$1} END 
{print "Total cores: " corecount}'
 echo "Disk information:"
-df -h | grep "^/" 
+df -h | grep "^/"
 ;;
   darwin*)
 sysctl hw.memsize | awk '{print "Total System Memory = " $2/1048576 " MB"}'
@@ -82,10 +86,9 @@ case "${OSTYPE}" in
 sysctl hw.physicalcpu | cut -d: -f2 | awk '{print "Total Physical 
Processors:" $0}'
 sysctl machdep.cpu | grep 'machdep.cpu.core_count' | cut -d: -f2 | cut 
-d\@ -f2 | awk '{print "Total cores:" $0}'
 echo "Disk information:"
-df -h | grep "^/" 
+df -h | grep "^/"
 ;;
   *)
 echo "Unable to detect system resources for ${OSTYPE}"
 ;;
 esac
-