[06/50] [abbrv] metron git commit: METRON-1568: Stellar should have a _ special variable which returns the message in map form closes apache/incubator-metron#1021

2018-06-11 Thread cestella 
METRON-1568: Stellar should have a _ special variable which returns the message 
in map form closes apache/incubator-metron#1021


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b9828e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b9828e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b9828e6

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 1b9828e6a68fd39fd308ebb80f1d79cd490ceaec
Parents: 523c38c
Author: cstella 
Authored: Thu Jun 7 10:54:05 2018 -0400
Committer: cstella 
Committed: Thu Jun 7 10:54:05 2018 -0400

--
 .../enrichment/handler/StellarConfig.java   |  11 +-
 .../StellarTransformationTest.java  |  30 ++-
 .../adapters/stellar/StellarAdapterTest.java|  26 ++
 .../integration/EnrichmentIntegrationTest.java  |   2 +
 .../main/config/zookeeper/enrichments/test.json |   1 +
 .../metron/pcap/filter/PcapFieldResolver.java   |   5 +
 metron-stellar/stellar-common/README.md |   1 +
 .../metron/stellar/common/utils/ConcatMap.java  | 256 +++
 .../common/utils/StellarProcessorUtils.java |  51 +++-
 .../metron/stellar/dsl/MapVariableResolver.java |   8 +
 .../metron/stellar/dsl/VariableResolver.java|   1 +
 .../dsl/functions/DataStructureFunctions.java   |   4 +
 .../stellar/common/utils/ConcatMapTest.java |  83 ++
 .../stellar/dsl/functions/BasicStellarTest.java |  18 ++
 14 files changed, 487 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 62e0263..9f96a22 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -19,6 +19,7 @@ package 
org.apache.metron.common.configuration.enrichment.handler;
 
 import org.apache.metron.stellar.common.StellarAssignment;
 import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.dsl.VariableResolver;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -142,8 +143,14 @@ public class StellarConfig implements Config {
   {
 
 Map messageSegment = new HashMap<>();
-for(String variable : stellarFields) {
-  messageSegment.put(variable, message.get(variable));
+if(stellarFields.contains(VariableResolver.ALL_FIELDS)) {
+  //we need to include all of the fields here.
+  messageSegment.putAll(message);
+}
+else {
+  for (String variable : stellarFields) {
+messageSegment.put(variable, message.get(variable));
+  }
 }
 return messageSegment;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index fc91844..3b7c7bb 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -78,6 +78,32 @@ public class StellarTransformationTest {
   @Multiline
   public static String badConfig;
 
+  /**
+   { "fieldTransformations" : [
+{ "transformation" : "STELLAR"
+,"output" : [ "new_field"]
+,"config" : {
+  "new_field" : "MAP_GET('source.type', _)"
+}
+}
+]
+  }
+   */
+ @Multiline
+ public static String configAll;
+
+  @Test
+  public void testConfigAll() throws Exception {
+SensorParserConfig c = 
SensorParserConfig.fromBytes(Bytes.toBytes(configAll));
+JSONObject input = new JSONObject();
+input.put("source.type", "test");
+for (FieldTransformer handler : c.getFieldTransformations()) {
+  handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+}
+Assert.assertEquals(2,

metron git commit: METRON-1568: Stellar should have a _ special variable which returns the message in map form closes apache/incubator-metron#1021

2018-06-07 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master 523c38cf6 -> 1b9828e6a


METRON-1568: Stellar should have a _ special variable which returns the message 
in map form closes apache/incubator-metron#1021


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b9828e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b9828e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b9828e6

Branch: refs/heads/master
Commit: 1b9828e6a68fd39fd308ebb80f1d79cd490ceaec
Parents: 523c38c
Author: cstella 
Authored: Thu Jun 7 10:54:05 2018 -0400
Committer: cstella 
Committed: Thu Jun 7 10:54:05 2018 -0400

--
 .../enrichment/handler/StellarConfig.java   |  11 +-
 .../StellarTransformationTest.java  |  30 ++-
 .../adapters/stellar/StellarAdapterTest.java|  26 ++
 .../integration/EnrichmentIntegrationTest.java  |   2 +
 .../main/config/zookeeper/enrichments/test.json |   1 +
 .../metron/pcap/filter/PcapFieldResolver.java   |   5 +
 metron-stellar/stellar-common/README.md |   1 +
 .../metron/stellar/common/utils/ConcatMap.java  | 256 +++
 .../common/utils/StellarProcessorUtils.java |  51 +++-
 .../metron/stellar/dsl/MapVariableResolver.java |   8 +
 .../metron/stellar/dsl/VariableResolver.java|   1 +
 .../dsl/functions/DataStructureFunctions.java   |   4 +
 .../stellar/common/utils/ConcatMapTest.java |  83 ++
 .../stellar/dsl/functions/BasicStellarTest.java |  18 ++
 14 files changed, 487 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
--
diff --git 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 62e0263..9f96a22 100644
--- 
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++ 
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -19,6 +19,7 @@ package 
org.apache.metron.common.configuration.enrichment.handler;
 
 import org.apache.metron.stellar.common.StellarAssignment;
 import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.dsl.VariableResolver;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -142,8 +143,14 @@ public class StellarConfig implements Config {
   {
 
 Map messageSegment = new HashMap<>();
-for(String variable : stellarFields) {
-  messageSegment.put(variable, message.get(variable));
+if(stellarFields.contains(VariableResolver.ALL_FIELDS)) {
+  //we need to include all of the fields here.
+  messageSegment.putAll(message);
+}
+else {
+  for (String variable : stellarFields) {
+messageSegment.put(variable, message.get(variable));
+  }
 }
 return messageSegment;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index fc91844..3b7c7bb 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -78,6 +78,32 @@ public class StellarTransformationTest {
   @Multiline
   public static String badConfig;
 
+  /**
+   { "fieldTransformations" : [
+{ "transformation" : "STELLAR"
+,"output" : [ "new_field"]
+,"config" : {
+  "new_field" : "MAP_GET('source.type', _)"
+}
+}
+]
+  }
+   */
+ @Multiline
+ public static String configAll;
+
+  @Test
+  public void testConfigAll() throws Exception {
+SensorParserConfig c = 
SensorParserConfig.fromBytes(Bytes.toBytes(configAll));
+JSONObject input = new JSONObject();
+input.put("source.type", "test");
+for (FieldTransformer handler : c.getFieldTransformations()) {
+  handler.transformAndUpdate(input,