Repository: metron Updated Branches: refs/heads/master 9bc9e0c52 -> a9079f546
METRON-1135: Make Stellar field transformations handle nulls properly this closes apache/incubator-metron#718 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a9079f54 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a9079f54 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a9079f54 Branch: refs/heads/master Commit: a9079f546345d449647daf4f681d5d259d0829cd Parents: 9bc9e0c Author: cstella <ceste...@gmail.com> Authored: Tue Aug 29 09:49:17 2017 -0400 Committer: cstella <ceste...@gmail.com> Committed: Tue Aug 29 09:49:17 2017 -0400 ---------------------------------------------------------------------- .../transformation/StellarTransformation.java | 11 +++++ .../StellarTransformationTest.java | 47 ++++++++++++++++++++ metron-platform/metron-parsers/README.md | 43 ++++++++++++++++++ 3 files changed, 101 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a9079f54/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java index a5d8689..2a22e21 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java @@ -55,6 +55,17 @@ public class StellarTransformation implements FieldTransformation { intermediateVariables.put(oField, o); } } + else { + if(outputs.contains(oField)) { + ret.put(oField, o); + } + if( o != null ) { + intermediateVariables.put(oField, o); + } else { + // remove here, in case there are other statements + intermediateVariables.remove(oField); + } + } } catch(Exception ex) { throw new IllegalStateException( "Unable to process transformation: " + transformObj.toString() http://git-wip-us.apache.org/repos/asf/metron/blob/a9079f54/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java index 12f8b5c..0a3cbb0 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java @@ -51,6 +51,53 @@ public class StellarTransformationTest { /** { "fieldTransformations" : [ { "transformation" : "STELLAR" + ,"output" : [ "new_field", "new_field2", "old_field", "old_field2"] + ,"config" : { + "new_field" : "old_field" + ,"new_field2" : "old_field2" + ,"old_field" : "null" + ,"old_field2" : "null" + } + } + ] + } + */ + @Multiline + public static String configRename; + + @Test + public void testStellarRename() throws Exception { + + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configRename)); + { + JSONObject input = new JSONObject(); + input.put("old_field", "val"); + input.put("old_field2", "val2"); + for (FieldTransformer handler : c.getFieldTransformations()) { + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + } + Assert.assertEquals(2, input.size()); + Assert.assertTrue(input.containsKey("new_field")); + Assert.assertEquals("val", input.get("new_field")); + Assert.assertEquals("val2", input.get("new_field2")); + Assert.assertTrue(!input.containsKey("old_field")); + Assert.assertTrue(!input.containsKey("old_field2")); + } + { + JSONObject input = new JSONObject(); + input.put("old_field", "val"); + for (FieldTransformer handler : c.getFieldTransformations()) { + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + } + + Assert.assertEquals(1, input.size()); + Assert.assertTrue(input.containsKey("new_field")); + Assert.assertEquals("val", input.get("new_field")); + } + } + + /** { "fieldTransformations" : [ + { "transformation" : "STELLAR" ,"output" : [ "full_hostname", "domain_without_subdomains" ] ,"config" : { "full_hostname" : "URL_TO_HOST('http://1234567890123456789012345678901234567890123456789012345678901234567890/index.html')" http://git-wip-us.apache.org/repos/asf/metron/blob/a9079f54/metron-platform/metron-parsers/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index f03abdf..9de4341 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -212,6 +212,49 @@ into `{ "protocol" : "TCP", "source.type" : "bro", ...}` * `STELLAR` : This transformation executes a set of transformations expressed as [Stellar Language](../metron-common) statements. +### Assignment to `null` + +If, in your field transformation, you assign a field to `null`, the field will be removed. +You can use this capability to rename variables. + +Consider this example: +``` + "fieldTransformations" : [ + { "transformation" : "STELLAR" + ,"output" : [ "new_field", "old_field"] + ,"config" : { + "new_field" : "old_field" + ,"old_field" : "null" + } + } + ] +``` +This would set `new_field` to the value of `old_field` and remove `old_field`. + +### Warning: Transforming the same field twice + +Currently, the stellar expressions are expressed in the form of a map where the keys define +the fields and the values define the Stellar expressions. You order the expression evaluation +in the `output` field. A consequence of this choice to store the assignments as a map is that +the same field cannot appear in the map as a key twice. + +For instance, the following will not function as expected: +``` + "fieldTransformations" : [ + { "transformation" : "STELLAR" + ,"output" : [ "new_field"] + ,"config" : { + "new_field" : "TO_UPPER(field1)" + ,"new_field" : "TO_LOWER(new_field)" + } + } + ] +``` + +In the above example, the last instance of `new_field` will win and `TO_LOWER(new_field)` will be evaluated +while `TO_UPPER(field1)` will be skipped. + +### Example Consider the following sensor parser config to add three new fields to a message: * `utc_timestamp` : The unix epoch timestamp based on the `timestamp` field, a `dc` field which is the data center the message comes from and a `dc2tz` map mapping data centers to timezones