Repository: metron Updated Branches: refs/heads/feature/METRON-1136-extensions-parsers ffcb91ed0 -> 436466500
http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java index 12f8b5c..0a3cbb0 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java @@ -51,6 +51,53 @@ public class StellarTransformationTest { /** { "fieldTransformations" : [ { "transformation" : "STELLAR" + ,"output" : [ "new_field", "new_field2", "old_field", "old_field2"] + ,"config" : { + "new_field" : "old_field" + ,"new_field2" : "old_field2" + ,"old_field" : "null" + ,"old_field2" : "null" + } + } + ] + } + */ + @Multiline + public static String configRename; + + @Test + public void testStellarRename() throws Exception { + + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configRename)); + { + JSONObject input = new JSONObject(); + input.put("old_field", "val"); + input.put("old_field2", "val2"); + for (FieldTransformer handler : c.getFieldTransformations()) { + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + } + Assert.assertEquals(2, input.size()); + Assert.assertTrue(input.containsKey("new_field")); + Assert.assertEquals("val", input.get("new_field")); + Assert.assertEquals("val2", input.get("new_field2")); + Assert.assertTrue(!input.containsKey("old_field")); + Assert.assertTrue(!input.containsKey("old_field2")); + } + { + JSONObject input = new JSONObject(); + input.put("old_field", "val"); + for (FieldTransformer handler : c.getFieldTransformations()) { + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + } + + Assert.assertEquals(1, input.size()); + Assert.assertTrue(input.containsKey("new_field")); + Assert.assertEquals("val", input.get("new_field")); + } + } + + /** { "fieldTransformations" : [ + { "transformation" : "STELLAR" ,"output" : [ "full_hostname", "domain_without_subdomains" ] ,"config" : { "full_hostname" : "URL_TO_HOST('http://1234567890123456789012345678901234567890123456789012345678901234567890/index.html')" http://git-wip-us.apache.org/repos/asf/metron/blob/43646650/metron-platform/metron-parsers/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index 3f8acbb..ea4f1dd 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -212,6 +212,49 @@ into `{ "protocol" : "TCP", "source.type" : "bro", ...}` * `STELLAR` : This transformation executes a set of transformations expressed as [Stellar Language](../metron-common) statements. +### Assignment to `null` + +If, in your field transformation, you assign a field to `null`, the field will be removed. +You can use this capability to rename variables. + +Consider this example: +``` + "fieldTransformations" : [ + { "transformation" : "STELLAR" + ,"output" : [ "new_field", "old_field"] + ,"config" : { + "new_field" : "old_field" + ,"old_field" : "null" + } + } + ] +``` +This would set `new_field` to the value of `old_field` and remove `old_field`. + +### Warning: Transforming the same field twice + +Currently, the stellar expressions are expressed in the form of a map where the keys define +the fields and the values define the Stellar expressions. You order the expression evaluation +in the `output` field. A consequence of this choice to store the assignments as a map is that +the same field cannot appear in the map as a key twice. + +For instance, the following will not function as expected: +``` + "fieldTransformations" : [ + { "transformation" : "STELLAR" + ,"output" : [ "new_field"] + ,"config" : { + "new_field" : "TO_UPPER(field1)" + ,"new_field" : "TO_LOWER(new_field)" + } + } + ] +``` + +In the above example, the last instance of `new_field` will win and `TO_LOWER(new_field)` will be evaluated +while `TO_UPPER(field1)` will be skipped. + +### Example Consider the following sensor parser config to add three new fields to a message: * `utc_timestamp` : The unix epoch timestamp based on the `timestamp` field, a `dc` field which is the data center the message comes from and a `dc2tz` map mapping data centers to timezones
