METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b4f0b84 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b4f0b84 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b4f0b84 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 2b4f0b84062d65f9400421d66ec3b7d6d093bebf Parents: 1c5435c Author: cstella <ceste...@gmail.com> Authored: Wed Apr 25 11:49:56 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Wed Apr 25 11:49:56 2018 -0400 ---------------------------------------------------------------------- .../common/configuration/FieldTransformer.java | 4 +- .../transformation/FieldTransformations.java | 1 + .../transformation/RenameTransformation.java | 55 +++++++++++ .../transformation/FieldTransformationTest.java | 17 +--- .../RenameTransformationTest.java | 99 ++++++++++++++++++++ metron-platform/metron-parsers/README.md | 25 ++++- 6 files changed, 183 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index df80691..43ce9d8 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -94,7 +94,9 @@ public class FieldTransformer implements Serializable { if (output == null || output.isEmpty()) { if (input == null || input.isEmpty()) { - throw new IllegalStateException("You must specify an input field if you want to leave the output fields empty"); + //both are empty, so let's set them both to null + output = null; + input = null; } else { output = input; } http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java index a905123..95ff390 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java @@ -25,6 +25,7 @@ public enum FieldTransformations { ,REMOVE(new RemoveTransformation()) ,STELLAR(new StellarTransformation()) ,SELECT(new SelectTransformation()) + ,RENAME(new RenameTransformation()) ; FieldTransformation mapping; FieldTransformations(FieldTransformation mapping) { http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java new file mode 100644 index 0000000..f8b9374 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.field.transformation; + +import org.apache.metron.stellar.dsl.Context; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class RenameTransformation implements FieldTransformation{ + @Override + public Map<String, Object> map( Map<String, Object> input + , List<String> outputField + , LinkedHashMap<String, Object> fieldMappingConfig + , Context context + , Map<String, Object>... sensorConfig + ) + { + if(fieldMappingConfig == null || fieldMappingConfig.isEmpty()) { + return input; + } + Map<String, Object> ret = new HashMap<>(); + for(Map.Entry<String, Object> kv : input.entrySet()) { + Object renamed = fieldMappingConfig.get(kv.getKey()); + if(renamed != null) { + //if we're renaming, then we want to copy the field to the new name + ret.put(renamed.toString(), kv.getValue()); + //and remove the old field + ret.put(kv.getKey(), null); + } + else { + ret.put(kv.getKey(), kv.getValue()); + } + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java index 71a0298..b7557e8 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java @@ -85,18 +85,6 @@ public class FieldTransformationTest { { "fieldTransformations" : [ { - "transformation" : "IP_PROTOCOL" - } - ] - } - */ - @Multiline - public static String badConfigMissingInput; - - /** - { - "fieldTransformations" : [ - { "input" : "protocol" } ] @@ -113,10 +101,7 @@ public class FieldTransformationTest { Assert.assertEquals(ImmutableList.of("protocol"), c.getFieldTransformations().get(0).getInput()); } - @Test(expected = IllegalStateException.class) - public void testInValidSerde_missingInput() throws IOException { - SensorParserConfig.fromBytes(Bytes.toBytes(badConfigMissingInput)); - } + @Test(expected = IllegalStateException.class) public void testInValidSerde_missingMapping() throws IOException { http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java new file mode 100644 index 0000000..cacc818 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.field.transformation; + +import com.google.common.collect.Iterables; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metron.common.configuration.FieldTransformer; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.stellar.dsl.Context; +import org.json.simple.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; + +public class RenameTransformationTest { + /** + { + "fieldTransformations" : [ + { + "transformation" : "RENAME", + "config" : { + "old_field1" : "new_field1", + "old_field2" : "new_field2" + } + } + ] + } + */ + @Multiline + public static String smoketestConfig; + + @Test + public void smokeTest() throws Exception { + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(smoketestConfig)); + FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); + JSONObject input = new JSONObject(new HashMap<String, Object>() {{ + for(int i = 1;i <= 10;++i) { + put("old_field" + i, "f" + i); + } + }}); + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + Assert.assertEquals("f1", input.get("new_field1")); + Assert.assertEquals("f2", input.get("new_field2")); + for(int i = 3;i <= 10;++i) { + Assert.assertEquals("f" + i, input.get("old_field" + i)); + } + Assert.assertFalse(input.containsKey("old_field1")); + Assert.assertFalse(input.containsKey("old_field2")); + Assert.assertEquals(10, input.size()); + } + + /** + { + "fieldTransformations" : [ + { + "transformation" : "RENAME", + "config" : { + "old_field1" : "new_field1" + } + } + ] + } + */ + @Multiline + public static String renameMissingField; + @Test + public void renameMissingField() throws Exception { + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(renameMissingField)); + FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); + JSONObject input = new JSONObject(new HashMap<String, Object>() {{ + for(int i = 2;i <= 10;++i) { + put("old_field" + i, "f" + i); + } + }}); + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + Assert.assertFalse(input.containsKey("new_field1")); + for(int i = 2;i <= 10;++i) { + Assert.assertEquals("f" + i, input.get("old_field" + i)); + } + Assert.assertEquals(9, input.size()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-parsers/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index 1d2d834..e8b2896 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -313,10 +313,33 @@ into `{ "protocol" : "TCP", "source.type" : "bro", ...}` * `STELLAR` : This transformation executes a set of transformations expressed as [Stellar Language](../metron-common) statements. +* `RENAME` : This transformation allows users to rename a set of fields. Specifically, +the config is presumed to be the mapping. The keys to the config are the existing field names +and the values for the config map are the associated new field name. + +The following config will rename the fields `old_field` and `different_old_field` to +`new_field` and `different_new_field` respectively: +``` +{ +... + "fieldTransformations" : [ + { + "transformation" : "RENAME", + , "config" : { + "old_field" : "new_field", + "different_old_field" : "different_new_field" + } + } + ] +} +``` + + ### Assignment to `null` If, in your field transformation, you assign a field to `null`, the field will be removed. -You can use this capability to rename variables. +You can use this capability to rename variables. It is preferred, however, that the `RENAME` +field transformation is used in this situation as it is less awkward. Consider this example: ```
