Repository: metron Updated Branches: refs/heads/master 2e78df67c -> 610540ef0
METRON-1341 Projection FieldTransformation (simonellistonball via ottobackwards) closes apache/metron#861 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/610540ef Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/610540ef Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/610540ef Branch: refs/heads/master Commit: 610540ef034b7bd2555acb347f5c9be20a485954 Parents: 2e78df6 Author: simonellistonball <si...@simonellistonball.com> Authored: Mon Dec 11 16:09:40 2017 -0500 Committer: otto <o...@apache.org> Committed: Mon Dec 11 16:09:40 2017 -0500 ---------------------------------------------------------------------- .../transformation/FieldTransformations.java | 1 + .../transformation/SelectTransformation.java | 57 ++++++++++++ .../SelectTransformationTest.java | 98 ++++++++++++++++++++ metron-platform/metron-parsers/README.md | 17 ++++ 4 files changed, 173 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/610540ef/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java index 3565609..a905123 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java @@ -24,6 +24,7 @@ public enum FieldTransformations { IP_PROTOCOL(new IPProtocolTransformation()) ,REMOVE(new RemoveTransformation()) ,STELLAR(new StellarTransformation()) + ,SELECT(new SelectTransformation()) ; FieldTransformation mapping; FieldTransformations(FieldTransformation mapping) { http://git-wip-us.apache.org/repos/asf/metron/blob/610540ef/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java new file mode 100644 index 0000000..6f0defd --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SelectTransformation.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.field.transformation; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.metron.stellar.dsl.Context; + +public class SelectTransformation implements FieldTransformation { + + private static final List<String> systemFields = Arrays.asList("timestamp", "original_string", "source.type"); + + @Override + public Map<String, Object> map(Map<String, Object> input, List<String> outputField, + LinkedHashMap<String, Object> fieldMappingConfig, Context context, Map<String, Object>... sensorConfig) { + // note that we have to set the value to null for any field not in the output + // list, since FieldTransformer will otherwise put them back again from the + // original output. + + // note, this cannot be implemented with streams because HashMap merge guards + // against null values + + HashMap<String, Object> output = new HashMap<String, Object>(); + for (Entry<String, Object> e : input.entrySet()) { + if (outputField.contains(e.getKey())) { + output.put(e.getKey(), e.getValue()); + } else { + if (!systemFields.contains(e.getKey())) { + output.put(e.getKey(), null); + } + } + } + return output; + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/610540ef/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/SelectTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/SelectTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/SelectTransformationTest.java new file mode 100644 index 0000000..9219c6b --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/SelectTransformationTest.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.common.field.transformation; + +import java.util.HashMap; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metron.common.configuration.FieldTransformer; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.stellar.dsl.Context; +import org.json.simple.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.Iterables; + +public class SelectTransformationTest { + + public static String selectSingleFieldConfig = "{ \"fieldTransformations\" : [{\"output\": [\"field1\"] , \"transformation\": \"SELECT\" } ] }"; + + public static String selectMultiFieldConfig = "{ \"fieldTransformations\" : [{\"output\": [\"field1\", \"field2\"] , \"transformation\": \"SELECT\" } ] }"; + + @Test + public void testSingleFieldReturned() throws Exception { + SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectSingleFieldConfig)); + FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); + JSONObject input = new JSONObject(new HashMap<String, Object>() { + { + put("field1", "foo"); + put("field2", "bar"); + } + }); + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + + Assert.assertTrue(input.containsKey("field1")); + Assert.assertFalse(input.containsKey("field2")); + Assert.assertEquals(1, input.size()); + } + + @Test + public void testMulitpleFieldReturned() throws Exception { + SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectMultiFieldConfig)); + FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); + JSONObject input = new JSONObject(new HashMap<String, Object>() { + { + put("field1", "foo"); + put("field2", "bar"); + put("field3", "bar2"); + } + }); + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + + Assert.assertTrue(input.containsKey("field1")); + Assert.assertTrue(input.containsKey("field2")); + Assert.assertFalse(input.containsKey("field3")); + Assert.assertEquals(2, input.size()); + } + + @Test + public void testPreserveSystemFields() throws Exception { + SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectSingleFieldConfig)); + FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); + JSONObject input = new JSONObject(new HashMap<String, Object>() { + { + put("timestamp", 12345); + put("original_string", "foo,bar"); + put("source.type", "test"); + put("field1", "foo"); + put("field2", "bar"); + } + }); + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + + Assert.assertTrue(input.containsKey("timestamp")); + Assert.assertTrue(input.containsKey("original_string")); + Assert.assertTrue(input.containsKey("source.type")); + Assert.assertTrue(input.containsKey("field1")); + Assert.assertFalse(input.containsKey("field2")); + Assert.assertEquals(4, input.size()); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/610540ef/metron-platform/metron-parsers/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index 54dbef2..88dde44 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -216,6 +216,23 @@ whenever `field2` exists and whose corresponding equal to 'foo': } ``` +* `SELECT`: This transformation filters the fields in the message to include only the configured output fields, and drops any not explicitly included. + +For example: +``` +{ +... + "fieldTransformations" : [ + { + "output" : ["field1", "field2" ] + , "transformation" : "SELECT" + } + ] +} +``` + +when applied to a message containing keys field1, field2 and field3, will only output the first two. It is also worth noting that two standard fields - timestamp and original_source - will always be passed along whether they are listed in output or not, since they are considered core required fields. + * `IP_PROTOCOL` : This transformation maps IANA protocol numbers to consistent string representations. Consider the following sensor parser config to map the `protocol` field
