Repository: metron Updated Branches: refs/heads/master 85d12475c -> 9c5d9d766
METRON-1397 Support for JSON Path and complex documents in JSONMapParser closes apache/incubator-metron#914 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9c5d9d76 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9c5d9d76 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9c5d9d76 Branch: refs/heads/master Commit: 9c5d9d76644fc07bae36644906f52e0422f33d0e Parents: 85d1247 Author: ottobackwards <ottobackwa...@gmail.com> Authored: Thu Mar 15 14:17:31 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Thu Mar 15 14:17:31 2018 -0400 ---------------------------------------------------------------------- dependencies_with_url.csv | 3 + .../docker/rpm-docker/SPECS/metron.spec | 1 + .../jsonMapQuery/parsed/jsonMapExampleParsed | 2 + .../data/jsonMapQuery/raw/jsonMapExampleOutput | 1 + metron-platform/metron-parsers/README.md | 12 ++ metron-platform/metron-parsers/pom.xml | 5 + .../config/zookeeper/parsers/jsonMapQuery.json | 5 + .../metron/parsers/json/JSONMapParser.java | 145 +++++++++---- .../JSONMapQueryIntegrationTest.java | 36 ++++ .../validation/SampleDataValidation.java | 2 +- .../parsers/json/JSONMapParserQueryTest.java | 201 +++++++++++++++++++ .../metron/test/utils/ValidationUtils.java | 46 ++++- 12 files changed, 406 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/dependencies_with_url.csv ---------------------------------------------------------------------- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index e2b947b..1e73eb1 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -22,6 +22,9 @@ com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.4:compile,Apache v2, https://github.c com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD license,http://code.google.com/p/protobuf com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD license,http://code.google.com/p/protobuf com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/ +com.jayway.jsonpath:json-path:jar:2.3.0:compile,Apache v2,https://github.com/json-path/JsonPath +net.minidev:accessors-smart:jar:1.2:compile,Apache v2,https://github.com/netplex/json-smart-v2 +net.minidev:json-smart:jar:2.3:compile,Apache v2,https://github.com/netplex/json-smart-v2 com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 3.0,https://github.com/maxmind/MaxMind-DB com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache v2,https://github.com/maxmind/GeoIP2-java com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:compile,CDDL,http://jaxb.java.net/ http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 265d595..cc01d7c 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -147,6 +147,7 @@ This package installs the Metron Parser files %{metron_home}/bin/start_parser_topology.sh %{metron_home}/config/zookeeper/parsers/bro.json %{metron_home}/config/zookeeper/parsers/jsonMap.json +%{metron_home}/config/zookeeper/parsers/jsonMapQuery.json %{metron_home}/config/zookeeper/parsers/snort.json %{metron_home}/config/zookeeper/parsers/squid.json %{metron_home}/config/zookeeper/parsers/websphere.json http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed new file mode 100644 index 0000000..e614bda --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed @@ -0,0 +1,2 @@ +{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ], "original_string":"{ \"string\" : \"bar\", \"number\" : 2, \"ignored\" : [ \"blah\" ] }","timestamp":1000000000000, "source.type":"jsonMapQuery","guid":"this-is-random-uuid-will-be-36-chars" } +{ "number" : 7 , "original_string" : "{ \"number\" : 7 }", "source.type":"jsonMapQuery","timestamp":1000000000000,"guid":"this-is-random-uuid-will-be-36-chars"} http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/raw/jsonMapExampleOutput ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/raw/jsonMapExampleOutput b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/raw/jsonMapExampleOutput new file mode 100644 index 0000000..8f25f4f --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/raw/jsonMapExampleOutput @@ -0,0 +1 @@ +{"foo":[{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ] },{ "number" : 7 }]} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-parsers/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md index ade0f51..3d9fdfe 100644 --- a/metron-platform/metron-parsers/README.md +++ b/metron-platform/metron-parsers/README.md @@ -43,6 +43,7 @@ There are two general types types of parsers: * `UNFOLD` : Unfold inner maps. So `{ "foo" : { "bar" : 1} }` would turn into `{"foo.bar" : 1}` * `ALLOW` : Allow multidimensional maps * `ERROR` : Throw an error when a multidimensional map is encountered + * `jsonpQuery` : A [JSON Path](#json_path) query string. If present, the result of the JSON Path query should be a list of messages. This is useful if you have a JSON document which contains a list or array of messages embedded in it, and you do not have another means of splitting the message. * A field called `timestamp` is expected to exist and, if it does not, then current time is inserted. ## Parser Architecture @@ -520,3 +521,14 @@ be customized by modifying the arguments sent to this utility. Finally, if workers and executors are new to you, the following might be of use to you: * [Understanding the Parallelism of a Storm Topology](http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/) + +## JSON Path + +> "JSONPath expressions always refer to a JSON structure in the same way as XPath expression are used in combination with an XML document." +> ~ Stefan Goessner + + +- [JSON Path concept](http://goessner.net/articles/JsonPath/) +- [Read about JSON Path library Apache Metron uses](https://github.com/json-path/JsonPath) +- [Try JSON Path expressions online](http://jsonpath.herokuapp.com) + http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-parsers/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index f856654..c481864 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -256,6 +256,11 @@ <version>2.2.6</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.jayway.jsonpath</groupId> + <artifactId>json-path</artifactId> + <version>2.3.0</version> + </dependency> </dependencies> <build> <plugins> http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json new file mode 100644 index 0000000..7dad779 --- /dev/null +++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMapQuery.json @@ -0,0 +1,5 @@ +{ + "parserClassName":"org.apache.metron.parsers.json.JSONMapParser", + "sensorTopic":"jsonMapQuery", + "parserConfig": {"jsonpQuery":"$.foo"} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java index 7e5468f..bddf35d 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,65 +15,116 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.json; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.parsers.BasicParser; -import org.json.simple.JSONObject; - +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.TypeRef; +import com.jayway.jsonpath.spi.cache.CacheProvider; +import com.jayway.jsonpath.spi.cache.LRUCache; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.json.JsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import com.jayway.jsonpath.spi.mapper.MappingProvider; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.parsers.BasicParser; +import org.json.simple.JSONObject; public class JSONMapParser extends BasicParser { - private static interface Handler { + + private interface Handler { + JSONObject handle(String key, Map value, JSONObject obj); } - public static enum MapStrategy implements Handler { - DROP((key, value, obj) -> obj) - ,UNFOLD( (key, value, obj) -> { - return recursiveUnfold(key,value,obj); - }) - ,ALLOW((key, value, obj) -> { + + @SuppressWarnings("unchecked") + public enum MapStrategy implements Handler { + DROP((key, value, obj) -> obj), UNFOLD((key, value, obj) -> { + return recursiveUnfold(key, value, obj); + }), ALLOW((key, value, obj) -> { obj.put(key, value); return obj; - }) - ,ERROR((key, value, obj) -> { - throw new IllegalStateException("Unable to process " + key + " => " + value + " because value is a map."); - }) - ; + }), ERROR((key, value, obj) -> { + throw new IllegalStateException( + "Unable to process " + key + " => " + value + " because value is a map."); + }); Handler handler; + MapStrategy(Handler handler) { this.handler = handler; } - private static JSONObject recursiveUnfold(String key, Map value, JSONObject obj){ + @SuppressWarnings("unchecked") + private static JSONObject recursiveUnfold(String key, Map value, JSONObject obj) { Set<Map.Entry<Object, Object>> entrySet = value.entrySet(); - for(Map.Entry<Object, Object> kv : entrySet) { + for (Map.Entry<Object, Object> kv : entrySet) { String newKey = Joiner.on(".").join(key, kv.getKey().toString()); - if(kv.getValue() instanceof Map){ - recursiveUnfold(newKey,(Map)kv.getValue(),obj); - }else { + if (kv.getValue() instanceof Map) { + recursiveUnfold(newKey, (Map) kv.getValue(), obj); + } else { obj.put(newKey, kv.getValue()); } } return obj; } + @Override public JSONObject handle(String key, Map value, JSONObject obj) { return handler.handle(key, value, obj); } } + public static final String MAP_STRATEGY_CONFIG = "mapStrategy"; + public static final String JSONP_QUERY = "jsonpQuery"; + private MapStrategy mapStrategy = MapStrategy.DROP; + private TypeRef<List<Map<String, Object>>> typeRef = new TypeRef<List<Map<String, Object>>>() { + }; + private String jsonpQuery = null; + @Override public void configure(Map<String, Object> config) { String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, MapStrategy.DROP.name()); mapStrategy = MapStrategy.valueOf(strategyStr); + if (config.containsKey(JSONP_QUERY)) { + jsonpQuery = (String) config.get(JSONP_QUERY); + Configuration.setDefaults(new Configuration.Defaults() { + + private final JsonProvider jsonProvider = new JacksonJsonProvider(); + private final MappingProvider mappingProvider = new JacksonMappingProvider(); + + @Override + public JsonProvider jsonProvider() { + return jsonProvider; + } + + @Override + public MappingProvider mappingProvider() { + return mappingProvider; + } + + @Override + public Set<Option> options() { + return EnumSet.of(Option.SUPPRESS_EXCEPTIONS); + } + }); + + if (CacheProvider.getCache() == null) { + CacheProvider.setCache(new LRUCache(100)); + } + } } /** @@ -87,22 +138,36 @@ public class JSONMapParser extends BasicParser { /** * Take raw data and convert it to a list of messages. * - * @param rawMessage * @return If null is returned, this is treated as an empty list. */ @Override + @SuppressWarnings("unchecked") public List<JSONObject> parse(byte[] rawMessage) { try { String originalString = new String(rawMessage); - //convert the JSON blob into a String -> Object map - Map<String, Object> rawMap = JSONUtils.INSTANCE.load(originalString, JSONUtils.MAP_SUPPLIER); - JSONObject ret = normalizeJSON(rawMap); - ret.put("original_string", originalString ); - if(!ret.containsKey("timestamp")) { - //we have to ensure that we have a timestamp. This is one of the pre-requisites for the parser. - ret.put("timestamp", System.currentTimeMillis()); + List<Map<String, Object>> messages = new ArrayList<>(); + + if (!StringUtils.isEmpty(jsonpQuery)) { + Object parsedObject = JsonPath.parse(new String(rawMessage)).read(jsonpQuery, typeRef); + if(parsedObject != null) { + messages.addAll((List<Map<String,Object>>)parsedObject); + } + } else { + messages.add(JSONUtils.INSTANCE.load(originalString, JSONUtils.MAP_SUPPLIER)); + } + + ArrayList<JSONObject> parsedMessages = new ArrayList<>(); + for (Map<String, Object> rawMessageMap : messages) { + JSONObject originalJsonObject = new JSONObject(rawMessageMap); + JSONObject ret = normalizeJson(rawMessageMap); + // the original string is the original for THIS sub message + ret.put("original_string", originalJsonObject.toJSONString()); + if (!ret.containsKey("timestamp")) { + ret.put("timestamp", System.currentTimeMillis()); + } + parsedMessages.add(ret); } - return ImmutableList.of(ret); + return Collections.unmodifiableList(parsedMessages); } catch (Throwable e) { String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage(); LOG.error(message, e); @@ -111,18 +176,16 @@ public class JSONMapParser extends BasicParser { } /** - * Process all sub-maps via the MapHandler. We have standardized on one-dimensional maps as our data model.. - * - * @param map - * @return + * Process all sub-maps via the MapHandler. + * We have standardized on one-dimensional maps as our data model. */ - private JSONObject normalizeJSON(Map<String, Object> map) { + @SuppressWarnings("unchecked") + private JSONObject normalizeJson(Map<String, Object> map) { JSONObject ret = new JSONObject(); - for(Map.Entry<String, Object> kv : map.entrySet()) { - if(kv.getValue() instanceof Map) { + for (Map.Entry<String, Object> kv : map.entrySet()) { + if (kv.getValue() instanceof Map) { mapStrategy.handle(kv.getKey(), (Map) kv.getValue(), ret); - } - else { + } else { ret.put(kv.getKey(), kv.getValue()); } } http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapQueryIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapQueryIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapQueryIntegrationTest.java new file mode 100644 index 0000000..cec6242 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapQueryIntegrationTest.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.integration; + +import java.util.ArrayList; +import java.util.List; +import org.apache.metron.parsers.integration.validation.SampleDataValidation; + +public class JSONMapQueryIntegrationTest extends ParserIntegrationTest { + @Override + String getSensorType() { + return "jsonMapQuery"; + } + + @Override + List<ParserValidation> getValidations() { + return new ArrayList<ParserValidation>() {{ + add(new SampleDataValidation()); + }}; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java index 9ea9b71..1dff22f 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/validation/SampleDataValidation.java @@ -41,7 +41,7 @@ public class SampleDataValidation implements ParserValidation { String expectedMessage = new String(expectedMessages.get(i)); String actualMessage = new String(actualMessages.get(i)); try { - ValidationUtils.assertJSONEqual(expectedMessage, actualMessage); + ValidationUtils.assertJsonEqual(expectedMessage, actualMessage); } catch (Throwable t) { System.out.println("expected: " + expectedMessage); System.out.println("actual: " + actualMessage); http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java new file mode 100644 index 0000000..9f8c26b --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserQueryTest.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.json; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.List; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.log4j.Level; +import org.apache.metron.parsers.BasicParser; +import org.apache.metron.test.utils.UnitTestHelper; +import org.json.simple.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +public class JSONMapParserQueryTest { + + /** + * { + * "foo" : + * [ + * { "name" : "foo1", "value" : "bar", "number" : 1.0 }, + * { "name" : "foo2", "value" : "baz", "number" : 2.0 } + * ] + * } + */ + @Multiline + static String JSON_LIST; + + /** + * { "name" : "foo1", "value" : "bar", "number" : 1.0 } + */ + @Multiline + static String JSON_SINGLE; + + /** + * { "name" : "foo2", "value" : "baz", "number" : 2.0 } + */ + @Multiline + static String JSON_SINGLE2; + + @Test + public void testHappyPath() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(new HashMap<String, Object>() {{ + put(JSONMapParser.JSONP_QUERY, "$.foo"); + }}); + List<JSONObject> output = parser.parse(JSON_LIST.getBytes()); + Assert.assertEquals(output.size(), 2); + //don't forget the timestamp field! + Assert.assertEquals(output.get(0).size(), 5); + JSONObject message = output.get(0); + Assert.assertEquals("foo1", message.get("name")); + Assert.assertEquals("bar", message.get("value")); + Assert.assertEquals(1.0, message.get("number")); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + Assert.assertNotNull(message.get("number")); + Assert.assertTrue(message.get("number") instanceof Number); + + message = output.get(1); + Assert.assertEquals("foo2", message.get("name")); + Assert.assertEquals("baz", message.get("value")); + Assert.assertEquals(2.0, message.get("number")); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + Assert.assertNotNull(message.get("number")); + Assert.assertTrue(message.get("number") instanceof Number); + + } + + @Test(expected = IllegalStateException.class) + public void testInvalidJSONPathThrows() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(new HashMap<String, Object>() {{ + put(JSONMapParser.JSONP_QUERY, "$$..$$SDSE$#$#."); + }}); + List<JSONObject> output = parser.parse(JSON_LIST.getBytes()); + + } + + @Test + public void testNoMatchesNoExceptions() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(new HashMap<String, Object>() {{ + put(JSONMapParser.JSONP_QUERY, "$.foo"); + }}); + List<JSONObject> output = parser.parse(JSON_SINGLE.getBytes()); + Assert.assertEquals(0, output.size()); + } + + /** + * { + * "foo" : + * [ + * { + * "collection" : { "blah" : 7, "blah2" : "foo", "bigblah" : { "innerBlah" : "baz", "reallyInnerBlah" : { "color" : "grey" }}} + * }, + * { + * "collection" : { "blah" : 8, "blah2" : "bar", "bigblah" : { "innerBlah" : "baz2", "reallyInnerBlah" : { "color" : "blue" }}} + * } + * ] + * } + */ + @Multiline + static String collectionHandlingJSON; + + @Test + public void testCollectionHandlingDrop() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(new HashMap<String, Object>() {{ + put(JSONMapParser.JSONP_QUERY, "$.foo"); + }}); + List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes()); + Assert.assertEquals(output.size(), 2); + + //don't forget the timestamp field! + Assert.assertEquals(output.get(0).size(), 2); + + JSONObject message = output.get(0); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + + message = output.get(1); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + } + + @Test(expected = IllegalStateException.class) + public void testCollectionHandlingError() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(ImmutableMap + .of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name(), + JSONMapParser.JSONP_QUERY, "$.foo")); + UnitTestHelper.setLog4jLevel(BasicParser.class, Level.FATAL); + parser.parse(collectionHandlingJSON.getBytes()); + UnitTestHelper.setLog4jLevel(BasicParser.class, Level.ERROR); + } + + + @Test + public void testCollectionHandlingAllow() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(ImmutableMap + .of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name(), + JSONMapParser.JSONP_QUERY, "$.foo")); + List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes()); + Assert.assertEquals(output.size(), 2); + Assert.assertEquals(output.get(0).size(), 3); + JSONObject message = output.get(0); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + + Assert.assertEquals(output.get(1).size(), 3); + message = output.get(1); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + } + + @Test + public void testCollectionHandlingUnfold() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(ImmutableMap + .of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name(), + JSONMapParser.JSONP_QUERY, "$.foo")); + List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes()); + Assert.assertEquals(output.size(), 2); + Assert.assertEquals(output.get(0).size(), 6); + JSONObject message = output.get(0); + Assert.assertEquals(message.get("collection.blah"), 7); + Assert.assertEquals(message.get("collection.blah2"), "foo"); + Assert.assertEquals(message.get("collection.bigblah.innerBlah"), "baz"); + Assert.assertEquals(message.get("collection.bigblah.reallyInnerBlah.color"), "grey"); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + + Assert.assertEquals(output.get(1).size(), 6); + message = output.get(1); + Assert.assertEquals(message.get("collection.blah"), 8); + Assert.assertEquals(message.get("collection.blah2"), "bar"); + Assert.assertEquals(message.get("collection.bigblah.innerBlah"), "baz2"); + Assert.assertEquals(message.get("collection.bigblah.reallyInnerBlah.color"), "blue"); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ValidationUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ValidationUtils.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ValidationUtils.java index 279caa3..98fd258 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ValidationUtils.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ValidationUtils.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,33 +15,57 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.metron.test.utils; -import org.codehaus.jackson.map.ObjectMapper; -import org.junit.Assert; +package org.apache.metron.test.utils; import java.io.IOException; import java.util.Map; +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.Assert; public class ValidationUtils { - public static void assertJSONEqual(String expected, String actual) throws IOException { + /** + * Validates that two JSON Strings are equal in value. + * Since JSON does not guarentee order of fields, we cannot just compare Strings. + * <p> + * This utility understands that the 'original_string' field may itself hold JSON, + * and will attempt to validate that field as json if it fails straight string compare + * </p> + * @param expected the expected string value + * @param actual the actual string value + * @throws IOException if there is an issue parsing as json + */ + public static void assertJsonEqual(String expected, String actual) throws IOException { ObjectMapper mapper = new ObjectMapper(); Map m1 = mapper.readValue(expected, Map.class); Map m2 = mapper.readValue(actual, Map.class); - for(Object k : m1.keySet()) { + for (Object k : m1.keySet()) { Object v1 = m1.get(k); Object v2 = m2.get(k); - if(v2 == null) { + if (v2 == null) { Assert.fail("Unable to find key: " + k + " in output"); } - if(k.equals("timestamp") || k.equals("guid")) { + if (k.equals("timestamp") || k.equals("guid")) { //TODO: Take the ?!?@ timestamps out of the reference file. Assert.assertEquals(v1.toString().length(), v2.toString().length()); - } - else if(!v2.equals(v1)) { - Assert.assertEquals("value mismatch for " + k ,v1, v2); + } else if (!v2.equals(v1)) { + boolean goodDeepDown = false; + // if this fails, but is the original_string it may be in json format + // where the field/value order may be random + if (((String) k).equals("original_string")) { + try { + mapper.readValue((String) v1, Map.class); + assertJsonEqual((String) v1, (String) v2); + goodDeepDown = true; + } catch (Exception e) { + // nothing, the original fail stands + } + } + if (!goodDeepDown) { + Assert.assertEquals("value mismatch for " + k, v1, v2); + } } } Assert.assertEquals(m1.size(), m2.size());