Repository: metron Updated Branches: refs/heads/feature/METRON-1136-extensions-parsers 1c63c1eb3 -> 24b668b0b
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java new file mode 100644 index 0000000..1b4df1e --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.dsl.functions; + +import com.google.common.collect.Iterables; +import org.apache.metron.stellar.dsl.BaseStellarFunction; +import org.apache.metron.stellar.dsl.Stellar; + +import java.util.*; + +public class SetFunctions { + @Stellar(name="INIT" + , namespace="SET" + , description="Creates a new set" + , params = { "input? - An initialization of the set"} + , returns = "A Set" + ) + public static class SetInit extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + LinkedHashSet<Object> ret = new LinkedHashSet<>(); + if(list.size() == 1) { + Object o = list.get(0); + if(o != null) { + if (o instanceof Iterable) { + Iterables.addAll(ret, (Iterable) o); + } + else { + throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass()); + } + } + + } + return ret; + } + } + + @Stellar(name="ADD" + , namespace="SET" + , description="Adds to a set" + , params = {"set - The set to add to" + ,"o - object to add to set" + } + , returns = "A Set" + ) + public static class SetAdd extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + if(list.size() < 1) { + return null; + } + LinkedHashSet<Object> ret = (LinkedHashSet<Object>)list.get(0); + if(ret == null) { + ret = new LinkedHashSet<>(); + } + for(int i = 1;i < list.size();++i) { + Object o = list.get(i); + if (o != null) { + ret.add(o); + } + } + return ret; + } + } + + @Stellar(name="REMOVE" + , namespace="SET" + , description="Removes from a set" + , params = {"set - The set to add to" + ,"o - object to add to set" + } + , returns = "A Set" + ) + public static class SetRemove extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + if(list.size() < 1) { + return null; + } + LinkedHashSet<Object> ret = (LinkedHashSet<Object>)list.get(0); + if(ret == null) { + ret = new LinkedHashSet<>(); + } + for(int i = 1;i < list.size();++i) { + Object o = list.get(i); + if (o != null) { + ret.remove(o); + } + } + return ret; + } + } + + @Stellar(name="MERGE" + , namespace="SET" + , description="Merges a list of sets" + , params = {"sets - A collection of sets to merge" + } + , returns = "A Set" + ) + public static class SetMerge extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + if(list.size() < 1) { + return null; + } + LinkedHashSet<Object> ret = new LinkedHashSet<>(); + if(list.size() > 0) { + Object o = list.get(0); + if(o != null) { + if(!(o instanceof Iterable)) { + throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass()); + } + Iterable<? extends Iterable> sets = (Iterable<? extends Iterable>) o; + + for (Iterable s : sets) { + if (s != null) { + if(!(s instanceof Iterable)) { + throw new IllegalArgumentException("Expected an Iterable, but " + s + " is of type " + s.getClass()); + } + Iterables.addAll(ret, s); + } + } + } + } + return ret; + } + } + + @Stellar(name="INIT" + , namespace="MULTISET" + , description="Creates an empty multiset, which is a map associating objects to their instance counts." + , params = { "input? - An initialization of the multiset"} + , returns = "A multiset" + ) + public static class MultiSetInit extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + LinkedHashMap<Object, Integer> ret = new LinkedHashMap<>(); + if (list.size() >= 1) { + Object o = list.get(0); + if (o != null) { + if (!(o instanceof Iterable)) { + throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass()); + } + for (Object obj : (Iterable) o) { + ret.merge(obj, 1, (k, one) -> k + one); + } + } + } + return ret; + } + } + + @Stellar(name="ADD" + , namespace="MULTISET" + , description="Adds to a multiset, which is a map associating objects to their instance counts." + , params = {"set - The multiset to add to" + ,"o - object to add to multiset" + } + , returns = "A multiset" + ) + public static class MultiSetAdd extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + if(list.size() < 1) { + return null; + } + LinkedHashMap<Object, Integer> ret = (LinkedHashMap<Object, Integer>)list.get(0); + if(ret == null) { + ret = new LinkedHashMap<>(); + } + for(int i = 1;i < list.size();++i) { + Object o = list.get(i); + if (o != null) { + ret.merge(o, 1, (k, one) -> k + one); + } + } + return ret; + } + } + + @Stellar(name="REMOVE" + , namespace="MULTISET" + , description="Removes from a multiset, which is a map associating objects to their instance counts." + , params = {"set - The multiset to add to" + ,"o - object to remove from multiset" + } + , returns = "A multiset" + ) + public static class MultiSetRemove extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + if(list.size() < 1) { + return null; + } + LinkedHashMap<Object, Integer> ret = (LinkedHashMap<Object, Integer>)list.get(0); + if(ret == null) { + ret = new LinkedHashMap<>(); + } + for(int i = 1;i < list.size();++i) { + Object o = list.get(i); + if (o != null) { + Integer cnt = ret.get(o); + if(cnt == null) { + continue; + } + if(cnt == 1) { + ret.remove(o); + } + else { + ret.put(o, cnt - 1); + } + } + } + return ret; + } + } + + @Stellar(name="MERGE" + , namespace="MULTISET" + , description="Merges a list of multisets, which is a map associating objects to their instance counts." + , params = {"sets - A collection of multisets to merge" + } + , returns = "A multiset" + ) + public static class MultiSetMerge extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + if(list.size() < 1) { + return null; + } + LinkedHashMap<Object, Integer> ret = new LinkedHashMap<>(); + if(list.size() > 0) { + Iterable<Map<Object, Integer>> maps = (Iterable<Map<Object, Integer>>)list.get(0); + for(Map<Object, Integer> s : maps) { + if(s != null) { + for (Map.Entry<Object, Integer> kv : s.entrySet()) { + ret.merge(kv.getKey(), kv.getValue(), (k, cnt) -> k + cnt); + } + } + } + } + return ret; + } + } + + + @Stellar(name="TO_SET" + , namespace="MULTISET" + , description="Create a set out of a multiset, which is a map associating objects to their instance counts." + , params = {"multiset - The multiset to convert." + } + , returns = "The set of objects in the multiset ignoring multiplicity" + ) + public static class MultiSetToSet extends BaseStellarFunction { + @Override + public Object apply(List<Object> list) { + if(list.size() < 1) { + return null; + } + LinkedHashSet<Object> ret = new LinkedHashSet<>(); + if(list.size() == 1) { + Map<Object, Integer> multiset = (Map<Object, Integer>)list.get(0); + if(multiset != null) { + ret.addAll(multiset.keySet()); + } + } + return ret; + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java index 289fa7f..4dc4790 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java @@ -18,15 +18,20 @@ package org.apache.metron.stellar.dsl.functions; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.utils.JSONUtils; import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.common.utils.ConversionUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -321,6 +326,58 @@ public class StringFunctions { } } + @Stellar( name="SUBSTRING" + , description = "Returns a substring of a string" + , params = { + "input - The string to take the substring of", + "start - The starting position (0-based and inclusive)", + "end? - The ending position (0-based and exclusive)" + } + , returns = "The substring of the input" + ) + public static class Substring extends BaseStellarFunction { + + @Override + public Object apply(List<Object> strings) { + + if(strings == null || strings.size() < 2 ) { + throw new IllegalArgumentException("SUBSTRING requires (at least) 2 arguments: the input and the start position (inclusive)"); + } + Object varObj = strings.get(0); + if(varObj != null && !(varObj instanceof String)) { + throw new IllegalArgumentException("SUBSTRING input must be a String"); + } + String var = varObj == null?null: (String) varObj; + Object startObj = strings.get(1); + if(startObj != null && !(startObj instanceof Number)) { + throw new IllegalArgumentException("SUBSTRING start must be an Number"); + } + Integer start = startObj == null?null:((Number)startObj).intValue(); + Integer end = null; + if(strings.size() > 2) { + Object endObj = strings.get(2); + if(endObj != null && !(endObj instanceof Number)) { + throw new IllegalArgumentException("SUBSTRING end must be an Number"); + } + end = endObj == null ? null : ((Number) endObj).intValue(); + } + if(var == null || start == null) { + return null; + } + else if(var.length() == 0) { + return var; + } + else { + if(end == null) { + return var.substring(start); + } + else { + return var.substring(start, end); + } + } + } + } + @Stellar( name="CHOMP" , description = "Removes one newline from end of a String if it's there, otherwise leave it alone. A newline is \"\\n\", \"\\r\", or \"\\r\\n\"" , params = { "the String to chomp a newline from, may be null"} @@ -450,4 +507,114 @@ public class StringFunctions { } } + @Stellar(name = "TO_JSON_OBJECT" + , description = "Returns a JSON object for the specified JSON string" + , params = { + "str - the JSON String to convert, may be null" + } + , returns = "an Object containing the parsed JSON string" + ) + public static class ToJsonObject extends BaseStellarFunction { + + @Override + public Object apply(List<Object> strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[TO_JSON_OBJECT] incorrect arguments. Usage: TO_JSON_OBJECT <String>"); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return JSON Object + try { + return JSONUtils.INSTANCE.load((String) strings.get(0), Object.class); + } catch (JsonProcessingException ex) { + throw new ParseException("Valid JSON string not supplied", ex); + } catch (IOException e) { + e.printStackTrace(); + } + } + return new ParseException("Unable to parse JSON string"); + } + } + + @Stellar(name = "TO_JSON_MAP" + , description = "Returns a MAP object for the specified JSON string" + , params = { + "str - the JSON String to convert, may be null" + } + , returns = "a MAP object containing the parsed JSON string" + ) + public static class ToJsonMap extends BaseStellarFunction { + + @Override + public Object apply(List<Object> strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[TO_JSON_MAP] incorrect arguments. Usage: TO_JSON_MAP <JSON String>"); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return parsed JSON Object as a HashMap + try { + return JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<Map<String, Object>>(){}); + } catch (JsonProcessingException ex) { + throw new ParseException("Valid JSON string not supplied", ex); + } catch (IOException e) { + e.printStackTrace(); + } + } + return new ParseException("Unable to parse JSON string"); + } + } + + @Stellar(name = "TO_JSON_LIST" + , description = "Returns a List object for the specified JSON string" + , params = { + "str - the JSON String to convert, may be null" + } + , returns = "a List object containing the parsed JSON string" + ) + public static class ToJsonList extends BaseStellarFunction { + + @Override + public Object apply(List<Object> strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[TO_JSON_LIST] incorrect arguments. Usage: TO_JSON_LIST <JSON String>"); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return parsed JSON Object as a List + try { + return (List) JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<List<Object>>(){}); + } catch (JsonProcessingException ex) { + throw new ParseException("Valid JSON string not supplied", ex); + } catch (IOException e) { + e.printStackTrace(); + throw new ParseException("Valid JSON string not supplied", e); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java index d6c3713..af86902 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@ -625,6 +625,18 @@ public class BasicStellarTest { } @Test + + public void testShortCircuit_mixedBoolOps() throws Exception { + final Map<String, String> variableMap = new HashMap<String, String>(); + Assert.assertTrue(runPredicate("(false && true) || true" + , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v)))); + Assert.assertTrue(runPredicate("(false && false) || true" + , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v)))); + Assert.assertFalse(runPredicate("(true || true) && false" + , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v)))); + } + + @Test public void testInString() throws Exception { final Map<String, String> variableMap = new HashMap<String, String>() {{ put("foo", "casey"); http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java new file mode 100644 index 0000000..1aa8eb4 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java @@ -0,0 +1,326 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.dsl.functions; + +import org.apache.metron.stellar.common.utils.StellarProcessorUtils; +import org.apache.metron.stellar.dsl.ParseException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class SetFunctionsTest { + + @Test(expected=ParseException.class) + public void multisetInitTest_wrongType() throws Exception { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT({ 'foo' : 'bar'})", new HashMap<>()); + } + + @Test + public void multisetInitTest() throws Exception { + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT()", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + //int initialization + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT([1,2,3,2])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.containsKey(1)); + Assert.assertEquals(1,(int)s.get(1)); + Assert.assertTrue(s.containsKey(2)); + Assert.assertEquals(2,(int)s.get(2)); + Assert.assertTrue(s.containsKey(3)); + Assert.assertEquals(1,(int)s.get(3)); + } + //string initialization + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT(['one','two','three','two'])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.containsKey("one")); + Assert.assertEquals(1,(int)s.get("one")); + Assert.assertTrue(s.containsKey("two")); + Assert.assertEquals(2,(int)s.get("two")); + Assert.assertTrue(s.containsKey("three")); + Assert.assertEquals(1,(int)s.get("three")); + + } + } + + @Test + public void multisetAddTest() throws Exception { + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(MULTISET_INIT(), 1)", new HashMap<>()); + Assert.assertEquals(1, s.size()); + Assert.assertTrue(s.containsKey(1)); + Assert.assertEquals(1,(int)s.get(1)); + } + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(null, 1)", new HashMap<>()); + Assert.assertEquals(1, s.size()); + Assert.assertTrue(s.containsKey(1)); + Assert.assertEquals(1,(int)s.get(1)); + } + //int + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(MULTISET_INIT([1,2,3,4,4]), 4)", new HashMap<>()); + Assert.assertEquals(4, s.size()); + Assert.assertTrue(s.containsKey(1)); + Assert.assertEquals(1,(int)s.get(1)); + Assert.assertTrue(s.containsKey(2)); + Assert.assertEquals(1,(int)s.get(2)); + Assert.assertTrue(s.containsKey(3)); + Assert.assertEquals(1,(int)s.get(3)); + Assert.assertTrue(s.containsKey(4)); + Assert.assertEquals(3,(int)s.get(4)); + } + //string + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(MULTISET_INIT(['one','two','three', 'four', 'four']), 'four')", new HashMap<>()); + Assert.assertEquals(4, s.size()); + Assert.assertTrue(s.containsKey("one")); + Assert.assertEquals(1,(int)s.get("one")); + Assert.assertTrue(s.containsKey("two")); + Assert.assertEquals(1,(int)s.get("two")); + Assert.assertTrue(s.containsKey("three")); + Assert.assertEquals(1,(int)s.get("three")); + Assert.assertTrue(s.containsKey("four")); + Assert.assertEquals(3,(int)s.get("four")); + } + } +@Test + public void multisetRemoveTest() throws Exception { + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(MULTISET_INIT([1]), 1)", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(null, 1)", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + //int + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(MULTISET_INIT([1,2,3,2]), 2)", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.containsKey(1)); + Assert.assertEquals(1, (int)s.get(1)); + Assert.assertTrue(s.containsKey(2)); + Assert.assertEquals(1, (int)s.get(2)); + Assert.assertTrue(s.containsKey(3)); + Assert.assertEquals(1, (int)s.get(3)); + } + //string + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(MULTISET_INIT(['one','two','three', 'two']), 'two')", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.containsKey("one")); + Assert.assertEquals(1, (int)s.get("one")); + Assert.assertTrue(s.containsKey("two")); + Assert.assertEquals(1, (int)s.get("two")); + Assert.assertTrue(s.containsKey("three")); + Assert.assertEquals(1, (int)s.get("three")); + } + } + + @Test(expected=ParseException.class) + public void multisetMergeTest_wrongType() throws Exception { + + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_MERGE({ 'bar' : 'foo' } )", new HashMap<>()); + } + + @Test + public void multisetMergeTest() throws Exception { + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_MERGE([MULTISET_INIT(), MULTISET_INIT(null), null])", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + //int + { + Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_MERGE([MULTISET_INIT([1,2]), MULTISET_INIT([2,3]), null, MULTISET_INIT()])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.containsKey(1)); + Assert.assertEquals(1, (int)s.get(1)); + Assert.assertTrue(s.containsKey(2)); + Assert.assertEquals(2, (int)s.get(2)); + Assert.assertTrue(s.containsKey(3)); + Assert.assertEquals(1, (int)s.get(3)); + } + //string + { + Map<Object, Integer> s = (Map<Object, Integer>)StellarProcessorUtils.run("MULTISET_MERGE([MULTISET_INIT(['one','two']), MULTISET_INIT(['two', 'three'])])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.containsKey("one")); + Assert.assertEquals(1, (int)s.get("one")); + Assert.assertTrue(s.containsKey("two")); + Assert.assertEquals(2, (int)s.get("two")); + Assert.assertTrue(s.containsKey("three")); + Assert.assertEquals(1, (int)s.get("three")); + } + } + + @Test(expected=ParseException.class) + public void setInitTest_wrongType() throws Exception { + Set s = (Set) StellarProcessorUtils.run("SET_INIT({ 'foo' : 2})", new HashMap<>()); + } + + @Test + public void setInitTest() throws Exception { + { + Set s = (Set) StellarProcessorUtils.run("SET_INIT()", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + //int initialization + { + Set s = (Set) StellarProcessorUtils.run("SET_INIT([1,2,3])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.contains(1)); + Assert.assertTrue(s.contains(2)); + Assert.assertTrue(s.contains(3)); + } + //string initialization + { + Set s = (Set) StellarProcessorUtils.run("SET_INIT(['one','two','three'])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.contains("one")); + Assert.assertTrue(s.contains("two")); + Assert.assertTrue(s.contains("three")); + } + } + + @Test + public void multisetToSetTest() throws Exception { + { + Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(MULTISET_INIT(), 1))", new HashMap<>()); + Assert.assertEquals(1, s.size()); + Assert.assertTrue(s.contains(1)); + } + { + Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(null, 1))", new HashMap<>()); + Assert.assertEquals(1, s.size()); + Assert.assertTrue(s.contains(1)); + } + //int + { + Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(MULTISET_INIT([1,2,3]), 4))", new HashMap<>()); + Assert.assertEquals(4, s.size()); + Assert.assertTrue(s.contains(1)); + Assert.assertTrue(s.contains(2)); + Assert.assertTrue(s.contains(3)); + Assert.assertTrue(s.contains(4)); + } + //string + { + Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(MULTISET_INIT(['one','two','three']), 'four'))", new HashMap<>()); + Assert.assertEquals(4, s.size()); + Assert.assertTrue(s.contains("one")); + Assert.assertTrue(s.contains("two")); + Assert.assertTrue(s.contains("three")); + Assert.assertTrue(s.contains("four")); + } + } + + @Test + public void setAddTest() throws Exception { + { + Set s = (Set) StellarProcessorUtils.run("SET_ADD(SET_INIT(), 1)", new HashMap<>()); + Assert.assertEquals(1, s.size()); + Assert.assertTrue(s.contains(1)); + } + { + Set s = (Set) StellarProcessorUtils.run("SET_ADD(null, 1)", new HashMap<>()); + Assert.assertEquals(1, s.size()); + Assert.assertTrue(s.contains(1)); + } + //int + { + Set s = (Set) StellarProcessorUtils.run("SET_ADD(SET_INIT([1,2,3]), 4)", new HashMap<>()); + Assert.assertEquals(4, s.size()); + Assert.assertTrue(s.contains(1)); + Assert.assertTrue(s.contains(2)); + Assert.assertTrue(s.contains(3)); + Assert.assertTrue(s.contains(4)); + } + //string + { + Set s = (Set) StellarProcessorUtils.run("SET_ADD(SET_INIT(['one','two','three']), 'four')", new HashMap<>()); + Assert.assertEquals(4, s.size()); + Assert.assertTrue(s.contains("one")); + Assert.assertTrue(s.contains("two")); + Assert.assertTrue(s.contains("three")); + Assert.assertTrue(s.contains("four")); + } + } + + @Test + public void setRemoveTest() throws Exception { + { + Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(SET_INIT([1]), 1)", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + { + Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(null, 1)", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + //int + { + Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(SET_INIT([1,2,3]), 2)", new HashMap<>()); + Assert.assertEquals(2, s.size()); + Assert.assertTrue(s.contains(1)); + Assert.assertTrue(s.contains(3)); + } + //string + { + Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(SET_INIT(['one','two','three']), 'three')", new HashMap<>()); + Assert.assertEquals(2, s.size()); + Assert.assertTrue(s.contains("one")); + Assert.assertTrue(s.contains("two")); + } + } + + @Test(expected=ParseException.class) + public void setMergeTest_wrongType() throws Exception { + Set s = (Set) StellarProcessorUtils.run("SET_MERGE({ 'foo' : 'bar'} )", new HashMap<>()); + } + + @Test + public void setMergeTest() throws Exception { + { + Set s = (Set) StellarProcessorUtils.run("SET_MERGE([SET_INIT(), SET_INIT(null), null])", new HashMap<>()); + Assert.assertEquals(0, s.size()); + } + //int + { + Set s = (Set) StellarProcessorUtils.run("SET_MERGE([SET_INIT([1,2]), SET_INIT([3]), null, SET_INIT()])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.contains(1)); + Assert.assertTrue(s.contains(2)); + Assert.assertTrue(s.contains(3)); + } + //string + { + Set s = (Set) StellarProcessorUtils.run("SET_MERGE([SET_INIT(['one','two']), SET_INIT(['three'])])", new HashMap<>()); + Assert.assertEquals(3, s.size()); + Assert.assertTrue(s.contains("one")); + Assert.assertTrue(s.contains("two")); + Assert.assertTrue(s.contains("three")); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java index 858a043..418bf2d 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java @@ -20,15 +20,18 @@ package org.apache.metron.stellar.dsl.functions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.collections4.map.HashedMap; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.ParseException; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; @@ -408,6 +411,41 @@ public class StringFunctionsTest { } + @Test + public void testSubstring() throws Exception { + Map<String, Object> variables = ImmutableMap.of("s", "apache metron"); + Assert.assertEquals("metron", run("SUBSTRING(s, 7)", variables)); + Assert.assertEquals("me", run("SUBSTRING(s, 7, 9)", variables)); + Assert.assertNull(run("SUBSTRING(null, 7, 9)", new HashMap<>())); + Assert.assertNull(run("SUBSTRING(null, null, 9)", new HashMap<>())); + Assert.assertNull(run("SUBSTRING(s, null, 9)", variables)); + Assert.assertNull(run("SUBSTRING(null, null, null)", new HashMap<>())); + Assert.assertEquals("metron", run("SUBSTRING(s, 7, null)", variables)); + } + + @Test(expected=ParseException.class) + public void testSubstring_invalidEmpty() throws Exception { + Assert.assertEquals("metron", run("SUBSTRING()", new HashMap<>())); + } + + @Test(expected=ParseException.class) + public void testSubstring_invalidWrongTypeStart() throws Exception { + Map<String, Object> variables = ImmutableMap.of("s", "apache metron"); + Assert.assertEquals("metron", (String) run("SUBSTRING(s, '7')", variables)); + } + + @Test(expected=ParseException.class) + public void testSubstring_invalidWrongTypeEnd() throws Exception { + Map<String, Object> variables = ImmutableMap.of("s", "apache metron"); + Assert.assertEquals("metron", (String) run("SUBSTRING(s, 7, '9')", variables)); + } + + @Test(expected=ParseException.class) + public void testSubstring_invalidWrongTypeInput() throws Exception { + Map<String, Object> variables = ImmutableMap.of("s", 7); + Assert.assertEquals("metron", (String) run("SUBSTRING(s, 7, '9')", variables)); + } + /** * COUNT_MATCHES StringFunction */ @@ -449,4 +487,262 @@ public class StringFunctionsTest { Assert.assertTrue(thrown); } + + /** + * TO_JSON_OBJECT StringFunction + */ + + // Input strings to be used + /** + { "foo" : 2 } + */ + @Multiline + private String string1; + + /** + { + "foo" : "abc", + "bar" : "def" + } + */ + @Multiline + private String string2; + + /** + [ "foo", 2 ] + */ + @Multiline + private String string3; + + /** + [ "foo", "bar", "car" ] + */ + @Multiline + private String string4; + + /** + [ + { + "foo1":"abc", + "bar1":"def" + }, + { + "foo2":"ghi", + "bar2":"jkl" + } + ] + */ + @Multiline + private String string5; + + @Test + public void testToJsonObject() throws Exception { + //JSON Object + Object ret1 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string1)); + Assert.assertNotNull(ret1); + Assert.assertTrue (ret1 instanceof HashMap); + + Object ret2 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string2)); + Assert.assertNotNull(ret2); + Assert.assertTrue (ret2 instanceof HashMap); + Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2))); + + //Simple Arrays + Object ret3 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string3)); + Assert.assertNotNull(ret3); + Assert.assertTrue (ret3 instanceof ArrayList); + List<Object> result3 = (List<Object>) ret3; + Assert.assertEquals(2, result3.get(1)); + + Object ret4 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string4)); + Assert.assertNotNull(ret4); + Assert.assertTrue (ret4 instanceof ArrayList); + List<Object> result4 = (List<Object>) ret4; + Assert.assertEquals("car", result4.get(2)); + + //JSON Array + Object ret5 = run( "TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string5)); + Assert.assertNotNull(ret5); + Assert.assertTrue (ret5 instanceof ArrayList); + List<List<Object>> result5 = (List<List<Object>>) ret5; + HashMap<String,String> results5Map1 = (HashMap) result5.get(0); + Assert.assertEquals("def", results5Map1.get("bar1")); + HashMap<String,String> results5Map2 = (HashMap) result5.get(1); + Assert.assertEquals("ghi", results5Map2.get("foo2")); + + // No input + boolean thrown = false; + try { + run("TO_JSON_OBJECT()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Unable to parse")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Invalid input + try { + run("TO_JSON_OBJECT('123, 456')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Malformed JSON String + try { + run("TO_JSON_OBJECT('{\"foo\" : 2')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + } + + @Test + public void testToJsonMap() throws Exception { + //JSON Object + Object ret1 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string1)); + Assert.assertNotNull(ret1); + Assert.assertTrue (ret1 instanceof HashMap); + + Object ret2 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string2)); + Assert.assertNotNull(ret2); + Assert.assertTrue (ret2 instanceof HashMap); + Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2))); + + //Simple Arrays + boolean thrown = false; + try { + run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string3)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue(thrown); + + thrown = false; + try { + run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string4)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue (thrown); + + //JSON Array + thrown = false; + try { + run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string5)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue(thrown); + + + // No input + try { + run("TO_JSON_MAP()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Unable to parse")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Invalid input + try { + run("TO_JSON_MAP('123, 456')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Malformed JSON String + try { + run("TO_JSON_MAP('{\"foo\" : 2')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + } + + @Test + public void testToJsonList() throws Exception { + //Simple Arrays + Object ret3 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string3)); + Assert.assertNotNull(ret3); + Assert.assertTrue (ret3 instanceof ArrayList); + List<Object> result3 = (List<Object>) ret3; + Assert.assertEquals(2, result3.get(1)); + + Object ret4 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string4)); + Assert.assertNotNull(ret4); + Assert.assertTrue (ret4 instanceof ArrayList); + List<Object> result4 = (List<Object>) ret4; + Assert.assertEquals("car", result4.get(2)); + + //JSON Array + Object ret5 = run( "TO_JSON_LIST(msg)", ImmutableMap.of("msg", string5)); + Assert.assertNotNull(ret5); + Assert.assertTrue (ret5 instanceof ArrayList); + List<List<Object>> result5 = (List<List<Object>>) ret5; + HashMap<String,String> results5Map1 = (HashMap) result5.get(0); + Assert.assertEquals("def", results5Map1.get("bar1")); + HashMap<String,String> results5Map2 = (HashMap) result5.get(1); + Assert.assertEquals("ghi", results5Map2.get("foo2")); + + //JSON Object - throws exception + boolean thrown = false; + try { + run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string1)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue(thrown); + + thrown = false; + try { + run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string2)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue (thrown); + + // No input + thrown = false; + try { + run("TO_JSON_LIST()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Unable to parse")); + } + Assert.assertTrue(thrown); + + // Invalid input + thrown = false; + try { + run("TO_JSON_LIST('123, 456')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + + // Malformed JSON String + thrown = false; + try { + run("TO_JSON_LIST('{\"foo\" : 2')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9a86314..dcadfe8 100644 --- a/pom.xml +++ b/pom.xml @@ -344,8 +344,12 @@ <!-- ACE editor assets are covered in the metron-config NOTICE file --> <exclude>**/src/assets/ace/**</exclude> <exclude>dist/assets/ace/**</exclude> - <!-- Generated svg containing Font Awesome fonts are covered in the metron-config README and NOTICE file --> + <!-- Generated svg and bundle.css containing Font Awesome fonts are covered in the + metron-interface/metron-config README and NOTICE files --> <exclude>dist/*.svg</exclude> + <exclude>dist/styles.a0b6b99c10d9a13dc67e.bundle.css</exclude> + <!-- 3rdpartylicenses.txt is an empty file carried along by imported libraries --> + <exclude>dist/3rdpartylicenses.txt</exclude> <exclude>e2e/*.js.map</exclude> <!-- Checkstyle is LGPL. We derive ours from their base, but don't ship it, so it's fine use. http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/use-cases/README.md ---------------------------------------------------------------------- diff --git a/use-cases/README.md b/use-cases/README.md new file mode 100644 index 0000000..02be32d --- /dev/null +++ b/use-cases/README.md @@ -0,0 +1,4 @@ +# Worked Examples + +The following are worked examples of use-cases that showcase some (or +many) component(s) of Metron. http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/use-cases/geographic_login_outliers/README.md ---------------------------------------------------------------------- diff --git a/use-cases/geographic_login_outliers/README.md b/use-cases/geographic_login_outliers/README.md new file mode 100644 index 0000000..99e9a5b --- /dev/null +++ b/use-cases/geographic_login_outliers/README.md @@ -0,0 +1,267 @@ +# Problem Statement + +One way to find anomalous behavior in a network is by inspecting user +login behavior. In particular, if a user is logging in via vastly +differing geographic locations in a short period of time, this may be +evidence of malicious behavior. + +More formally, we can encode this potentially malicious event in terms +of how far from the geographic centroid of the user's historic logins +as compared to all users. For instance, if we track all users and the +median distance from the central geographic location of all of their +logins for the last 2 hours is 3 km and the standard deviation is 1 km, +if we see a user logging in 1700 km from the central geographic location of +their logins for the last 2 hours, then they MAY be exhibiting a +deviation that we want to monitor since it would be hard to travel that +distance in 4 hours. On the other hand, the user may have +just used a VPN or proxy. Ultimately, this sort of analytic must be +considered only one piece of evidence in addition to many others before +we want to indicate an alert. + +# Demonstration Design +For the purposes of demonstration, we will construct synthetic data +whereby 2 users are logging into a system rather quickly (once per +second) from various hosts. Each user's locations share the same first +2 octets, but will choose the last 2 randomly. We will then inject a +data point indicating `user1` is logging in via a russian IP address. + +## Preliminaries +We assume that the following environment variables are set: +* `METRON_HOME` - the home directory for metron +* `ZOOKEEPER` - The zookeeper quorum (comma separated with port specified: e.g. `node1:2181` for full-dev) +* `BROKERLIST` - The Kafka broker list (comma separated with port specified: e.g. `node1:6667` for full-dev) +* `ES_HOST` - The elasticsearch master (and port) e.g. `node1:9200` for full-dev. + +Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol. + +Before editing configurations, be sure to pull the configs from zookeeper locally via +``` +$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f +``` + +## Configure the Profiler +First, we'll configure the profiler to emit a profiler every 1 minute: +* In Ambari, set the profiler period duration to `1` minute via the Profiler config section. +* Adjust `$METRON_HOME/config/zookeeper/global.json` to adjust the capture duration: +``` + "profiler.client.period.duration" : "1", + "profiler.client.period.duration.units" : "MINUTES" +``` + +## Create the Data Generator +We want to create a new sensor for our synthetic data called `auth`. To +feed it, we need a synthetic data generator. In particular, we want a +process which will feed authentication events per second for a set of +users where the IPs are randomly chosen, but each user's login ip +addresses share the same first 2 octets. + +Edit `~/gen_data.py` and paste the following into it: +``` +#!/usr/bin/python + +import random +import sys +import time + +domains = { 'user1' : '173.90', 'user2' : '156.33' } + +def get_ip(base): + return base + '.' + str(random.randint(1,255)) + '.' + str(random.randint(1, 255)) + +def main(): + freq_s = 1 + while True: + user='user' + str(random.randint(1,len(domains))) + epoch_time = int(time.time()) + ip=get_ip(domains[user]) + print user + ',' + ip + ',' + str(epoch_time) + sys.stdout.flush() + time.sleep(freq_s) + +if __name__ == '__main__': + main() +``` + +## Create the `auth` Parser + +The message format for our simple synthetic data is a CSV with: +* username +* login ip address +* timestamp + +We will need to parse this via our `CSVParser` and add the geohash of the login ip address. + +* To create this parser, edit `$METRON_HOME/config/zookeeper/parsers/auth.json` and paste the following: +``` +{ + "parserClassName" : "org.apache.metron.parsers.csv.CSVParser" + ,"sensorTopic" : "auth" + ,"parserConfig" : { + "columns" : { + "user" : 0, + "ip" : 1, + "timestamp" : 2 + } + } + ,"fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : [ "hash" ] + ,"config" : { + "hash" : "GEOHASH_FROM_LOC(GEO_GET(ip))" + } + } + ] +} +``` +* Create the kafka topic via: +``` +/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic auth --partitions 1 --replication-factor 1 +``` + +## Create the Profiles for Enrichment + +We will need to track 2 profiles to accomplish this task: +* `locations_by_user` - The geohashes of the locations the user has logged in from. This is a multiset of geohashes per user. Note that the multiset in this case is effectively a map of geohashes to occurrance counts. +* `geo_distribution_from_centroid` - The statistical distribution of the distance between a login location and the geographic centroid of the user's previous logins from the last 2 minutes. Note, in a real installation this would be a larger temporal lookback. + +We can represent these in the `$METRON_HOME/config/zookeeper/profiler.json` via the following: +``` +{ + "profiles": [ + { + "profile": "geo_distribution_from_centroid", + "foreach": "'global'", + "onlyif": "exists(geo_distance) && geo_distance != null", + "init" : { + "s": "STATS_INIT()" + }, + "update": { + "s": "STATS_ADD(s, geo_distance)" + }, + "result": "s" + }, + { + "profile": "locations_by_user", + "foreach": "user", + "onlyif": "exists(hash) && hash != null && LENGTH(hash) > 0", + "init" : { + "s": "MULTISET_INIT()" + }, + "update": { + "s": "MULTISET_ADD(s, hash)" + }, + "result": "s" + } + ] +} +``` + +## Enrich authentication Events + +We will need to enrich the authentication records in a couple of ways to use in the threat triage section as well as the profiles: +* `geo_distance`: representing the distance between the current geohash and the geographic centroid for the last 2 minutes. +* `geo_centroid`: representing the geographic centroid for the last 2 minutes + +Beyond that, we will need to determine if the authentication event is a geographic outlier by computing the following fields: +* `dist_median` : representing the median distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the median of the `geo_distance` values across all users). +* `dist_sd` : representing the standard deviation of the distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the standard deviation of the `geo_distance` values across all users). +* `geo_outlier` : whether `geo_distance` is more than 5 standard deviations from the median across all users. + +We also want to set up a triage rule associating a score and setting an alert if `geo_outlier` is true. In reality, this would be more complex as this metric is at best circumstantial and would need supporting evidence, but for simplicity we'll deal with the false positives. + +* Edit `$METRON_HOME/config/zookeeper/enrichments/auth.json` and paste the following: +``` +{ + "enrichment": { + "fieldMap": { + "stellar" : { + "config" : [ + "geo_locations := MULTISET_MERGE( PROFILE_GET( 'locations_by_user', user, PROFILE_FIXED( 2, 'MINUTES')))", + "geo_centroid := GEOHASH_CENTROID(geo_locations)", + "geo_distance := TO_INTEGER(GEOHASH_DIST(geo_centroid, hash))", + "geo_locations := null" + ] + } + } + ,"fieldToTypeMap": { } + }, + "threatIntel": { + "fieldMap": { + "stellar" : { + "config" : [ + "geo_distance_distr:= STATS_MERGE( PROFILE_GET( 'geo_distribution_from_centroid', 'global', PROFILE_FIXED( 2, 'MINUTES')))", + "dist_median := STATS_PERCENTILE(geo_distance_distr, 50.0)", + "dist_sd := STATS_SD(geo_distance_distr)", + "geo_outlier := ABS(dist_median - geo_distance) >= 5*dist_sd", + "is_alert := exists(is_alert) && is_alert", + "is_alert := is_alert || (geo_outlier != null && geo_outlier == true)", + "geo_distance_distr := null" + ] + } + + }, + "fieldToTypeMap": { }, + "triageConfig" : { + "riskLevelRules" : [ + { + "name" : "Geographic Outlier", + "comment" : "Determine if the user's geographic distance from the centroid of the historic logins is an outlier as compared to all users.", + "rule" : "geo_outlier != null && geo_outlier", + "score" : 10, + "reason" : "FORMAT('user %s has a distance (%d) from the centroid of their last login is 5 std deviations (%f) from the median (%f)', user, geo_distance, dist_sd, dist_median)" + } + ], + "aggregator" : "MAX" + } + } +} +``` + +## Execute Demonstration + +From here, we've set up our configuration and can push the configs: +* Push the configs to zookeeper via +``` +$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z node1:2181 -i $METRON_HOME/config/zookeeper/ +``` +* Start the parser via: +``` +$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s auth +``` +* Push synthetic data into the `auth` topic via +``` +python ~/gen_data.py | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic auth +``` +* Wait for about `5` minutes and kill the previous command +* Push a synthetic record indicating `user1` has logged in from a russian IP (`109.252.227.173`): +``` +echo -e "import time\nprint 'user1,109.252.227.173,'+str(int(time.time()))" | python | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic auth +``` +* Execute the following to search elasticsearch for our geographic login outliers: +``` +curl -XPOST "http://$ES_HOST/auth*/_search?pretty"; -d ' +{ + "_source" : [ "is_alert", "threat:triage:rules:0:reason", "user", "ip", "geo_distance" ], + "query": { "exists" : { "field" : "threat:triage:rules:0:reason" } } +} +' +``` + +You should see, among a few other false positive results, something like the following: +``` +{ + "_index" : "auth_index_2017.09.07.20", + "_type" : "auth_doc", + "_id" : "f5bdbf76-9d78-48cc-b21d-bc434c96e62e", + "_score" : 1.0, + "_source" : { + "geo_distance" : 7879, + "threat:triage:rules:0:reason" : "user user1 has a distance (7879) from the centroid of their last login is 5 std deviations (334.814719) from the median (128.000000)", + "ip" : "109.252.227.173", + "is_alert" : "true", + "user" : "user1" + } +} +``` +
