METRON-1377: Stellar function to generate typosquatted domains (similar to dnstwist) closes apache/incubator-metron#878
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0996b734 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0996b734 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0996b734 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 0996b7348eca14fea1b1b3c4dd57861b3a30bdeb Parents: 9108072 Author: cstella <ceste...@gmail.com> Authored: Mon Jan 8 09:30:58 2018 -0500 Committer: cstella <ceste...@gmail.com> Committed: Mon Jan 8 09:30:58 2018 -0500 ---------------------------------------------------------------------- .../common/typosquat/AdditionStrategy.java | 41 +++ .../common/typosquat/BitsquattingStrategy.java | 61 +++++ .../common/typosquat/HomoglyphStrategy.java | 115 ++++++++ .../common/typosquat/HyphenationStrategy.java | 41 +++ .../common/typosquat/InsertionStrategy.java | 56 ++++ .../metron/common/typosquat/Keyboards.java | 151 +++++++++++ .../common/typosquat/OmissionStrategy.java | 41 +++ .../common/typosquat/RepetitionStrategy.java | 48 ++++ .../common/typosquat/ReplacementStrategy.java | 49 ++++ .../common/typosquat/SubdomainStrategy.java | 48 ++++ .../common/typosquat/TranspositionStrategy.java | 46 ++++ .../typosquat/TyposquattingStrategies.java | 109 ++++++++ .../common/typosquat/TyposquattingStrategy.java | 25 ++ .../common/typosquat/VowelSwapStrategy.java | 57 ++++ .../typosquat/TyposquattingStrategiesTest.java | 126 +++++++++ .../src/test/resources/typosquat/amazon.csv | 259 +++++++++++++++++++ .../src/test/resources/typosquat/github.csv | 227 ++++++++++++++++ .../dsl/functions/FunctionalFunctions.java | 9 +- 18 files changed, 1504 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java new file mode 100644 index 0000000..871052d --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/AdditionStrategy.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * A typo strategy based on adding characters between ascii 97 and 123. + */ +public class AdditionStrategy implements TyposquattingStrategy { + @Override + public Set<String> generateCandidates(String domain) { + Set<String> ret = new HashSet<>(); + for(int i = 97;i < 123;++i) { + char c = Character.toChars(i)[0]; + ret.add(domain + c); + } + return ret; + } + + @Override + public String name() { + return "Addition"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java new file mode 100644 index 0000000..32d4184 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/BitsquattingStrategy.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * See http://dinaburg.org/bitsquatting.html for more + */ +public class BitsquattingStrategy implements TyposquattingStrategy { + public static int[] MASK = new int[] { 1, 2, 4, 8, 16, 32, 64, 128}; + @Override + public Set<String> generateCandidates(String originalString) { + Set<String> ret = new HashSet<>(); + char[] str = originalString.toCharArray(); + for(int i = 0;i < str.length;++i) { + char c = str[i]; + for(int j : MASK) { + int maskedNum = (int)c ^ j; + char maskedChar = (char)maskedNum; + if((maskedNum >= 48 && maskedNum <= 57) || (maskedNum >= 97 && maskedNum <= 122) || maskedNum == 45) { + ret.add(pasteTogether(str, i, maskedChar)); + } + } + } + return ret; + } + + @Override + public String name() { + return "Bitsquatting"; + } + + private static String pasteTogether(char[] str, int replacementPoint, char maskedChar) { + String ret = ""; + for(int i = 0;i < replacementPoint;++i) { + ret += str[i]; + } + ret += maskedChar; + for(int i = replacementPoint+1;i < str.length;++i) { + ret += str[i]; + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java new file mode 100644 index 0000000..19479de --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HomoglyphStrategy.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandles; +import java.net.IDN; +import java.util.*; + +/** + * Substituting characters for ascii or unicode analogues which are visually similar (e.g. latlmes.com for latimes.com) + * + */ +public class HomoglyphStrategy implements TyposquattingStrategy{ + + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final Map<Character, List<String>> glyphs = new HashMap<Character, List<String>>() + {{ + put('a', ImmutableList.of("à ", "á", "â", "ã", "ä", "Ã¥", "É", "а", "ạ", "Ç", "Ä", "ȧ", "Ó")); + put('b', ImmutableList.of("d", "lb", "ib", "Ê", "Ь", "bÌ", "É", "Ð")); + put('c', ImmutableList.of("ϲ", "Ñ", "Æ", "Ä", "Ä", "ç")); + put('d', ImmutableList.of("b", "cl", "dl", "di", "Ô", "Õª", "É", "Ä")); + put('e', ImmutableList.of("é", "ê", "ë", "Ä", "Ä", "Ä", "Ä", "е", "ẹ", "Ä", "Ñ", "ϵ", "Ò½")); + put('f', ImmutableList.of("Ï", "Æ", "Ò")); + put('g', ImmutableList.of("q", "É¢", "É¡", "Ô", "Ô", "Ä¡", "Ä", "Ö", "ǵ", "Ä£")); + put('h', ImmutableList.of("lh", "ih", "Ò»", "Õ°", "á", "н")); + put('i', ImmutableList.of("1", "l", "á¥", "Ã", "ï", "ı", "É©", "ι", "ê", "Ç", "Ä")); + put('j', ImmutableList.of("Ñ", "Ê", "ϳ", "É")); + put('k', ImmutableList.of("lk", "ik", "lc", "κ", "â²", "κ")); + put('l', ImmutableList.of("1", "i", "É«", "Å")); + put('m', ImmutableList.of("n", "nn", "rn", "rr", "á¹", "á´", "м", "ɱ")); + put('n', ImmutableList.of("m", "r", "Å")); + put('o', ImmutableList.of("0", "Î", "ο", "Ð", "о", "Õ", "ȯ", "á»", "á»", "Æ¡", "ó", "ö", "Ó§")); + put('p', ImmutableList.of("Ï", "Ñ", "Æ¿", "Ï·", "Ã")); + put('q', ImmutableList.of("g", "Õ¦", "Ô", "Õ£", "Ê ")); + put('r', ImmutableList.of("Ê", "Ð", "á´¦", "ɼ", "ɽ")); + put('s', ImmutableList.of("á½", "á", "Ê", "Å", "Ñ")); + put('t', ImmutableList.of("Ï", "Ñ", "Å£")); + put('u', ImmutableList.of("μ", "Ï ", "Õ", "Õ½", "Ñ", "á´", "Ç", "Å")); + put('v', ImmutableList.of("ѵ", "ν", "vÌ")); + put('w', ImmutableList.of("vv", "Ñ¡", "Õ¡", "Ô")); + put('x', ImmutableList.of("Ñ ", "Ò³", "áº")); + put('y', ImmutableList.of("Ê", "γ", "Ñ", "Ò®", "ý")); + put('z', ImmutableList.of("Ê", "ż", "ź", "Ê", "á´¢")); + }}; + + @Override + public Set<String> generateCandidates(String originalString) { + Set<String> result = new HashSet<>(); + String domain = originalString; + if(StringUtils.isEmpty(domain)) { + return result; + } + if(isAce(domain)) { + //this is an ace domain. + domain = IDN.toUnicode(domain); + } + for(int ws = 0;ws < domain.length();ws++) { + for(int i = 0;i < domain.length() - ws + 1;++i) { + String win = domain.substring(i, i+ws); + for(int j = 0;j < ws;j++) { + char c = win.charAt(j); + if( glyphs.containsKey(c)) { + for( String g : glyphs.get(c)) { + String winNew = win.replaceAll("" + c, g); + String d = domain.substring(0, i) + winNew + domain.substring(i + ws); + result.add(d); + if(!isAce(d)) { + try { + String dAscii = IDN.toASCII(d, IDN.ALLOW_UNASSIGNED); + if (!d.equals(dAscii)) { + result.add(dAscii); + } + } + catch(IllegalArgumentException iae) { + LOG.debug("Unable to parse " + d + ": " + iae.getMessage(), iae); + } + } + } + } + } + } + } + return result; + } + + public static boolean isAce(String domainRaw) { + String domain = domainRaw.toLowerCase(); + return domain.startsWith("xn--") || domain.contains(".xn--"); + } + + @Override + public String name() { + return "Homoglyph"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java new file mode 100644 index 0000000..ca6ce8f --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/HyphenationStrategy.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * Typos based on random hyphenation (e.g. am-azon.com vs amazon.com) + */ +public class HyphenationStrategy implements TyposquattingStrategy { + @Override + public Set<String> generateCandidates(String originalString) { + Set<String> ret = new HashSet<>(); + for(int i = 1;i < originalString.length();++i) { + ret.add(originalString.substring(0, i) + "-" + originalString.substring(i)); + } + return ret; + } + + @Override + public String name() { + return "Hyphenation"; + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java new file mode 100644 index 0000000..b601a35 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/InsertionStrategy.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * Typo strategy based on random insertion of common typos based on keyboard layout proximity. + */ +public class InsertionStrategy implements TyposquattingStrategy { + @Override + public Set<String> generateCandidates(String domain) { + Set<String> ret = new HashSet<>(); + for(int i = 1;i < domain.length() - 1;++i) { + for(Keyboards keyboard : Keyboards.values()) { + String mapping = keyboard.getMapping().get(domain.charAt(i)); + if(mapping != null) { + for(Character c : mapping.toCharArray()) { + ret.add(domain.substring(0, i) + + c + + domain.charAt(i) + + domain.substring(i+1, domain.length()) + ); + ret.add(domain.substring(0, i) + + domain.charAt(i) + + c + + domain.substring(i+1, domain.length()) + ); + } + } + } + } + return ret; + } + + @Override + public String name() { + return "Insertion"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java new file mode 100644 index 0000000..6fbd5ec --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/Keyboards.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashMap; +import java.util.Map; + +/** + * This provides a mapping of nearby keys for a variety of keyboard layouts. This is useful in determining likely + * typos. + */ +public enum Keyboards { + QWERTY(new HashMap<Character, String>() + {{ + put('j', "ikmnhu"); + put('w', "3esaq2"); + put('v', "cfgb"); + put('l', "kop"); + put('y', "7uhgt6"); + put('x', "zsdc"); + put('r', "5tfde4"); + put('u', "8ijhy7"); + put('a', "qwsz"); + put('q', "12wa"); + put('c', "xdfv"); + put('b', "vghn"); + put('e', "4rdsw3"); + put('d', "rfcxse"); + put('g', "yhbvft"); + put('p', "lo0"); + put('i', "9okju8"); + put('h', "ujnbgy"); + put('k', "olmji"); + put('f', "tgvcdr"); + put('m', "njk"); + put('s', "edxzaw"); + put('o', "0plki9"); + put('n', "bhjm"); + put('1', "2q"); + put('0', "po9"); + put('3', "4ew2"); + put('2', "3wq1"); + put('5', "6tr4"); + put('4', "5re3"); + put('7', "8uy6"); + put('6', "7yt5"); + put('9', "0oi8"); + put('8', "9iu7"); + put('z', "asx"); + put('t', "6ygfr5"); + }}), + QWERTZ(new HashMap<Character, String>() {{ + put('j', "ikmnhu"); + put('w', "3esaq2"); + put('v', "cfgb"); + put('l', "kop"); + put('y', "asx"); + put('x', "ysdc"); + put('r', "5tfde4"); + put('u', "8ijhz7"); + put('a', "qwsy"); + put('q', "12wa"); + put('c', "xdfv"); + put('b', "vghn"); + put('e', "4rdsw3"); + put('d', "rfcxse"); + put('g', "zhbvft"); + put('p', "lo0"); + put('i', "9okju8"); + put('h', "ujnbgz"); + put('k', "olmji"); + put('f', "tgvcdr"); + put('m', "njk"); + put('s', "edxyaw"); + put('o', "0plki9"); + put('n', "bhjm"); + put('1', "2q"); + put('0', "po9"); + put('3', "4ew2"); + put('2', "3wq1"); + put('5', "6tr4"); + put('4', "5re3"); + put('7', "8uz6"); + put('6', "7zt5"); + put('9', "0oi8"); + put('8', "9iu7"); + put('z', "7uhgt6"); + put('t', "6zgfr5"); + }}), + AZERTY(new HashMap<Character, String>() {{ + put('j', "iknhu"); + put('w', "sxq"); + put('v', "cfgb"); + put('l', "kopm"); + put('y', "7uhgt6"); + put('x', "zsdc"); + put('r', "5tfde4"); + put('u', "8ijhy7"); + put('a', "2zq1"); + put('q', "zswa"); + put('c', "xdfv"); + put('b', "vghn"); + put('e', "4rdsz3"); + put('d', "rfcxse"); + put('g', "yhbvft"); + put('p', "lo0m"); + put('i', "9okju8"); + put('h', "ujnbgy"); + put('k', "olji"); + put('f', "tgvcdr"); + put('m', "lp"); + put('s', "edxwqz"); + put('o', "0plki9"); + put('n', "bhj"); + put('1', "2a"); + put('0', "po9"); + put('3', "4ez2"); + put('2', "3za1"); + put('5', "6tr4"); + put('4', "5re3"); + put('7', "8uy6"); + put('6', "7yt5"); + put('9', "0oi8"); + put('8', "9iu7"); + put('z', "3esqa2"); + put('t', "6ygfr5"); + }}) + ; + private Map<Character, String> mapping; + Keyboards(Map<Character, String> mapping) { + this.mapping = mapping; + } + public Map<Character, String> getMapping() { + return mapping; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java new file mode 100644 index 0000000..c49dbb0 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/OmissionStrategy.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * A typo strategy based on omitting characters. + */ +public class OmissionStrategy implements TyposquattingStrategy { + + @Override + public Set<String> generateCandidates(String domain) { + HashSet<String> ret = new HashSet<>(); + for(int i = 0;i < domain.length();++i) { + ret.add(domain.substring(0, i) + domain.substring(i+1, domain.length())); + } + return ret; + } + + @Override + public String name() { + return "Omission"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java new file mode 100644 index 0000000..fa1b0aa --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/RepetitionStrategy.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * Typo strategy around repeating existing characters. + */ +public class RepetitionStrategy implements TyposquattingStrategy { + @Override + public Set<String> generateCandidates(String domain) { + Set<String> ret = new HashSet<>(); + for(int i = 0;i < domain.length();++i) { + Character c = domain.charAt(i); + if(Character.isLetter(c)) { + ret.add( domain.substring(0, i) + + c + + c + + domain.substring(i+1, domain.length()) + ); + } + } + return ret; + } + + @Override + public String name() { + return "Repetition"; + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java new file mode 100644 index 0000000..78254d9 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/ReplacementStrategy.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +public class ReplacementStrategy implements TyposquattingStrategy { + @Override + public Set<String> generateCandidates(String domain) { + Set<String> ret = new HashSet<>(); + for(int i = 0;i < domain.length();++i) { + for(Keyboards keyboard : Keyboards.values()) { + String mapping = keyboard.getMapping().get(domain.charAt(i)); + if(mapping != null) { + for(Character c : mapping.toCharArray()) { + ret.add( domain.substring(0, i) + + c + + domain.substring(i+1, domain.length()) + ); + } + } + } + } + return ret; + } + + @Override + public String name() { + return "Replacement"; + } + + +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java new file mode 100644 index 0000000..2dc06e4 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/SubdomainStrategy.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * A typo strategy around random subdomains (e.g. am.azon.com vs amazon.com) + */ +public class SubdomainStrategy implements TyposquattingStrategy{ + + @Override + public Set<String> generateCandidates(String domain) { + Set<String> ret = new HashSet<>(); + for(int i = 1;i < domain.length();++i) { + Character c = domain.charAt(i); + Character prevC = domain.charAt(i-1); + if(c != '-' && c != '.' + && prevC != '-' && prevC != '.' + ) + { + ret.add(domain.substring(0, i) + "." + domain.substring(i, domain.length())); + } + } + return ret; + } + + @Override + public String name() { + return "Subdomain"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java new file mode 100644 index 0000000..7962553 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TranspositionStrategy.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +public class TranspositionStrategy implements TyposquattingStrategy { + @Override + public Set<String> generateCandidates(String domain) { + + Set<String> ret = new HashSet<>(); + for(int i = 0; i < domain.length()-1;++i) { + char nextC = domain.charAt(i+1); + char c = domain.charAt(i); + if(nextC != c) { + ret.add( domain.substring(0, i) + + nextC + + c + + domain.substring(i+2) + ); + } + } + return ret; + } + + @Override + public String name() { + return "Transposition"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java new file mode 100644 index 0000000..a9f9a41 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategies.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * This is a set of strategies for generating likely typos from domains. + */ +public enum TyposquattingStrategies implements TyposquattingStrategy { + ADDITION(new AdditionStrategy()), + BITSQUATTING(new BitsquattingStrategy()), + HOMOGLYPH(new HomoglyphStrategy()), + HYPHENATION(new HyphenationStrategy()), + INSERTION(new InsertionStrategy()), + OMISSION(new OmissionStrategy()), + REPETITION(new RepetitionStrategy()), + REPLACEMENT(new ReplacementStrategy()), + SUBDOMAIN(new SubdomainStrategy()), + TRANSPOSITION(new TranspositionStrategy()), + VOWELSWAP(new VowelSwapStrategy()), + ; + TyposquattingStrategy strategy; + TyposquattingStrategies(TyposquattingStrategy strategy) { + this.strategy = strategy; + } + + @Override + public Set<String> generateCandidates(String originalString) { + Set<String> candidates = strategy.generateCandidates(originalString); + candidates.remove(originalString); + return candidates; + } + + public static Set<String> generateAllCandidates(String originalString) { + Set<String> ret = new HashSet<>(); + for(TyposquattingStrategy s : values() ) { + ret.addAll(s.generateCandidates(originalString)); + } + return ret; + } + + public static TyposquattingStrategies byName(String name) { + for(TyposquattingStrategies s : values()) { + if(s.strategy.name().equals(name)) { + return s; + } + } + return null; + } + + @Stellar(namespace="DOMAIN" + ,name="TYPOSQUAT" + ,description="Generate potential typosquatted domain from a passed domain. Strategies largely match https://github.com/elceef/dnstwist"; + ,params = { + "domain - Domain (without subdomains or TLD) to generate typosquatted domains for." + } + ,returns = "A set of potential typosquatted domains." + ) + public static class Generate implements StellarFunction{ + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + if(args.size() == 0) { + return null; + } + Object dnObj = args.get(0); + if(dnObj == null) { + return null; + } + if(!(dnObj instanceof String)) { + throw new IllegalStateException("Expected a domain without subdomains or a TLD, but got " + dnObj); + } + return TyposquattingStrategies.generateAllCandidates((String)dnObj); + } + + @Override + public void initialize(Context context) { + + } + + @Override + public boolean isInitialized() { + return true; + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java new file mode 100644 index 0000000..37e3f16 --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/TyposquattingStrategy.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.Set; + +public interface TyposquattingStrategy { + Set<String> generateCandidates(String domain); + String name(); +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java new file mode 100644 index 0000000..0a3388d --- /dev/null +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/typosquat/VowelSwapStrategy.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import java.util.HashSet; +import java.util.Set; + +/** + * A typo strategy around swapping vowels (e.g. omazon.com vs amazon.com) + */ +public class VowelSwapStrategy implements TyposquattingStrategy { + private static Set<Character> VOWELS = new HashSet<Character>() {{ + add('a'); + add('e'); + add('i'); + add('o'); + add('u'); + }}; + + @Override + public Set<String> generateCandidates(String domain) { + + HashSet<String> ret = new HashSet<>(); + for(int i = 0;i < domain.length();++i) { + char c = domain.charAt(i); + for(char vowel : VOWELS) { + if(VOWELS.contains(c)) { + ret.add( domain.substring(0, i) + + vowel + + domain.substring(i+1) + ); + } + } + } + return ret; + } + + @Override + public String name() { + return "Vowel-swap"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java new file mode 100644 index 0000000..d7f99f0 --- /dev/null +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/typosquat/TyposquattingStrategiesTest.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.common.typosquat; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import org.apache.metron.stellar.common.utils.StellarProcessorUtils; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.*; + +@RunWith(Parameterized.class) +public class TyposquattingStrategiesTest { + + /* + Skipping the miscellaneous typosquatted domains as they rely on the TLD. + */ + static Set<String> typesToSkip = new HashSet<String>() {{ + add("Various"); + add("Original*"); + }}; + + //These are the output from DNS Twist ( https://github.com/elceef/dnstwist ). We want to ensure we match their output at minimum + static Map<String, EnumMap<TyposquattingStrategies, Set<String>>> expected = new HashMap<String, EnumMap<TyposquattingStrategies, Set<String>>>() + {{ + put("amazon", new EnumMap<>(TyposquattingStrategies.class)); + put("github", new EnumMap<>(TyposquattingStrategies.class)); + }}; + + @BeforeClass + public static void setup() throws Exception { + for(Map.Entry<String, EnumMap<TyposquattingStrategies, Set<String>>> kv : expected.entrySet()) { + try(BufferedReader br = new BufferedReader(new FileReader( "src/test/resources/typosquat/" + kv.getKey() + ".csv") ) ) + { + for(String line = null;(line = br.readLine()) != null;) { + if(line.startsWith("#")) { + continue; + } + Iterable<String> tokens = Splitter.on(",").split(line); + String name = Iterables.get(tokens, 0); + String domain = Iterables.get(tokens, 1); + domain = domain.replaceAll(".com", ""); + EnumMap<TyposquattingStrategies, Set<String>> expectedValues = kv.getValue(); + if(typesToSkip.contains(name)) { + continue; + } + TyposquattingStrategies strategy = TyposquattingStrategies.byName(name); + Assert.assertNotNull("Couldn't find " + name, strategy); + Set<String> s = expectedValues.get(strategy); + if(s == null) { + s = new HashSet<>(); + expectedValues.put(strategy, s); + } + s.add(domain); + } + } + } + } + + @Parameterized.Parameters + public static Collection<Object[]> strategies() { + List<Object[]> ret = new ArrayList<>(); + for(TyposquattingStrategies strategy : TyposquattingStrategies.values()) { + ret.add(new Object[] { strategy }); + } + return ret; + } + + TyposquattingStrategies strategy; + public TyposquattingStrategiesTest(TyposquattingStrategies strategy) { + this.strategy = strategy; + } + + public void assertExpected(String domain, TyposquattingStrategies strategy) { + Set<String> expectedValues = expected.get(domain).get(strategy); + Set<String> actualValues = strategy.generateCandidates(domain); + Assert.assertFalse(actualValues.contains(domain)); + { + Sets.SetView<String> vals = Sets.difference(expectedValues, actualValues); + String diff = Joiner.on(",").join(vals); + Assert.assertTrue(strategy.name() + ": Found values expected but not generated: " + diff, vals.isEmpty()); + } + } + + @Test + public void test() { + for(String domain : expected.keySet()) { + assertExpected(domain, strategy); + } + } + + @Test + public void testStellar() { + for(String domain : expected.keySet()) { + Set<String> expectedAll = TyposquattingStrategies.generateAllCandidates(domain); + Set<String> generatedAll = (Set<String>) StellarProcessorUtils.run("DOMAIN_TYPOSQUAT(domain)", ImmutableMap.of("domain", domain)); + Assert.assertTrue(Sets.symmetricDifference(expectedAll, generatedAll).isEmpty()); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv b/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv new file mode 100644 index 0000000..ceb4e93 --- /dev/null +++ b/metron-platform/metron-common/src/test/resources/typosquat/amazon.csv @@ -0,0 +1,259 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#fuzzer,domain-name,dns-a,dns-aaaa,dns-mx,dns-ns,geoip-country,whois-created,whois-updated,ssdeep-score +Original*,amazon.com,176.32.103.205,,amazon-smtp.amazon.com,ns1.p31.dynect.net,,,, +Addition,amazona.com,198.50.233.235,,,ns1.domainmx.com,,,, +Addition,amazonb.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonc.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazond.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Addition,amazone.com,87.238.81.156,,,,,,, +Addition,amazonf.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazong.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonh.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazoni.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonj.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonk.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonl.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonm.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonn.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazono.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonp.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonq.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonr.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazons.com,54.197.246.244,,,ns-1019.awsdns-63.net,,,, +Addition,amazont.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonu.com,,,,,,,, +Addition,amazonv.com,207.171.166.22,,,,,,, +Addition,amazonw.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazonx.com,207.171.166.22,,,ns-1.amazon.com,,,, +Addition,amazony.com,207.171.166.22,,,,,,, +Addition,amazonz.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Bitsquatting,cmazon.com,103.224.182.251,,mx92.m1bp.com,ns1.above.com,,,, +Bitsquatting,emazon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Bitsquatting,imazon.com,185.53.178.6,,mail.h-email.net,ns1.parkingcrew.net,,,, +Bitsquatting,qmazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,alazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,aoazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,aiazon.com,50.63.202.55,,mailstore1.secureserver.net,ns77.domaincontrol.com,,,, +Bitsquatting,aeazon.com,,,,,,,, +Bitsquatting,a-azon.com,50.63.202.4,,,ns07.domaincontrol.com,,,, +Bitsquatting,amczon.com,207.171.166.22,,,,,,, +Bitsquatting,amezon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amizon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amqzon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amaxon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amaron.com,72.143.24.146,,mx05-1.mycloudmailbox.com,ns61.worldnic.com,,,, +Bitsquatting,amajon.com,185.53.177.30,,,ns1.parkingspa.com,,,, +Bitsquatting,amaznn.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amazmn.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amazkn.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amazgn.com,185.53.178.7,,mail.h-email.net,ns1.parkingcrew.net,,,, +Bitsquatting,amazoo.com,209.236.72.34,,amazoo.com,ns1.westservers.net,,,, +Bitsquatting,amazol.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amazoj.com,207.171.166.22,,,ns-1.amazon.com,,,, +Bitsquatting,amazof.com,,,,ns-1.amazon.com,,,, +Homoglyph,xn--mazon-qwa.com,,,,,,,, +Homoglyph,xn--amzon-ucc.com,104.27.182.15,2400:cb00:2048:1::681b:b60f,,cass.ns.cloudflare.com,,,, +Homoglyph,xn--mazon-zjc.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--mzon-pzbb.com,,,,,,,, +Homoglyph,xn--mzon-9nab.com,,,,,,,, +Homoglyph,amaz0n.com,207.171.166.22,,,ns-1.amazon.com,,,, +Homoglyph,xn--mzon-4nab.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--amzon-1jc.com,54.163.185.105,,,ns01.domaincontrol.com,,,, +Homoglyph,xn--amzon-k11b.com,104.18.48.131,2400:cb00:2048:1::6812:3083,,kara.ns.cloudflare.com,,,, +Homoglyph,annazon.com,207.171.166.22,,,,,,, +Homoglyph,xn--mazon-i11b.com,,,,,,,, +Homoglyph,xn--mzon-koab.com,,,,,,,, +Homoglyph,xn--amzon-nra.com,,,,,,,, +Homoglyph,arrazon.com,213.186.33.2,,mx1.ovh.net,dns.ovh.net,,,, +Homoglyph,xn--amaon-x59a.com,,,,ns1gmz.name.com,,,, +Homoglyph,xn--amazn-i91b.com,184.168.221.36,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,, +Homoglyph,xn--amzon-sqa.com,198.105.244.111,2620:118:7002::1111,,a1-144.akam.net,,,, +Homoglyph,xn--amazn-uce.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--mzon-znab.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--aazon-6xe.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--amaon-vuc.com,107.180.1.108,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,, +Homoglyph,xn--mzon-poab.com,,,,,,,, +Homoglyph,xn--amzon-swa.com,,,,,,,, +Homoglyph,anazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Homoglyph,xn--mzon-fseb.com,198.105.244.228,,,,,,, +Homoglyph,xn--amzon-yqa.com,69.64.147.242,,,ns1bdg.name.com,,,, +Homoglyph,xn--aazon-919a.com,198.105.244.228,,,,,,, +Homoglyph,xn--amzon-4qa.com,198.105.244.228,,,,,,, +Homoglyph,xn--mzon-4q5ab.com,,,,,,,, +Homoglyph,xn--amzon-hra.com,,,,,,,, +Homoglyph,xn--mzon-foab.com,198.105.244.228,,,,,,, +Homoglyph,xn--mzon-p5bb.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--amazn-xob.com,50.63.202.43,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,, +Homoglyph,xn--amazn-581b.com,184.168.221.57,,mailstore1.secureserver.net,pdns09.domaincontrol.com,,,, +Homoglyph,xn--mazon-3ve.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--mazon-jwb.com,,,,,,,, +Homoglyph,xn--amaon-kib.com,,,,ns-1417.awsdns-49.org,,,, +Homoglyph,xn--amazn-okg.com,,,,,,,, +Homoglyph,xn--amazn-mye.com,,,,,,,, +Homoglyph,xn--amazn-lsf.com,,,,,,,, +Homoglyph,xn--mzon-zsab.com,198.105.244.228,,,,,,, +Homoglyph,xn--amzon-bra.com,50.63.202.58,,,ns33.domaincontrol.com,,,, +Homoglyph,xn--mzon-zmbb.com,198.105.244.228,,,,,,, +Homoglyph,amazor.com,68.178.213.61,,,ns1.namefind.com,,,, +Homoglyph,xn--mazon-1of.com,,,,,,,, +Homoglyph,amazom.com,174.129.240.167,,,ns-1.amazon.com,,,, +Homoglyph,xn--mazon-wqa.com,66.96.149.32,,mx.xn--mazon-wqa.com,ns1.dotster.com,,,, +Homoglyph,xn--amazn-3ta.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--amazn-uce.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--mazon-fra.com,,,,ns1.markmonitor.com,,,, +Homoglyph,arnazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Homoglyph,xn--amazn-mua.com,184.168.221.32,,mailstore1.secureserver.net,ns19.domaincontrol.com,,,, +Homoglyph,xn--aazon-ipc.com,,,,,,,, +Homoglyph,xn--mazon-8qa.com,184.168.221.32,,mailstore1.secureserver.net,ns05.domaincontrol.com,,,, +Homoglyph,xn--amaon-7hb.com,207.244.67.138,,,ns1.wombatdns.com,,,, +Homoglyph,xn--aazon-fl1b.com,91.215.153.210,,mailstore1.secureserver.net,ns23.domaincontrol.com,,,, +Homoglyph,xn--amazn-mye.com,,,,,,,, +Homoglyph,xn--mazon-2qa.com,103.224.212.222,,mx92.m1bp.com,ns1.above.com,,,, +Homoglyph,xn--mazon-scc.com,,,,,,,, +Homoglyph,xn--mazon-qqa.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,, +Homoglyph,xn--mazon-lra.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--amzon-lwb.com,,,,,,,, +Homoglyph,xn--amzon-5ve.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--amazn-9dc.com,184.168.221.46,,mailstore1.secureserver.net,ns37.domaincontrol.com,,,, +Homoglyph,xn--amazo-07a.com,,,,,,,, +Homoglyph,xn--mzon-43db.com,,,,ns1.markmonitor.com,,,, +Homoglyph,xn--amzon-3of.com,,,,,,,, +Hyphenation,a-mazon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Hyphenation,am-azon.com,,,,,,,, +Hyphenation,ama-zon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Hyphenation,amaz-on.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Hyphenation,amazo-n.com,,,,ns-1.amazon.com,,,, +Insertion,amsazon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Insertion,amyazon.com,207.171.166.22,,,,,,, +Insertion,amazqon.com,103.224.182.214,,mx92.m1bp.com,ns1.above.com,,,, +Insertion,amjazon.com,,,,ns-1.amazon.com,,,, +Insertion,anmazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,ama3zon.com,198.105.244.228,,,,,,, +Insertion,ajmazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amayzon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,apmazon.com,103.224.182.207,,mx92.m1bp.com,ns1.above.com,,,, +Insertion,amazxon.com,,,,,,,, +Insertion,am1azon.com,,,,,,,, +Insertion,ama2zon.com,,,,,,,, +Insertion,amwazon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Insertion,amazokn.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,ama6zon.com,,,,,,,, +Insertion,amazuon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Insertion,amnazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amazpon.com,,,,,,,, +Insertion,amaz6on.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Insertion,amazton.com,,,,ns-1.amazon.com,,,, +Insertion,amaz9on.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,am2azon.com,,,,,,,, +Insertion,amazson.com,54.235.212.68,,mail.amazson.com,ns1.pql.net,,,, +Insertion,ampazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,ama1zon.com,,,,,,,, +Insertion,amlazon.com,,,,,,,, +Insertion,amawzon.com,184.168.221.15,,mailstore1.secureserver.net,ns11.domaincontrol.com,,,, +Insertion,amaz7on.com,,,,,,,, +Insertion,amazo9n.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,almazon.com,208.91.197.128,,p.webcom.ctmail.com,ns45.worldnic.com,,,, +Insertion,amazlon.com,,,,,,,, +Insertion,amaszon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amatzon.com,,,,ns-1.amazon.com,,,, +Insertion,amazopn.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amahzon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amazgon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Insertion,amaz0on.com,207.171.166.22,,,,,,, +Insertion,amazoin.com,207.171.166.22,,,,,,, +Insertion,amazaon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amaezon.com,207.171.166.22,,,,,,, +Insertion,amkazon.com,207.171.166.22,,,,,,, +Insertion,amqazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amaxzon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amauzon.com,,,,,,,, +Insertion,amazzon.com,,,,ns-1.amazon.com,,,, +Insertion,akmazon.com,,,,ns-1.amazon.com,,,, +Insertion,amazhon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amazion.com,,,,ns-1.amazon.com,,,, +Insertion,amaazon.com,,,,ns-1.amazon.com,,,, +Insertion,amazoln.com,207.171.166.22,,,,,,, +Insertion,amagzon.com,185.53.178.7,,mail.h-email.net,ns1.parkingcrew.net,,,, +Insertion,amazeon.com,208.73.210.202,,,ns1.dsredirection.com,,,, +Insertion,amaz3on.com,198.105.244.228,,,,,,, +Insertion,amzazon.com,,,,,,,, +Insertion,amazo0n.com,207.171.166.22,,,ns-1.amazon.com,,,, +Insertion,amaqzon.com,207.171.166.22,,,,,,, +Insertion,amaz2on.com,,,,,,,, +Insertion,ama7zon.com,,,,,,,, +Insertion,amazkon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Omission,amaon.com,207.171.166.22,,,,,,, +Omission,mazon.com,184.168.221.23,,mazon-com.mail.protection.outlook.com,ns05.domaincontrol.com,,,, +Omission,amzon.com,207.171.166.22,,,,,,, +Omission,aazon.com,144.76.0.242,,,ns1.ndsplitter.com,,,, +Omission,amazo.com,144.76.0.242,,,ns1.ndsplitter.com,,,, +Omission,amazn.com,207.171.166.22,,,pdns1.ultradns.net,,,, +Repetition,aamazon.com,,,,,,,, +Repetition,amazoon.com,,,,,,,, +Repetition,ammazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Replacement,amazln.com,207.171.166.22,,,,,,, +Replacement,amwzon.com,,,,pdns1.ultradns.net,,,, +Replacement,ajazon.com,,,,,,,, +Replacement,am2zon.com,198.105.244.228,,,,,,, +Replacement,amaton.com,208.91.197.26,,,,,,, +Replacement,amzzon.com,,,,,,,, +Replacement,ama3on.com,184.168.221.43,,mailstore1.secureserver.net,ns03.domaincontrol.com,,,, +Replacement,amaaon.com,72.52.4.122,,localhost,ns1.sedoparking.com,,,, +Replacement,ama2on.com,207.171.166.22,,,ns-1.amazon.com,,,, +Replacement,ama7on.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Replacement,amahon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Replacement,ymazon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Replacement,amagon.com,208.91.197.39,,,sk.s5.ans1.ns121.ztomy.com,,,, +Replacement,wmazon.com,,,,ns-1.amazon.com,,,, +Replacement,amazin.com,,,,,,,, +Replacement,amaeon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Replacement,apazon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Replacement,amazpn.com,,,,ns-1.amazon.com,,,, +Replacement,am1zon.com,216.239.32.21,2001:4860:4802:32::15,,ns-cloud-b1.googledomains.com,,,, +Replacement,amaz9n.com,,,,ns-1.amazon.com,,,, +Replacement,amazoh.com,,,,,,,, +Replacement,ama6on.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Replacement,amyzon.com,72.52.10.14,,,ns1.markmonitor.com,,,, +Replacement,amszon.com,,,,pdns1.ultradns.net,,,, +Replacement,amazob.com,207.171.166.22,,,,,,, +Replacement,1mazon.com,172.217.8.211,2607:f8b0:4009:811::2013,,,,,, +Replacement,amason.com,212.47.223.137,,smx1.web-hosting.com,dns1.namecheaphosting.com,,,, +Replacement,amauon.com,207.171.166.22,,,,,,, +Replacement,smazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Replacement,zmazon.com,,,,,,,, +Replacement,amaqon.com,,,,,,,, +Replacement,2mazon.com,198.105.244.228,,,,,,, +Replacement,akazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Subdomain,a.mazon.com,,,,,,,, +Subdomain,am.azon.com,199.16.189.254,,,,,,, +Subdomain,ama.zon.com,184.168.221.104,,,ns1.afternic.com,,,, +Subdomain,amaz.on.com,208.78.94.160,,,,,,, +Subdomain,amazo.n.com,,,,,,,, +Transposition,maazon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Transposition,aamzon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Transposition,amzaon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Transposition,amaozn.com,207.171.166.22,,,,,,, +Transposition,amazno.com,,,,ns-1.amazon.com,,,, +Vowel-swap,omazon.com,208.91.196.152,,,sk.s7.ans1.ns147.klczy.com,,,, +Vowel-swap,amuzon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Vowel-swap,amazan.com,54.255.202.167,,,ns1.meganameservers.eu,,,, +Vowel-swap,amazen.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,, +Vowel-swap,amozon.com,207.171.166.22,,,ns-1.amazon.com,,,, +Vowel-swap,amazun.com,216.21.239.197,,mail2.siteamerica.com,dns143.b.register.com,,,, +Vowel-swap,umazon.com,52.71.185.125,,,ns1.namebrightdns.com,,,, +Various,amazoncom.com,208.73.210.202,,,ns1.dsredirection.com,,,, http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-platform/metron-common/src/test/resources/typosquat/github.csv ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/resources/typosquat/github.csv b/metron-platform/metron-common/src/test/resources/typosquat/github.csv new file mode 100644 index 0000000..1965bf2 --- /dev/null +++ b/metron-platform/metron-common/src/test/resources/typosquat/github.csv @@ -0,0 +1,227 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#fuzzer,domain-name,dns-a,dns-aaaa,dns-mx,dns-ns,geoip-country,whois-created,whois-updated,ssdeep-score +Original*,github.com,192.30.253.112,,ALT1.ASPMX.L.GOOGLE.com,ns-1283.awsdns-32.org,,,, +Addition,githuba.com,184.168.221.50,,mailstore1.secureserver.net,ns37.domaincontrol.com,,,, +Addition,githubb.com,103.16.130.124,,mail.ext.io,ns1.afraid.org,,,, +Addition,githubc.com,103.224.212.222,,mx92.m1bp.com,ns1.above.com,,,, +Addition,githubd.com,198.105.244.228,,,,,,, +Addition,githube.com,81.171.22.6,,,ns1.weaponizedcow.com,,,, +Addition,githubf.com,198.105.244.228,,,,,,, +Addition,githubg.com,198.105.244.228,,,,,,, +Addition,githubh.com,198.105.244.228,,,,,,, +Addition,githubi.com,184.168.221.50,,mailstore1.secureserver.net,ns37.domaincontrol.com,,,, +Addition,githubj.com,198.105.244.228,,,,,,, +Addition,githubk.com,198.105.244.228,,,,,,, +Addition,githubl.com,198.105.244.228,,,,,,, +Addition,githubm.com,198.105.244.228,,,,,,, +Addition,githubn.com,198.105.244.228,,,,,,, +Addition,githubo.com,119.28.71.213,,,f1g1ns1.dnspod.net,,,, +Addition,githubp.com,198.105.244.228,,,,,,, +Addition,githubq.com,54.172.109.248,,eforward1.registrar-servers.com,dns1.registrar-servers.com,,,, +Addition,githubr.com,74.208.236.31,2607:f1c0:100f:f000::252,mx00.1and1.com,ns1087.ui-dns.biz,,,, +Addition,githubs.com,162.255.119.247,,eforward1.registrar-servers.com,dns1.registrar-servers.com,,,, +Addition,githubt.com,67.227.226.240,,,ns1.parklogic.com,,,, +Addition,githubu.com,106.187.91.218,,,ns1cnb.name.com,,,, +Addition,githubv.com,,,,a.dnspod.com,,,, +Addition,githubw.com,198.105.244.228,,,,,,, +Addition,githubx.com,,,,,,,, +Addition,githuby.com,198.105.244.228,,,,,,, +Addition,githubz.com,198.105.244.228,,,,,,, +Bitsquatting,fithub.com,4.35.164.141,,alt1.aspmx.l.google.com,ns1.monikerdns.net,,,, +Bitsquatting,eithub.com,204.11.56.48,,,ns1626.ztomy.com,,,, +Bitsquatting,cithub.com,,,,,,,, +Bitsquatting,oithub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,withub.com,23.234.4.120,,,ns1.eedns.com,,,, +Bitsquatting,ghthub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,gkthub.com,104.18.62.97,2400:cb00:2048:1::6812:3e61,mx.yandex.ru,sue.ns.cloudflare.com,,,, +Bitsquatting,gmthub.com,49.236.200.154,,bkpmail.localdns.com,ns14.localdns.com,,,, +Bitsquatting,gathub.com,216.250.120.46,2607:f1c0:1000:70f6:8ab4:44d2:262a:e822,mx00.1and1.com,ns1028.ui-dns.biz,,,, +Bitsquatting,gythub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,giuhub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,givhub.com,185.53.178.7,,mail.h-email.net,ns1.parkingcrew.net,,,, +Bitsquatting,giphub.com,47.90.201.75,,smtp.giphub.com,dns7.hichina.com,,,, +Bitsquatting,gidhub.com,67.55.92.184,,mx7.gidhub.com,ns1.liverealdeals.com,,,, +Bitsquatting,gi4hub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,gitiub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,gitjub.com,91.134.139.14,,,ns65.domaincontrol.com,,,, +Bitsquatting,gitlub.com,166.78.103.6,,,,,,, +Bitsquatting,gitxub.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,githtb.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,githwb.com,108.59.81.6,,,,,,, +Bitsquatting,githqb.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,githeb.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,gith5b.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,githuc.com,172.246.29.53,,,juming.dnsdun.com,,,, +Bitsquatting,githuf.com,185.53.177.31,,,ns1.protectdelegation.ca,,,, +Bitsquatting,githuj.com,104.31.90.172,2400:cb00:2048:1::681f:5aac,,coco.ns.cloudflare.com,,,, +Bitsquatting,githur.com,104.27.140.182,2400:cb00:2048:1::681b:8cb6,dc-0831986e3235.githur.com,coco.ns.cloudflare.com,,,, +Homoglyph,xn--githb-oxb.com,198.105.244.228,,,,,,, +Homoglyph,xn--ithub-v1a.com,,,,,,,, +Homoglyph,xn--gitub-fye.com,198.105.244.228,,,,,,, +Homoglyph,xn--gitub-1gg.com,198.105.244.228,,,,,,, +Homoglyph,xn--ithub-72b.com,198.105.244.228,,,,,,, +Homoglyph,xn--gthub-zsa.com,,,,ns-cloud-c1.googledomains.com,,,, +Homoglyph,xn--gitub-0kf.com,198.105.244.228,,,,,,, +Homoglyph,xn--githb-bjg.com,198.105.244.228,,,,,,, +Homoglyph,xn--githu-fwe.com,198.105.244.228,,,,,,, +Homoglyph,g1thub.com,198.105.244.228,,,,,,, +Homoglyph,gitlhub.com,198.105.244.228,,,,,,, +Homoglyph,xn--githu-hwc.com,198.105.244.228,,,,,,, +Homoglyph,githuib.com,141.22.27.142,,,ns1.domaindiscount24.net,,,, +Homoglyph,xn--gitub-ify.com,198.105.244.228,,,,,,, +Homoglyph,xn--gihub-8ye.com,198.105.244.228,,,,,,, +Homoglyph,gitihub.com,173.239.5.6,,mx7.gitihub.com,ns1.expiereddnsmanager.com,,,, +Homoglyph,xn--githu-10e.com,198.105.244.228,,,,,,, +Homoglyph,xn--gthub-h9x.com,198.105.244.228,,,,,,, +Homoglyph,qithub.com,184.168.221.34,,,ns13.domaincontrol.com,,,, +Homoglyph,xn--githb-vde.com,198.105.244.228,,,,,,, +Homoglyph,xn--ithub-zyf.com,198.105.244.228,,,,,,, +Homoglyph,glthub.com,52.71.185.125,,,ns1.namebrightdns.com,,,, +Homoglyph,xn--gthub-n4a.com,,,ec2-52-42-4-3.us-west-2.compute.amazonaws.com,dns1.registrar-servers.com,,,, +Homoglyph,xn--gthub-wwb.com,198.105.244.228,,,,,,, +Homoglyph,xn--ithub-qmc.com,50.63.202.41,,mailstore1.secureserver.net,ns25.domaincontrol.com,,,, +Homoglyph,xn--gthub-y3a.com,198.105.244.228,,,,,,, +Homoglyph,xn--gthub-cta.com,162.255.119.248,,eforward1.registrar-servers.com,dns1.registrar-servers.com,,,, +Homoglyph,xn--gihub-8db.com,198.105.244.228,,,,,,, +Homoglyph,xn--gihub-nde.com,198.105.244.228,,,,,,, +Homoglyph,xn--githb-bjg.com,198.105.244.228,,,,,,, +Homoglyph,xn--ithub-71a.com,198.105.244.228,,,,,,, +Homoglyph,xn--ithub-vjg.com,198.105.244.228,,,,,,, +Homoglyph,xn--gthub-4nc.com,198.105.244.228,,,,,,, +Homoglyph,githulb.com,198.105.244.228,,,,,,, +Homoglyph,githud.com,198.105.244.228,,,,,,, +Homoglyph,xn--githb-x49a.com,69.64.147.242,,,ns1cny.name.com,,,, +Homoglyph,xn--ithub-j1a.com,198.105.244.228,,,,,,, +Homoglyph,xn--githu-hkc.com,198.105.244.228,,,,,,, +Homoglyph,xn--ithub-wmc.com,69.64.147.10,,,dns1.name-services.com,,,, +Homoglyph,xn--gthub-qbe.com,198.105.244.228,,,,,,, +Homoglyph,xn--gthub-k41s.com,198.105.244.228,,,,,,, +Homoglyph,xn--githb-cce.com,198.105.244.228,,,,,,, +Homoglyph,xn--githb-0fb.com,198.105.244.228,,,,,,, +Homoglyph,xn--github-g1d.com,198.105.244.228,,,,,,, +Homoglyph,xn--githb-zze.com,,,,,,,, +Hyphenation,g-ithub.com,198.105.244.228,,,,,,, +Hyphenation,gi-thub.com,198.105.244.228,,,,,,, +Hyphenation,git-hub.com,192.30.253.167,,,ns1.p16.dynect.net,,,, +Hyphenation,gith-ub.com,198.105.244.228,,,,,,, +Hyphenation,githu-b.com,198.105.244.228,,,,,,, +Insertion,girthub.com,209.99.40.222,,,dns10.parkpage.foundationapi.com,,,, +Insertion,githiub.com,67.227.226.240,,mx156.hostedmxserver.com,ns1.parklogic.com,,,, +Insertion,g8ithub.com,198.105.244.228,,,,,,, +Insertion,githu8b.com,198.105.244.228,,,,,,, +Insertion,giythub.com,198.105.244.228,,,,,,, +Insertion,gitbhub.com,67.55.92.182,,mx7.gitbhub.com,ns3.renewyourexpiereddomain.com,,,, +Insertion,gith8ub.com,198.105.244.228,,,,,,, +Insertion,githuyb.com,198.105.244.228,,,,,,, +Insertion,giothub.com,198.105.244.228,,,,,,, +Insertion,githyub.com,198.105.244.228,,,,,,, +Insertion,gi6thub.com,198.105.244.228,,,,,,, +Insertion,gitzhub.com,207.244.67.214,,,ns1.dnsnuts.com,,,, +Insertion,gityhub.com,,,,,,,, +Insertion,gi5thub.com,198.105.244.228,,,,,,, +Insertion,gituhub.com,69.162.80.57,,,ns1.hastydns.com,,,, +Insertion,goithub.com,,,,,,,, +Insertion,gitnhub.com,198.105.244.228,,,,,,, +Insertion,githgub.com,69.64.147.47,,,dns1.name-services.com,,,, +Insertion,githu7b.com,198.105.244.228,,,,,,, +Insertion,githnub.com,198.105.244.228,,,,,,, +Insertion,git6hub.com,198.105.244.228,,,,,,, +Insertion,githhub.com,173.239.22.42,,,ns1.mnsdnsmart.com,,,, +Insertion,guithub.com,78.41.204.28,,,ns1.torresdns.com,,,, +Insertion,githjub.com,198.105.244.228,,,,,,, +Insertion,gkithub.com,198.105.244.228,,,,,,, +Insertion,gigthub.com,198.105.244.228,,,,,,, +Insertion,githuzb.com,198.105.244.228,,,,,,, +Insertion,gi9thub.com,198.105.244.228,,,,,,, +Insertion,githuub.com,104.27.184.65,2400:cb00:2048:1::681b:b841,,april.ns.cloudflare.com,,,, +Insertion,githzub.com,198.105.244.228,,,,,,, +Insertion,gitjhub.com,198.105.244.228,,,,,,, +Insertion,gikthub.com,198.105.244.228,,,,,,, +Insertion,githujb.com,198.105.244.228,,,,,,, +Insertion,gitghub.com,185.151.28.157,2a07:7800::157,mx.stackmail.com,ns1.eco-dns.co.uk,,,, +Insertion,githbub.com,103.224.212.222,,mx92.m1bp.com,ns1.above.com,,,, +Insertion,gitrhub.com,141.8.224.93,,,ns7.rookdns.com,,,, +Insertion,gjithub.com,198.105.244.228,,,,,,, +Insertion,gizthub.com,198.105.244.228,,,,,,, +Insertion,githuhb.com,198.105.244.228,,,,,,, +Insertion,gi8thub.com,198.105.244.228,,,,,,, +Insertion,giuthub.com,95.211.219.67,,,ns1.hastydns.com,,,, +Insertion,gitfhub.com,198.105.244.228,,,,,,, +Insertion,git5hub.com,198.105.244.228,,,,,,, +Insertion,g9ithub.com,198.105.244.228,,,,,,, +Insertion,gifthub.com,176.74.176.187,,,ns1.uniregistrymarket.link,,,, +Insertion,gith7ub.com,198.105.244.228,,,,,,, +Insertion,gijthub.com,198.105.244.228,,,,,,, +Omission,gihub.com,208.73.210.202,,,ns1.dsredirection.com,,,, +Omission,githu.com,69.172.201.153,,,ns1.uniregistrymarket.link,,,, +Omission,ithub.com,,,,,,,, +Omission,gitub.com,68.178.213.61,,,ns1.namefind.com,,,, +Omission,gthub.com,72.52.179.174,,mx156.hostedmxserver.com,ns1.parklogic.com,,,, +Omission,githb.com,109.201.135.44,,,ns1.kirklanddc.com,,,, +Repetition,ggithub.com,52.69.166.231,,,,,,, +Repetition,giithub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,, +Repetition,gitthub.com,54.71.255.210,,gitthub.com,ns3.ifund.com,,,, +Replacement,gitgub.com,,,,,,,, +Replacement,githyb.com,185.53.179.7,,mail.h-email.net,ns1.parkingcrew.net,,,, +Replacement,yithub.com,121.42.226.133,,,dns10.hichina.com,,,, +Replacement,tithub.com,176.74.176.187,,,ns1.uniregistrymarket.link,,,, +Replacement,girhub.com,173.239.22.42,,,ns1.mnsdnsmart.com,,,, +Replacement,gighub.com,,,,,,,, +Replacement,zithub.com,,,,,,,, +Replacement,gith8b.com,198.105.244.228,,,,,,, +Replacement,gityub.com,50.63.202.50,,mailstore1.secureserver.net,ns01.domaincontrol.com,,,, +Replacement,guthub.com,192.30.253.167,,,ns1.p16.dynect.net,,,, +Replacement,hithub.com,50.63.202.32,,mailstore1.secureserver.net,ns23.domaincontrol.com,,,, +Replacement,gith7b.com,107.161.23.204,,,ns1.dnsowl.com,,,, +Replacement,githjb.com,34.233.88.36,,,ns09.domaincontrol.com,,,, +Replacement,vithub.com,78.46.90.242,,mail.vithub.com,ns1.first-ns.de,,,, +Replacement,g8thub.com,107.161.23.204,,,ns1.dnsowl.com,,,, +Replacement,gifhub.com,184.168.221.23,,mailstore1.secureserver.net,ns45.domaincontrol.com,,,, +Replacement,gitnub.com,,,,ns1.dnsimple.com,,,, +Replacement,gjthub.com,98.124.245.24,,mail.b-io.co,ns1.fabulous.com,,,, +Replacement,githzb.com,198.105.244.228,,,,,,, +Replacement,githuv.com,103.224.182.239,,mx92.m1bp.com,ns1.above.com,,,, +Replacement,giyhub.com,103.224.182.239,,mx92.m1bp.com,ns1.above.com,,,, +Replacement,gothub.com,50.63.202.25,,mailstore1.secureserver.net,ns05.domaincontrol.com,,,, +Replacement,g9thub.com,198.105.244.228,,,,,,, +Replacement,gi5hub.com,107.161.23.204,,,ns1.dnsowl.com,,,, +Replacement,gizhub.com,52.216.132.18,,alt1.aspmx.l.google.com,ns-1107.awsdns-10.org,,,, +Replacement,githhb.com,198.105.244.228,,,,,,, +Replacement,githug.com,74.124.210.249,,githug.com,ns.inmotionhosting.com,,,, +Replacement,githuh.com,97.107.140.81,,,dns1.registrar-servers.com,,,, +Replacement,githun.com,185.53.179.6,,mail.h-email.net,ns1.parkingcrew.net,,,, +Replacement,bithub.com,,,,,,,, +Replacement,githib.com,,,,,,,, +Replacement,gi6hub.com,198.105.244.228,,,,,,, +Replacement,gituub.com,184.168.221.44,,mailstore1.secureserver.net,ns57.domaincontrol.com,,,, +Replacement,gitbub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,, +Replacement,gitzub.com,198.105.244.228,,,,,,, +Subdomain,g.ithub.com,,,,,,,, +Subdomain,gi.thub.com,,,,,,,, +Subdomain,git.hub.com,,,,,,,, +Subdomain,gith.ub.com,13.228.110.155,,,,,,, +Subdomain,githu.b.com,,,,,,,, +Transposition,igthub.com,,,smtp1.phocine.net,ns1djs.name.com,,,, +Transposition,gtihub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,, +Transposition,gihtub.com,199.59.242.150,,mx76.m2bp.com,ns1.bodis.com,,,, +Transposition,gituhb.com,103.224.182.241,,mx92.m1bp.com,ns1.above.com,,,, +Transposition,githbu.com,167.114.142.2,,,ns1.chookdns.com,,,, +Vowel-swap,githob.com,173.239.5.6,,mx7.githob.com,ns1.expiereddnsmanager.com,,,, +Vowel-swap,githab.com,72.52.4.120,,localhost,ns1.sedoparking.com,,,, +Vowel-swap,gethub.com,,,,ns.ktnet.co.kr,,,, +Various,githubcom.com,185.53.179.7,,mail.h-email.net,ns1.parkingcrew.net,,,, http://git-wip-us.apache.org/repos/asf/metron/blob/0996b734/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java index d45ffcb..1e30dae 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/FunctionalFunctions.java @@ -41,7 +41,7 @@ public class FunctionalFunctions { @Override public Object apply(List<Object> args) { - List<Object> input = (List<Object>) args.get(0); + Iterable<Object> input = (Iterable<Object>) args.get(0); LambdaExpression expression = (LambdaExpression)args.get(1); if(input == null || expression == null) { return input; @@ -66,7 +66,7 @@ public class FunctionalFunctions { @Override public Object apply(List<Object> args) { - List<Object> input = (List<Object>) args.get(0); + Iterable<Object> input = (Iterable<Object>) args.get(0); LambdaExpression expression = (LambdaExpression) args.get(1); if(input == null || expression == null) { return input; @@ -95,7 +95,7 @@ public class FunctionalFunctions { @Override public Object apply(List<Object> args) { - List<Object> input = (List<Object>) args.get(0); + Iterable<Object> input = (Iterable<Object>) args.get(0); if(input == null || args.size() < 3) { return null; } @@ -105,8 +105,7 @@ public class FunctionalFunctions { if(expression == null || runningResult == null) { return null; } - for(int i = 0;i < input.size();++i) { - Object rhs = input.get(i); + for(Object rhs : input) { runningResult = expression.apply(listOf(runningResult, rhs)); } return runningResult;
