METRON-1858 BasicFireEyeParser check style cleanup and optimization (ottobackwards) closes apache/metron#1255
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b5712af5 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b5712af5 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b5712af5 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: b5712af5b78ff8c6e5e3d3f3c68552204fa7d877 Parents: c6b1b3c Author: ottobackwards <ottobackwa...@gmail.com> Authored: Thu Nov 8 14:14:03 2018 -0500 Committer: otto <o...@apache.org> Committed: Thu Nov 8 14:14:03 2018 -0500 ---------------------------------------------------------------------- .../parsers/fireeye/BasicFireEyeParser.java | 347 +++++++++---------- 1 file changed, 158 insertions(+), 189 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/b5712af5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java index 489eb00..1cf9fb7 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,18 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.metron.parsers.fireeye; import com.google.common.base.Joiner; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; -import org.apache.commons.lang3.StringUtils; -import org.apache.metron.parsers.utils.ParserUtils; -import org.apache.metron.parsers.BasicParser; -import org.json.simple.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; @@ -34,189 +30,162 @@ import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.metron.parsers.BasicParser; +import org.apache.metron.parsers.utils.ParserUtils; +import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class BasicFireEyeParser extends BasicParser { - - private static final long serialVersionUID = 6328907550159134550L; - protected static final Logger LOG = LoggerFactory - .getLogger(BasicFireEyeParser.class); - - - String tsRegex ="([a-zA-Z]{3})\\s+(\\d+)\\s+(\\d+\\:\\d+\\:\\d+)\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+)"; - - - Pattern tsPattern = Pattern.compile(tsRegex); - // private transient static MetronGrok grok; - // private transient static InputStream pattern_url; - - public BasicFireEyeParser() throws Exception { - // pattern_url = getClass().getClassLoader().getResourceAsStream( - // "patterns/fireeye"); - // - // File file = ParserUtils.stream2file(pattern_url); - // grok = MetronGrok.create(file.getPath()); - // - // grok.compile("%{FIREEYE_BASE}"); - } - - @Override - public void configure(Map<String, Object> parserConfig) { - - } - - @Override - public void init() { - - } - - @Override - public List<JSONObject> parse(byte[] raw_message) { - String toParse = ""; - List<JSONObject> messages = new ArrayList<>(); - try { - - toParse = new String(raw_message, "UTF-8"); - - // String[] mTokens = toParse.split(" "); - - String positveIntPattern = "<[1-9][0-9]*>"; - Pattern p = Pattern.compile(positveIntPattern); - Matcher m = p.matcher(toParse); - - String delimiter = ""; - - while (m.find()) { - delimiter = m.group(); - - } - - if (!StringUtils.isBlank(delimiter)) { - String[] tokens = toParse.split(delimiter); - - if (tokens.length > 1) - toParse = delimiter + tokens[1]; - - } - - JSONObject toReturn = parseMessage(toParse); - - toReturn.put("timestamp", getTimeStamp(toParse,delimiter)); - messages.add(toReturn); - return messages; - - } catch (Exception e) { - e.printStackTrace(); - return null; - } - - } - - private long getTimeStamp(String toParse,String delimiter) throws ParseException { - - long ts = 0; - String month = null; - String day = null; - String time = null; - Matcher tsMatcher = tsPattern.matcher(toParse); - if (tsMatcher.find()) { - month = tsMatcher.group(1); - day = tsMatcher.group(2); - time = tsMatcher.group(3); - ts = ParserUtils.convertToEpoch(month, day, time, true); - } else { - LOG.warn("Unable to find timestamp in message: {}", toParse); - } - - return ts; - } - - private JSONObject parseMessage(String toParse) { - - // System.out.println("Received message: " + toParse); - - // MetronMatch gm = grok.match(toParse); - // gm.captures(); - - JSONObject toReturn = new JSONObject(); - //toParse = toParse.replaceAll(" ", " "); - String[] mTokens = toParse.split("\\s+"); - //mTokens = toParse.split(" "); - - // toReturn.putAll(gm.toMap()); - - String id = mTokens[4]; - - // We are not parsing the fedata for multi part message as we cannot - // determine how we can split the message and how many multi part - // messages can there be. - // The message itself will be stored in the response. - - String[] tokens = id.split("\\."); - if (tokens.length == 2) { - - String[] array = Arrays.copyOfRange(mTokens, 1, mTokens.length - 1); - String syslog = Joiner.on(" ").join(array); - - Multimap<String, String> multiMap = formatMain(syslog); - - for (String key : multiMap.keySet()) { - - String value = Joiner.on(",").join(multiMap.get(key)); - toReturn.put(key, value.trim()); - } - - } - - toReturn.put("original_string", toParse); - - String ip_src_addr = (String) toReturn.get("dvc"); - String ip_src_port = (String) toReturn.get("src_port"); - String ip_dst_addr = (String) toReturn.get("dst_ip"); - String ip_dst_port = (String) toReturn.get("dst_port"); - - if (ip_src_addr != null) - toReturn.put("ip_src_addr", ip_src_addr); - if (ip_src_port != null) - toReturn.put("ip_src_port", ip_src_port); - if (ip_dst_addr != null) - toReturn.put("ip_dst_addr", ip_dst_addr); - if (ip_dst_port != null) - toReturn.put("ip_dst_port", ip_dst_port); - -// System.out.println(toReturn); - - return toReturn; - } - - private Multimap<String, String> formatMain(String in) { - Multimap<String, String> multiMap = ArrayListMultimap.create(); - String input = in.replaceAll("cn3", "dst_port") - .replaceAll("cs5", "cncHost").replaceAll("proto", "protocol") - .replaceAll("rt=", "timestamp=").replaceAll("cs1", "malware") - .replaceAll("dst=", "dst_ip=") - .replaceAll("shost", "src_hostname") - .replaceAll("dmac", "dst_mac").replaceAll("smac", "src_mac") - .replaceAll("spt", "src_port") - .replaceAll("\\bsrc\\b", "src_ip"); - String[] tokens = input.split("\\|"); - - if (tokens.length > 0) { - String message = tokens[tokens.length - 1]; - - String pattern = "([\\w\\d]+)=([^=]*)(?=\\s*\\w+=|\\s*$) "; - Pattern p = Pattern.compile(pattern); - Matcher m = p.matcher(message); - - while (m.find()) { - String[] str = m.group().split("="); - multiMap.put(str[0], str[1]); - - } - } - return multiMap; - } - +public class BasicFireEyeParser extends BasicParser { + private static final long serialVersionUID = 6328907550159134550L; + protected static final Logger LOG = LoggerFactory + .getLogger(MethodHandles.lookup().lookupClass()); + + + private static final String tsRegex = "([a-zA-Z]{3})\\s+(\\d+)\\s+(\\d+\\:\\d+\\:\\d+)" + + "\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+)"; + private static final Pattern tsPattern = Pattern.compile(tsRegex); + private static final String syslogPriorityRegex = "<[1-9][0-9]*>"; + private static final Pattern syslogPriorityPattern = Pattern.compile(syslogPriorityRegex); + private static final String nvRegex = "([\\w\\d]+)=([^=]*)(?=\\s*\\w+=|\\s*$) "; + private static final Pattern nvPattern = Pattern.compile(nvRegex); + + @Override + public void configure(Map<String, Object> parserConfig) {} + + @Override + public void init() {} + + + @Override + @SuppressWarnings("unchecked") + public List<JSONObject> parse(byte[] rawMessage) { + String toParse; + List<JSONObject> messages = new ArrayList<>(); + try { + + toParse = new String(rawMessage, StandardCharsets.UTF_8); + + // because we support what is basically a malformed syslog 3164 message having + // some form of text before the PRIORITY, we need to use the priority as + // a delimiter + Matcher m = syslogPriorityPattern.matcher(toParse); + + String delimiter = ""; + + while (m.find()) { + delimiter = m.group(); + } + + if (!StringUtils.isBlank(delimiter)) { + String[] tokens = toParse.split(delimiter); + if (tokens.length > 1) { + toParse = delimiter + tokens[1]; + } + } + + // parse the main message + JSONObject toReturn = parseMessage(toParse); + toReturn.put("timestamp", getTimeStamp(toParse)); + messages.add(toReturn); + return messages; + } catch (Exception e) { + String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); + } + } + + private long getTimeStamp(String toParse) throws ParseException { + long timestamp = 0; + String month; + String day; + String time; + Matcher tsMatcher = tsPattern.matcher(toParse); + if (tsMatcher.find()) { + month = tsMatcher.group(1); + day = tsMatcher.group(2); + time = tsMatcher.group(3); + timestamp = ParserUtils.convertToEpoch(month, day, time, true); + } else { + LOG.warn("Unable to find timestamp in message: {}", toParse); + } + return timestamp; + } + + @SuppressWarnings("unchecked") + private JSONObject parseMessage(String toParse) { + + JSONObject toReturn = new JSONObject(); + String[] messageTokens = toParse.split("\\s+"); + String id = messageTokens[4]; + + // We are not parsing the fedata for multi part message as we cannot + // determine how we can split the message and how many multi part + // messages can there be. + // The message itself will be stored in the response. + + String[] tokens = id.split("\\."); + if (tokens.length == 2) { + + String[] array = Arrays.copyOfRange(messageTokens, 1, messageTokens.length - 1); + String syslog = Joiner.on(" ").join(array); + + Multimap<String, String> multiMap = formatMain(syslog); + + for (String key : multiMap.keySet()) { + String value = Joiner.on(",").join(multiMap.get(key)); + toReturn.put(key, value.trim()); + } + } + + toReturn.put("original_string", toParse); + + final String ipSrcAddr = (String) toReturn.get("dvc"); + final String ipSrcPort = (String) toReturn.get("src_port"); + final String ipDstDddr = (String) toReturn.get("dst_ip"); + final String ipDstPort = (String) toReturn.get("dst_port"); + + if (ipSrcAddr != null) { + toReturn.put("ip_src_addr", ipSrcAddr); + } + if (ipSrcPort != null) { + toReturn.put("ip_src_port", ipSrcPort); + } + if (ipDstDddr != null) { + toReturn.put("ip_dst_addr", ipDstDddr); + } + if (ipDstPort != null) { + toReturn.put("ip_dst_port", ipDstPort); + } + return toReturn; + } + + private Multimap<String, String> formatMain(String in) { + Multimap<String, String> multiMap = ArrayListMultimap.create(); + String input = in.replaceAll("cn3", "dst_port") + .replaceAll("cs5", "cncHost").replaceAll("proto", "protocol") + .replaceAll("rt=", "timestamp=").replaceAll("cs1", "malware") + .replaceAll("dst=", "dst_ip=") + .replaceAll("shost", "src_hostname") + .replaceAll("dmac", "dst_mac").replaceAll("smac", "src_mac") + .replaceAll("spt", "src_port") + .replaceAll("\\bsrc\\b", "src_ip"); + String[] tokens = input.split("\\|"); + + if (tokens.length > 0) { + String message = tokens[tokens.length - 1]; + Matcher m = nvPattern.matcher(message); + + while (m.find()) { + String[] str = m.group().split("="); + multiMap.put(str[0], str[1]); + } + } + return multiMap; + } }
