http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java deleted file mode 100644 index 2ccc23a..0000000 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/TokenMgrError.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */ -/* JavaCCOptions: */ -package org.apache.metron.parsers.ise; - -/** Token Manager Error. */ -class TokenMgrError extends Error -{ - - /** - * The version identifier for this Serializable class. - * Increment only if the <i>serialized</i> form of the - * class changes. - */ - private static final long serialVersionUID = 1L; - - /* - * Ordinals for various reasons why an Error of this type can be thrown. - */ - - /** - * Lexical error occurred. - */ - static final int LEXICAL_ERROR = 0; - - /** - * An attempt was made to create a second instance of a static token manager. - */ - static final int STATIC_LEXER_ERROR = 1; - - /** - * Tried to change to an invalid lexical state. - */ - static final int INVALID_LEXICAL_STATE = 2; - - /** - * Detected (and bailed out of) an infinite loop in the token manager. - */ - static final int LOOP_DETECTED = 3; - - /** - * Indicates the reason why the exception is thrown. It will have - * one of the above 4 values. - */ - int errorCode; - - /** - * Replaces unprintable characters by their escaped (or unicode escaped) - * equivalents in the given string - */ - protected static final String addEscapes(String str) { - StringBuffer retval = new StringBuffer(); - char ch; - for (int i = 0; i < str.length(); i++) { - switch (str.charAt(i)) - { - case 0 : - continue; - case '\b': - retval.append("\\b"); - continue; - case '\t': - retval.append("\\t"); - continue; - case '\n': - retval.append("\\n"); - continue; - case '\f': - retval.append("\\f"); - continue; - case '\r': - retval.append("\\r"); - continue; - case '\"': - retval.append("\\\""); - continue; - case '\'': - retval.append("\\\'"); - continue; - case '\\': - retval.append("\\\\"); - continue; - default: - if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { - String s = "0000" + Integer.toString(ch, 16); - retval.append("\\u" + s.substring(s.length() - 4, s.length())); - } else { - retval.append(ch); - } - continue; - } - } - return retval.toString(); - } - - /** - * Returns a detailed message for the Error when it is thrown by the - * token manager to indicate a lexical error. - * Parameters : - * EOFSeen : indicates if EOF caused the lexical error - * curLexState : lexical state in which this error occurred - * errorLine : line number when the error occurred - * errorColumn : column number when the error occurred - * errorAfter : prefix that was seen before this error occurred - * curchar : the offending character - * Note: You can customize the lexical error message by modifying this method. - */ - protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { - return("Lexical error at line " + - errorLine + ", column " + - errorColumn + ". Encountered: " + - (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") + - "after : \"" + addEscapes(errorAfter) + "\""); - } - - /** - * You can also modify the body of this method to customize your error messages. - * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not - * of end-users concern, so you can return something like : - * - * "Internal Error : Please file a bug report .... " - * - * from this method for such cases in the release version of your parser. - */ - public String getMessage() { - return super.getMessage(); - } - - /* - * Constructors of various flavors follow. - */ - - /** No arg constructor. */ - public TokenMgrError() { - } - - /** Constructor with message and reason. */ - public TokenMgrError(String message, int reason) { - super(message); - errorCode = reason; - } - - /** Full Constructor. */ - public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) { - this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); - } -} -/* JavaCC - OriginalChecksum=5fbf6813c9d6a1d713f1d4a002af1322 (do not edit this line) */
http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java deleted file mode 100644 index b12e98f..0000000 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.parsers.lancope; - -import java.lang.invoke.MethodHandles; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import org.apache.metron.parsers.BasicParser; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings("serial") -public class BasicLancopeParser extends BasicParser { - // Sample Lancope Message - // {"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"} - - private static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Override - public void configure(Map<String, Object> parserConfig) { - - } - - @Override - public void init() { - - } - - //@SuppressWarnings("unchecked") - @Override - public List<JSONObject> parse(byte[] msg) { - - JSONObject payload = null; - List<JSONObject> messages = new ArrayList<>(); - try { - - String raw_message = new String(msg, "UTF-8"); - - payload = (JSONObject) JSONValue.parse(raw_message); - - - - String message = payload.get("message").toString(); - String[] parts = message.split(" "); - payload.put("ip_src_addr", parts[6]); - payload.put("ip_dst_addr", parts[7]); - - String fixed_date = parts[5].replace('T', ' '); - fixed_date = fixed_date.replace('Z', ' ').trim(); - - SimpleDateFormat formatter = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - - Date date; - - date = formatter.parse(fixed_date); - long timestamp = date.getTime(); - payload.put("timestamp", timestamp); - - payload.remove("@timestamp"); - payload.remove("message"); - payload.put("original_string", message); - - messages.add(payload); - return messages; - } catch (Exception e) { - - _LOG.error("Unable to parse message: {}", payload.toJSONString()); - return null; - } - } - - -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java deleted file mode 100644 index 2f5310c..0000000 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.parsers.logstash; - -import org.apache.metron.parsers.BasicParser; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class BasicLogstashParser extends BasicParser { - - @Override - public void configure(Map<String, Object> parserConfig) { - - } - - @Override - public void init() { - - } - - @Override - public List<JSONObject> parse(byte[] raw_message) { - List<JSONObject> messages = new ArrayList<>(); - try { - - /* - * We need to create a new JSONParser each time because its - * not serializable and the parser is created on the storm nimbus - * node, then transfered to the workers. - */ - JSONParser jsonParser = new JSONParser(); - String rawString = new String(raw_message, "UTF-8"); - JSONObject rawJson = (JSONObject) jsonParser.parse(rawString); - - // remove logstash meta fields - rawJson.remove("@version"); - rawJson.remove("type"); - rawJson.remove("host"); - rawJson.remove("tags"); - - // rename other keys - rawJson = mutate(rawJson, "message", "original_string"); - rawJson = mutate(rawJson, "src_ip", "ip_src_addr"); - rawJson = mutate(rawJson, "dst_ip", "ip_dst_addr"); - rawJson = mutate(rawJson, "src_port", "ip_src_port"); - rawJson = mutate(rawJson, "dst_port", "ip_dst_port"); - rawJson = mutate(rawJson, "src_ip", "ip_src_addr"); - - // convert timestamp to milli since epoch - long timestamp = LogstashToEpoch((String) rawJson.remove("@timestamp")); - rawJson.put("timestamp", timestamp); - messages.add(rawJson); - return messages; - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - private JSONObject mutate(JSONObject json, String oldKey, String newKey) { - if (json.containsKey(oldKey)) { - json.put(newKey, json.remove(oldKey)); - } - return json; - } - - private long LogstashToEpoch(String timestamp) throws java.text.ParseException { - SimpleDateFormat logstashDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - return logstashDateFormat.parse(timestamp).getTime(); - - } - - - -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java deleted file mode 100644 index 46155b3..0000000 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.parsers.paloalto; - - -import org.apache.metron.parsers.BasicParser; -import org.json.simple.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class BasicPaloAltoFirewallParser extends BasicParser { - - private static final Logger _LOG = LoggerFactory.getLogger - (BasicPaloAltoFirewallParser.class); - - private static final long serialVersionUID = 3147090149725343999L; - public static final String PaloAltoDomain = "palo_alto_domain"; - public static final String ReceiveTime = "receive_time"; - public static final String SerialNum = "serial_num"; - public static final String Type = "type"; - public static final String ThreatContentType = "threat_content_type"; - public static final String ConfigVersion = "config_version"; - public static final String GenerateTime = "generate_time"; - public static final String SourceAddress = "source_address"; - public static final String DestinationAddress = "destination_address"; - public static final String NATSourceIP = "nat_source_ip"; - public static final String NATDestinationIP = "nat_destination_ip"; - public static final String Rule = "rule"; - public static final String SourceUser = "source_user"; - public static final String DestinationUser = "destination_user"; - public static final String Application = "application"; - public static final String VirtualSystem = "virtual_system"; - public static final String SourceZone = "source_zone"; - public static final String DestinationZone = "destination_zone"; - public static final String InboundInterface = "inbound_interface"; - public static final String OutboundInterface = "outbound_interface"; - public static final String LogAction = "log_action"; - public static final String TimeLogged = "time_logged"; - public static final String SessionID = "session_id"; - public static final String RepeatCount = "repeat_count"; - public static final String SourcePort = "source_port"; - public static final String DestinationPort = "destination_port"; - public static final String NATSourcePort = "nats_source_port"; - public static final String NATDestinationPort = "nats_destination_port"; - public static final String Flags = "flags"; - public static final String IPProtocol = "ip_protocol"; - public static final String Action = "action"; - - //Threat - public static final String URL = "url"; - public static final String HOST = "host"; - public static final String ThreatContentName = "threat_content_name"; - public static final String Category = "category"; - public static final String Direction = "direction"; - public static final String Seqno = "seqno"; - public static final String ActionFlags = "action_flags"; - public static final String SourceCountry = "source_country"; - public static final String DestinationCountry = "destination_country"; - public static final String Cpadding = "cpadding"; - public static final String ContentType = "content_type"; - - //Traffic - public static final String Bytes = "content_type"; - public static final String BytesSent = "content_type"; - public static final String BytesReceived = "content_type"; - public static final String Packets = "content_type"; - public static final String StartTime = "content_type"; - public static final String ElapsedTimeInSec = "content_type"; - public static final String Padding = "content_type"; - public static final String PktsSent = "pkts_sent"; - public static final String PktsReceived = "pkts_received"; - - @Override - public void configure(Map<String, Object> parserConfig) { - - } - - @Override - public void init() { - - } - - @Override - @SuppressWarnings({"unchecked", "unused"}) - public List<JSONObject> parse(byte[] msg) { - - JSONObject outputMessage = new JSONObject(); - String toParse = ""; - List<JSONObject> messages = new ArrayList<>(); - try { - - toParse = new String(msg, "UTF-8"); - _LOG.debug("Received message: {}", toParse); - - - parseMessage(toParse, outputMessage); - long timestamp = System.currentTimeMillis(); - outputMessage.put("timestamp", System.currentTimeMillis()); - outputMessage.put("ip_src_addr", outputMessage.remove("source_address")); - outputMessage.put("ip_src_port", outputMessage.remove("source_port")); - outputMessage.put("ip_dst_addr", outputMessage.remove("destination_address")); - outputMessage.put("ip_dst_port", outputMessage.remove("destination_port")); - outputMessage.put("protocol", outputMessage.remove("ip_protocol")); - - outputMessage.put("original_string", toParse); - messages.add(outputMessage); - return messages; - } catch (Exception e) { - e.printStackTrace(); - _LOG.error("Failed to parse: {}", toParse); - return null; - } - } - - @SuppressWarnings("unchecked") - private void parseMessage(String message, JSONObject outputMessage) { - - String[] tokens = message.split(","); - - String type = tokens[3].trim(); - - //populate common objects - outputMessage.put(PaloAltoDomain, tokens[0].trim()); - outputMessage.put(ReceiveTime, tokens[1].trim()); - outputMessage.put(SerialNum, tokens[2].trim()); - outputMessage.put(Type, type); - outputMessage.put(ThreatContentType, tokens[4].trim()); - outputMessage.put(ConfigVersion, tokens[5].trim()); - outputMessage.put(GenerateTime, tokens[6].trim()); - outputMessage.put(SourceAddress, tokens[7].trim()); - outputMessage.put(DestinationAddress, tokens[8].trim()); - outputMessage.put(NATSourceIP, tokens[9].trim()); - outputMessage.put(NATDestinationIP, tokens[10].trim()); - outputMessage.put(Rule, tokens[11].trim()); - outputMessage.put(SourceUser, tokens[12].trim()); - outputMessage.put(DestinationUser, tokens[13].trim()); - outputMessage.put(Application, tokens[14].trim()); - outputMessage.put(VirtualSystem, tokens[15].trim()); - outputMessage.put(SourceZone, tokens[16].trim()); - outputMessage.put(DestinationZone, tokens[17].trim()); - outputMessage.put(InboundInterface, tokens[18].trim()); - outputMessage.put(OutboundInterface, tokens[19].trim()); - outputMessage.put(LogAction, tokens[20].trim()); - outputMessage.put(TimeLogged, tokens[21].trim()); - outputMessage.put(SessionID, tokens[22].trim()); - outputMessage.put(RepeatCount, tokens[23].trim()); - outputMessage.put(SourcePort, tokens[24].trim()); - outputMessage.put(DestinationPort, tokens[25].trim()); - outputMessage.put(NATSourcePort, tokens[26].trim()); - outputMessage.put(NATDestinationPort, tokens[27].trim()); - outputMessage.put(Flags, tokens[28].trim()); - outputMessage.put(IPProtocol, tokens[29].trim()); - outputMessage.put(Action, tokens[30].trim()); - - - if ("THREAT".equals(type.toUpperCase())) { - outputMessage.put(URL, tokens[31].trim()); - try { - URL url = new URL(tokens[31].trim()); - outputMessage.put(HOST, url.getHost()); - } catch (MalformedURLException e) { - } - outputMessage.put(ThreatContentName, tokens[32].trim()); - outputMessage.put(Category, tokens[33].trim()); - outputMessage.put(Direction, tokens[34].trim()); - outputMessage.put(Seqno, tokens[35].trim()); - outputMessage.put(ActionFlags, tokens[36].trim()); - outputMessage.put(SourceCountry, tokens[37].trim()); - outputMessage.put(DestinationCountry, tokens[38].trim()); - outputMessage.put(Cpadding, tokens[39].trim()); - outputMessage.put(ContentType, tokens[40].trim()); - - } else { - outputMessage.put(Bytes, tokens[31].trim()); - outputMessage.put(BytesSent, tokens[32].trim()); - outputMessage.put(BytesReceived, tokens[33].trim()); - outputMessage.put(Packets, tokens[34].trim()); - outputMessage.put(StartTime, tokens[35].trim()); - outputMessage.put(ElapsedTimeInSec, tokens[36].trim()); - outputMessage.put(Category, tokens[37].trim()); - outputMessage.put(Padding, tokens[38].trim()); - outputMessage.put(Seqno, tokens[39].trim()); - outputMessage.put(ActionFlags, tokens[40].trim()); - outputMessage.put(SourceCountry, tokens[41].trim()); - outputMessage.put(DestinationCountry, tokens[42].trim()); - outputMessage.put(Cpadding, tokens[43].trim()); - outputMessage.put(PktsSent, tokens[44].trim()); - outputMessage.put(PktsReceived, tokens[45].trim()); - } - - } - - -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java deleted file mode 100644 index 4dfe19a..0000000 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java +++ /dev/null @@ -1,214 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.parsers.snort; - -import com.google.common.collect.Lists; -import java.lang.invoke.MethodHandles; -import java.text.ParseException; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.StringUtils; -import org.apache.metron.common.Constants; -import org.apache.metron.common.csv.CSVConverter; -import org.apache.metron.parsers.BasicParser; -import org.json.simple.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings("serial") -public class BasicSnortParser extends BasicParser { - - private static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** - * The default field names for Snort Alerts. - */ - private String[] fieldNames = new String[] { - Constants.Fields.TIMESTAMP.getName(), - "sig_generator", - "sig_id", - "sig_rev", - "msg", - Constants.Fields.PROTOCOL.getName(), - Constants.Fields.SRC_ADDR.getName(), - Constants.Fields.SRC_PORT.getName(), - Constants.Fields.DST_ADDR.getName(), - Constants.Fields.DST_PORT.getName(), - "ethsrc", - "ethdst", - "ethlen", - "tcpflags", - "tcpseq", - "tcpack", - "tcplen", - "tcpwindow", - "ttl", - "tos", - "id", - "dgmlen", - "iplen", - "icmptype", - "icmpcode", - "icmpid", - "icmpseq" - }; - - - /** - * Snort alerts are received as CSV records - */ - private String recordDelimiter = ","; - - private transient CSVConverter converter; - - private static String defaultDateFormat = "MM/dd/yy-HH:mm:ss.SSSSSS"; - private transient DateTimeFormatter dateTimeFormatter; - - public BasicSnortParser() { - - } - - @Override - public void configure(Map<String, Object> parserConfig) { - dateTimeFormatter = getDateFormatter(parserConfig); - dateTimeFormatter = getDateFormatterWithZone(dateTimeFormatter, parserConfig); - init(); - } - - private DateTimeFormatter getDateFormatter(Map<String, Object> parserConfig) { - String format = (String) parserConfig.get("dateFormat"); - if (StringUtils.isNotEmpty(format)) { - _LOG.info("Using date format '{}'", format); - return DateTimeFormatter.ofPattern(format); - } else { - _LOG.info("Using default date format '{}'", defaultDateFormat); - return DateTimeFormatter.ofPattern(defaultDateFormat); - } - } - - private DateTimeFormatter getDateFormatterWithZone(DateTimeFormatter formatter, Map<String, Object> parserConfig) { - String timezone = (String) parserConfig.get("timeZone"); - if (StringUtils.isNotEmpty(timezone)) { - if(ZoneId.getAvailableZoneIds().contains(timezone)) { - _LOG.info("Using timezone '{}'", timezone); - return formatter.withZone(ZoneId.of(timezone)); - } else { - throw new IllegalArgumentException("Unable to find ZoneId '" + timezone + "'"); - } - } else { - _LOG.info("Using default timezone '{}'", ZoneId.systemDefault()); - return formatter.withZone(ZoneId.systemDefault()); - } - } - - @Override - public void init() { - if(converter == null) { - converter = new CSVConverter(); - Map<String, Object> config = new HashMap<>(); - config.put(CSVConverter.SEPARATOR_KEY, recordDelimiter); - config.put(CSVConverter.COLUMNS_KEY, Lists.newArrayList(fieldNames)); - converter.initialize(config); - } - } - - @Override - public List<JSONObject> parse(byte[] rawMessage) { - - JSONObject jsonMessage = new JSONObject(); - List<JSONObject> messages = new ArrayList<>(); - try { - // snort alerts expected as csv records - String csvMessage = new String(rawMessage, "UTF-8"); - Map<String, String> records = null; - try { - records = converter.toMap(csvMessage); - } - catch(ArrayIndexOutOfBoundsException aioob) { - throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " in " + csvMessage); - } - - // validate the number of fields - if (records.size() != fieldNames.length) { - throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.size()); - } - long timestamp = 0L; - // build the json record from each field - for (Map.Entry<String, String> kv : records.entrySet()) { - - String field = kv.getKey(); - String record = kv.getValue(); - - if("timestamp".equals(field)) { - - // convert the timestamp to epoch - timestamp = toEpoch(record); - jsonMessage.put("timestamp", timestamp); - - } else { - jsonMessage.put(field, record); - } - } - - // add original msg; required by 'checkForSchemaCorrectness' - jsonMessage.put("original_string", csvMessage); - jsonMessage.put("is_alert", "true"); - messages.add(jsonMessage); - } catch (Exception e) { - String message = "Unable to parse message: " + (rawMessage == null?"null" : new String(rawMessage)); - _LOG.error(message, e); - throw new IllegalStateException(message, e); - } - - return messages; - } - - /** - * Parses Snort's default date-time representation and - * converts to epoch. - * @param snortDatetime Snort's default date-time as String '01/27-16:01:04.877970' - * @return epoch time - * @throws java.text.ParseException - */ - private long toEpoch(String snortDatetime) throws ParseException { - ZonedDateTime zonedDateTime = ZonedDateTime.parse(snortDatetime.trim(), dateTimeFormatter); - return zonedDateTime.toInstant().toEpochMilli(); - } - - public String getRecordDelimiter() { - return this.recordDelimiter; - } - - public void setRecordDelimiter(String recordDelimiter) { - this.recordDelimiter = recordDelimiter; - } - - public String[] getFieldNames() { - return this.fieldNames; - } - - public void setFieldNames(String[] fieldNames) { - this.fieldNames = fieldNames; - } - -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java deleted file mode 100644 index 91faca2..0000000 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.parsers.sourcefire; - -import org.apache.metron.parsers.BasicParser; -import org.json.simple.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@SuppressWarnings("serial") -public class BasicSourcefireParser extends BasicParser { - - private static final Logger _LOG = LoggerFactory - .getLogger(BasicSourcefireParser.class); - - public static final String hostkey = "host"; - String domain_name_regex = "([^\\.]+)\\.([a-z]{2}|[a-z]{3}|([a-z]{2}\\.[a-z]{2}))$"; - String sidRegex = "(.*)(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; - //String sidRegex = "(\\[[0-9]+:[0-9]+:[0-9]\\])(.*)$"; - Pattern sidPattern = Pattern.compile(sidRegex); - Pattern pattern = Pattern.compile(domain_name_regex); - - @Override - public void configure(Map<String, Object> parserConfig) { - - } - - @Override - public void init() { - - } - - @Override - @SuppressWarnings({ "unchecked", "unused" }) - public List<JSONObject> parse(byte[] msg) { - - JSONObject payload = new JSONObject(); - String toParse = ""; - List<JSONObject> messages = new ArrayList<>(); - try { - - toParse = new String(msg, "UTF-8"); - _LOG.debug("Received message: {}", toParse); - - String tmp = toParse.substring(toParse.lastIndexOf("{")); - payload.put("key", tmp); - - String protocol = tmp.substring(tmp.indexOf("{") + 1, - tmp.indexOf("}")).toLowerCase(); - String source = tmp.substring(tmp.indexOf("}") + 1, - tmp.indexOf("->")).trim(); - String dest = tmp.substring(tmp.indexOf("->") + 2, tmp.length()) - .trim(); - - payload.put("protocol", protocol); - - String source_ip = ""; - String dest_ip = ""; - - if (source.contains(":")) { - String parts[] = source.split(":"); - payload.put("ip_src_addr", parts[0]); - payload.put("ip_src_port", parts[1]); - source_ip = parts[0]; - } else { - payload.put("ip_src_addr", source); - source_ip = source; - - } - - if (dest.contains(":")) { - String parts[] = dest.split(":"); - payload.put("ip_dst_addr", parts[0]); - payload.put("ip_dst_port", parts[1]); - dest_ip = parts[0]; - } else { - payload.put("ip_dst_addr", dest); - dest_ip = dest; - } - long timestamp = System.currentTimeMillis(); - payload.put("timestamp", timestamp); - - Matcher sidMatcher = sidPattern.matcher(toParse); - String originalString = null; - String signatureId = ""; - if (sidMatcher.find()) { - signatureId = sidMatcher.group(2); - originalString = sidMatcher.group(1) +" "+ sidMatcher.group(2) + " " + sidMatcher.group(3); - } else { - _LOG.warn("Unable to find SID in message: {}", toParse); - originalString = toParse; - } - payload.put("original_string", originalString); - payload.put("signature_id", signatureId); - messages.add(payload); - return messages; - } catch (Exception e) { - e.printStackTrace(); - _LOG.error("Failed to parse: {}", toParse); - return null; - } - } - - - - -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java index feac80b..53d3d99 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java @@ -36,7 +36,6 @@ import org.apache.metron.common.utils.ReflectionUtils; import org.apache.metron.parsers.bolt.ParserBolt; import org.apache.metron.parsers.bolt.WriterBolt; import org.apache.metron.parsers.bolt.WriterHandler; -import org.apache.metron.parsers.interfaces.MessageParser; import org.apache.metron.writer.AbstractWriter; import org.apache.metron.writer.kafka.KafkaWriter; import org.json.simple.JSONObject; @@ -78,6 +77,8 @@ public class ParserTopologyBuilder { Optional<String> outputTopic ) throws Exception { + + // fetch configuration from zookeeper ParserConfigurations configs = new ParserConfigurations(); SensorParserConfig parserConfig = getSensorParserConfig(zookeeperUrl, sensorType, configs); @@ -181,11 +182,6 @@ public class ParserTopologyBuilder { , Optional<String> outputTopic ) { - - // create message parser - MessageParser<JSONObject> parser = ReflectionUtils.createInstance(parserConfig.getParserClassName()); - parser.configure(parserConfig.getParserConfig()); - // create writer - if not configured uses a sensible default AbstractWriter writer = parserConfig.getWriterClassName() == null ? createKafkaWriter( brokerUrl @@ -198,7 +194,7 @@ public class ParserTopologyBuilder { // create a writer handler WriterHandler writerHandler = createWriterHandler(writer); - return new ParserBolt(zookeeperUrl, sensorType, parser, writerHandler); + return new ParserBolt(zookeeperUrl, sensorType, writerHandler); } /** http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java deleted file mode 100644 index 178719b..0000000 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.parsers.websphere; - -import org.apache.metron.parsers.GrokParser; -import org.json.simple.JSONObject; - -import java.text.ParseException; -import java.util.Calendar; -import java.util.Iterator; - -public class GrokWebSphereParser extends GrokParser { - - private static final long serialVersionUID = 4860439408055777358L; - - @Override - protected long formatTimestamp(Object value) { - long epochTimestamp = System.currentTimeMillis(); - if (value != null) { - try { - epochTimestamp = toEpoch(Calendar.getInstance().get(Calendar.YEAR) + " " + value); - } catch (ParseException e) { - //default to current time - } - } - return epochTimestamp; - } - - @Override - protected void postParse(JSONObject message) { - removeEmptyFields(message); - message.remove("timestamp_string"); - if (message.containsKey("message")) { - String messageValue = (String) message.get("message"); - if (messageValue.contains("logged into")) { - parseLoginMessage(message); - } - else if (messageValue.contains("logged out")) { - parseLogoutMessage(message); - } - else if (messageValue.contains("rbm(")) { - parseRBMMessage(message); - } - else { - parseOtherMessage(message); - } - } - } - - @SuppressWarnings("unchecked") - private void removeEmptyFields(JSONObject json) { - Iterator<Object> keyIter = json.keySet().iterator(); - while (keyIter.hasNext()) { - Object key = keyIter.next(); - Object value = json.get(key); - if (null == value || "".equals(value.toString())) { - keyIter.remove(); - } - } - } - - //Extracts the appropriate fields from login messages - @SuppressWarnings("unchecked") - private void parseLoginMessage(JSONObject json) { - json.put("event_subtype", "login"); - String message = (String) json.get("message"); - if (message.contains(":")){ - String parts[] = message.split(":"); - String user = parts[0]; - String ip_src_addr = parts[1]; - if (user.contains("user(") && user.contains(")")) { - user = user.substring(user.indexOf("user(") + "user(".length()); - user = user.substring(0, user.indexOf(")")); - json.put("username", user); - } - if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) { - ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1); - ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]")); - json.put("ip_src_addr", ip_src_addr); - } - json.remove("message"); - } - } - - //Extracts the appropriate fields from logout messages - @SuppressWarnings("unchecked") - private void parseLogoutMessage(JSONObject json) { - json.put("event_subtype", "logout"); - String message = (String) json.get("message"); - if (message.matches(".*'.*'.*'.*'.*")) { - String parts[] = message.split("'"); - String ip_src_addr = parts[0]; - if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) { - ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1); - ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]")); - json.put("ip_src_addr", ip_src_addr); - } - json.put("username", parts[1]); - json.put("security_domain", parts[3]); - json.remove("message"); - } - } - - //Extracts the appropriate fields from RBM messages - @SuppressWarnings("unchecked") - private void parseRBMMessage(JSONObject json) { - String message = (String) json.get("message"); - if (message.contains("(")) { - json.put("process", message.substring(0, message.indexOf("("))); - if (message.contains(":")) { - json.put("message", message.substring(message.indexOf(":") + 2)); - } - } - } - - //Extracts the appropriate fields from other messages - @SuppressWarnings("unchecked") - private void parseOtherMessage(JSONObject json) { - String message = (String) json.get("message"); - if (message.contains("(")) { - json.put("process", message.substring(0, message.indexOf("("))); - if (message.contains(":")) { - json.put("message", message.substring(message.indexOf(":") + 2)); - } - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/resources/patterns/asa ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/asa b/metron-platform/metron-parsers/src/main/resources/patterns/asa deleted file mode 100644 index dee2a37..0000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/asa +++ /dev/null @@ -1,179 +0,0 @@ -# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns - -USERNAME [a-zA-Z0-9._-]+ -USER %{USERNAME:UNWANTED} -INT (?:[+-]?(?:[0-9]+)) -BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) -NUMBER (?:%{BASE10NUM:UNWANTED}) -BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+)) -BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b - -POSINT \b(?:[1-9][0-9]*)\b -NONNEGINT \b(?:[0-9]+)\b -WORD \b\w+\b -NOTSPACE \S+ -SPACE \s* -DATA .*? -GREEDYDATA .* -#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`))) -QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) -UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} - -# Networking -MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) -CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) -WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) -COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) -IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED}) -HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b) -HOST %{HOSTNAME:UNWANTED} -IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED}) -HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT}) - -# paths -PATH (?:%{UNIXPATH}|%{WINPATH}) -UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+ -#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+ -TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) -WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ -URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? -URIHOST %{IPORHOST}(?::%{POSINT:port})? -# uripath comes loosely from RFC1738, but mostly from what Firefox -# doesn't turn into %XX -URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ -#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? -URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* -URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? -URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? - -# Months: January, Feb, 3, 03, 12, December -MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b -MONTHNUM (?:0?[1-9]|1[0-2]) -MONTHNUM2 (?:0[1-9]|1[0-2]) -MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) - -# Days: Monday, Tue, Thu, etc... -DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) - -# Years? -YEAR (?>\d\d){1,2} -# Time: HH:MM:SS -#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? -# I'm still on the fence about using grok to perform the time match, -# since it's probably slower. -# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? -HOUR (?:2[0123]|[01]?[0-9]) -MINUTE (?:[0-5][0-9]) -# '60' is a leap second in most time standards and thus is valid. -SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) -TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) -# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) -DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} -DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} -ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) -ISO8601_SECOND (?:%{SECOND}|60) -TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? -DATE %{DATE_US}|%{DATE_EU} -DATESTAMP %{DATE}[- ]%{TIME} -TZ (?:[PMCE][SD]T|UTC) -DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} -DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} -DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} -DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} -GREEDYDATA .* - -# Syslog Dates: Month Day HH:MM:SS -SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} -PROG (?:[\w._/%-]+) -SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? -SYSLOGHOST %{IPORHOST} -SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> -SYSLOGPRIORITY <%{POSINT:syslog_pri}> -HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} - -# Shortcuts -QS %{QUOTEDSTRING:UNWANTED} - -# Log formats -SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: - -MESSAGESLOG %{SYSLOGBASE} %{DATA} - -COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) -COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} - -# Log Levels -LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) - -#== Cisco ASA == -CISCO_TAGGED_SYSLOG ^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP}( %{SYSLOGHOST:syslog_host})?( %{SYSLOGPROG:syslog_prog})? ?:? %%{CISCOTAG}%{GREEDYDATA:message} -CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} -CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) - -# Common Particles -CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted -CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* -CISCO_DIRECTION Inbound|inbound|Outbound|outbound -CISCO_INTERVAL first hit|%{INT}-second interval -CISCO_XLATE_TYPE static|dynamic -# ASA-2-106001 -CISCOFW106001 : %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} -# ASA-2-106006, ASA-2-106007, ASA-2-106010 -CISCOFW106006_106007_106010 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\(%{DATA:dst_fwuser}\))? (?:on interface %{GREEDYDATA:interface}|due to %{CISCO_REASON:reason}) -# ASA-3-106014 -CISCOFW106014 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\) -# ASA-6-106015 -CISCOFW106015 : %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} -# ASA-1-106021 -CISCOFW106021 : %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} -# ASA-4-106023 -CISCOFW106023 : %{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:policy_id} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -# ASA-5-106100 -CISCOFW106100 : access-list %{WORD:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -# ASA-6-110002 -CISCOFW110002 : %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -# ASA-6-302010 -CISCOFW302010 : %{INT:connection_count} in use, %{INT:connection_count_max} most used -# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 -CISCOFW302013_302014_302015_302016 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? -# ASA-6-302020, ASA-6-302021 -CISCOFW302020_302021 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? -# ASA-6-305011 -CISCOFW305011 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} -# ASA-3-313001, ASA-3-313004, ASA-3-313008 -CISCOFW313001_313004_313008 : %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? -# ASA-4-313005 -CISCOFW313005 : %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\(%{DATA:orig_dst_fwuser}\))? -# ASA-4-402117 -CISCOFW402117 : %{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} -# ASA-4-402119 -CISCOFW402119 : %{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking -# ASA-4-419001 -CISCOFW419001 : %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason} -# ASA-4-419002 -CISCOFW419002 : %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number -# ASA-4-500004 -CISCOFW500004 : %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -# ASA-6-602303, ASA-6-602304 -CISCOFW602303_602304 : %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} -# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 -CISCOFW710001_710002_710003_710005_710006 : %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} -# ASA-6-713172 -CISCOFW713172 : Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device -# ASA-4-733100 -CISCOFW733100 : \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count} - - -# ASA-6-305012 -CISCOFW305012 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} duration %{TIME:duration} -# ASA-7-609001 -CISCOFW609001 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? -# ASA-7-609002 -CISCOFW609002 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? duration %{TIME:duration} - -# ASA-5-713041 -CISCOFW713041 : (Group = %{GREEDYDATA:group}, )?IP = %{IP:src_ip}, IKE Initiator: Rekeying Phase %{INT}, Intf %{DATA:src_interface}, IKE Peer %{DATA}\s+local Proxy Address %{IP}, remote Proxy Address %{IP},\s+Crypto map \(%{DATA}\) - -#== End Cisco ASA == \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/resources/patterns/fireeye ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/fireeye b/metron-platform/metron-parsers/src/main/resources/patterns/fireeye deleted file mode 100644 index 5dc99bf..0000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/fireeye +++ /dev/null @@ -1,9 +0,0 @@ -GREEDYDATA .* -POSINT \b(?:[1-9][0-9]*)\b -UID [0-9.]+ -DATA .*? - -FIREEYE_BASE ^<%{POSINT:syslog_pri}>fenotify-%{UID:uid}.alert: %{GREEDYDATA:syslog} -FIREEYE_MAIN <%{POSINT:syslog_pri}>fenotify-%{DATA:uid}.alert: %{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{DATA:meta}\|%{GREEDYDATA:fedata} -#\|(.?)\|(.?)\|(.?)\|(.?)\|%{DATA:type}\|(.?)\|%{GREEDYDATA:fedata} -FIREEYE_SUB ^<%{POSINT:syslog_pri}>fenotify-%{UID:uid}.alert: .?*\|.?*\|.?*\|.?*\|.?*\|%{DATA:type}\|.?*\|%{GREEDYDATA:fedata} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire b/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire deleted file mode 100644 index 672f684..0000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/sourcefire +++ /dev/null @@ -1,30 +0,0 @@ -POSINT \b(?:[1-9][0-9]*)\b -NONNEGINT \b(?:[0-9]+)\b -WORD \b\w+\b -NOTSPACE \S+ -SPACE \s* -DATA .*? -GREEDYDATA .* -QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) -UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} - -# Networking -MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) -CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) -WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) -COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) -IP (?:%{IPV6}|%{IPV4}) -HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b) -HOST %{HOSTNAME} -IPORHOST (?:%{HOSTNAME}|%{IP}) -HOSTPORT %{IPORHOST}:%{POSINT} - -#Sourcefire Logs -protocol \{[a-zA-Z0-9]+\} -ip_src_addr (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) -ip_dst_addr (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) -ip_src_port [0-9]+ -ip_dst_port [0-9]+ -SOURCEFIRE %{GREEDYDATA}%{protocol}\s%{ip_src_addr}\:%{ip_src_port}\s->\s%{ip_dst_addr}\:%{ip_dst_port} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/resources/patterns/squid ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/squid b/metron-platform/metron-parsers/src/main/resources/patterns/squid deleted file mode 100644 index bf6c5b7..0000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/squid +++ /dev/null @@ -1,2 +0,0 @@ -SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})? - http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/resources/patterns/websphere ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/websphere b/metron-platform/metron-parsers/src/main/resources/patterns/websphere deleted file mode 100644 index 546944c..0000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/websphere +++ /dev/null @@ -1,37 +0,0 @@ -# Months - only three-letter code is used -MONTH \b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec?)\b - -# Days - two digit number is used -DAY \d{1,2} - -# Time - two digit hour, minute, and second -TIME \d{2}:\d{2}:\d{2} - -# Timestamp - month, day, and time -TIMESTAMP %{MONTH:UNWANTED}\s+%{DAY:UNWANTED} %{TIME:UNWANTED} - -# Generic word field -WORD \w+ - -# Priority -PRIORITY \d+ - -# Log start - the first part of the log line -LOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname} - -# Security domain -SECURITY_DOMAIN [%{WORD:security_domain}] - -# Log middle - the middle part of the log line -LOGMIDDLE (\[%{WORD:security_domain}\])?\[%{WORD:event_code}\]\[%{WORD:event_type}\]\[%{WORD:severity}\] - -# Define IP address formats -IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) -IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED}) - -# Message - the message body of the log -MESSAGE .* - -# WebSphere - the entire log message -WEBSPHERE %{LOGSTART:UNWANTED} %{LOGMIDDLE:UNWANTED} %{MESSAGE:message} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/main/resources/patterns/yaf ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/resources/patterns/yaf b/metron-platform/metron-parsers/src/main/resources/patterns/yaf deleted file mode 100644 index c664586..0000000 --- a/metron-platform/metron-parsers/src/main/resources/patterns/yaf +++ /dev/null @@ -1,2 +0,0 @@ -YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED} -YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:protocol}\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\|%{SPACE:UNWANTED}%{INT:ip_src_port}\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java deleted file mode 100644 index 1a50dea..0000000 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.parsers; - -import com.google.common.collect.MapDifference; -import com.google.common.collect.Maps; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public abstract class GrokParserTest { - - @Test - public void test() throws IOException, ParseException { - - Map<String, Object> parserConfig = new HashMap<>(); - parserConfig.put("grokPath", getGrokPath()); - parserConfig.put("patternLabel", getGrokPatternLabel()); - parserConfig.put("timestampField", getTimestampField()); - parserConfig.put("dateFormat", getDateFormat()); - parserConfig.put("timeFields", getTimeFields()); - - GrokParser grokParser = new GrokParser(); - grokParser.configure(parserConfig); - grokParser.init(); - - JSONParser jsonParser = new JSONParser(); - Map<String,String> testData = getTestData(); - for( Map.Entry<String,String> e : testData.entrySet() ){ - - JSONObject expected = (JSONObject) jsonParser.parse(e.getValue()); - byte[] rawMessage = e.getKey().getBytes(); - - List<JSONObject> parsedList = grokParser.parse(rawMessage); - Assert.assertEquals(1, parsedList.size()); - compare(expected, parsedList.get(0)); - } - - } - - public boolean compare(JSONObject expected, JSONObject actual) { - MapDifference mapDifferences = Maps.difference(expected, actual); - if (mapDifferences.entriesOnlyOnLeft().size() > 0) Assert.fail("Expected JSON has extra parameters: " + mapDifferences.entriesOnlyOnLeft()); - if (mapDifferences.entriesOnlyOnRight().size() > 0) Assert.fail("Actual JSON has extra parameters: " + mapDifferences.entriesOnlyOnRight()); - Map actualDifferences = new HashMap(); - if (mapDifferences.entriesDiffering().size() > 0) { - Map differences = Collections.unmodifiableMap(mapDifferences.entriesDiffering()); - for (Object key : differences.keySet()) { - Object expectedValueObject = expected.get(key); - Object actualValueObject = actual.get(key); - if (expectedValueObject instanceof Long || expectedValueObject instanceof Integer) { - Long expectedValue = Long.parseLong(expectedValueObject.toString()); - Long actualValue = Long.parseLong(actualValueObject.toString()); - if (!expectedValue.equals(actualValue)) { - actualDifferences.put(key, differences.get(key)); - } - } else { - actualDifferences.put(key, differences.get(key)); - } - } - } - if (actualDifferences.size() > 0) Assert.fail("Expected and Actual JSON values don't match: " + actualDifferences); - return true; - } - - public abstract Map getTestData(); - public abstract String getGrokPath(); - public abstract String getGrokPatternLabel(); - public abstract List<String> getTimeFields(); - public abstract String getDateFormat(); - public abstract String getTimestampField(); -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java deleted file mode 100644 index e060559..0000000 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.parsers; - -import org.adrianwalker.multilinestring.Multiline; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class SampleGrokParserTest extends GrokParserTest { - - /** - * { - * "roct":0, - * "end_reason":"idle", - * "ip_dst_addr":"10.0.2.15", - * "iflags":"AS", - * "rpkt":0, - * "original_string":"1453994987000|2016-01-28 15:29:48| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle", - * "tag":0, - * "risn":0, - * "ip_dst_port":39468, - * "ruflags":0, - * "app":0, - * "protocol":6 - * ,"isn":"22efa001", - * "uflags":0,"duration":"0.000", - * "oct":44, - * "ip_src_port":80, - * "end_time":1453994988000, - * "start_time":1453994987000 - * "timestamp":1453994987000, - * "riflags":0, - * "rtt":"0.000", - * "rtag":0, - * "pkt":1, - * "ip_src_addr":"216.21.170.221" - * } - */ - @Multiline - public String result; - - - @Override - public Map getTestData() { - - Map testData = new HashMap<String,String>(); - String input = "1453994987000|2016-01-28 15:29:48| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle"; - testData.put(input,result); - return testData; - - } - - @Override - public String getGrokPath() { - return "../metron-integration-test/src/main/sample/patterns/test"; - } - - @Override - public String getGrokPatternLabel() { - return "YAF_DELIMITED"; - } - - @Override - public List<String> getTimeFields() { - return new ArrayList<String>() {{ - add("end_time"); - }}; - } - - @Override - public String getDateFormat() { - return "yyyy-MM-dd HH:mm:ss"; - } - - @Override - public String getTimestampField() { - return "start_time"; - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java deleted file mode 100644 index 1981bb8..0000000 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.metron.parsers; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.log4j.Level; -import org.apache.metron.common.Constants; -import org.apache.metron.parsers.snort.BasicSnortParser; -import org.apache.metron.test.utils.UnitTestHelper; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.time.ZoneId; -import java.util.HashMap; -import java.util.Map; -import java.util.TimeZone; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.startsWith; -import static org.junit.Assert.assertThat; - -public class SnortParserTest { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - /** - 01/27/16-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments, exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,, - **/ - @Multiline - public static String goodMessage; - - // we will test timestamp conversion/parsing separately - @Test - public void testGoodMessage() { - BasicSnortParser parser = new BasicSnortParser(); - parser.configure(new HashMap()); - Map out = parser.parse(goodMessage.getBytes()).get(0); - Assert.assertEquals(out.get("msg"), "Consecutive TCP small segments, exceeding threshold"); - Assert.assertEquals(out.get("sig_rev"), "1"); - Assert.assertEquals(out.get("ip_dst_addr"), "10.0.2.15"); - Assert.assertEquals(out.get("ip_dst_port"), "22"); - Assert.assertEquals(out.get("ethsrc"), "52:54:00:12:35:02"); - Assert.assertEquals(out.get("tcpseq"), "0x9AFF3D7"); - Assert.assertEquals(out.get("dgmlen"), "64"); - Assert.assertEquals(out.get("icmpid"), ""); - Assert.assertEquals(out.get("tcplen"), ""); - Assert.assertEquals(out.get("tcpwindow"), "0xFFFF"); - Assert.assertEquals(out.get("icmpseq").toString().trim(), ""); - Assert.assertEquals(out.get("tcpack"), "0xC8761D52"); - Assert.assertEquals(out.get("icmpcode"), ""); - Assert.assertEquals(out.get("tos"), "0"); - Assert.assertEquals(out.get("id"), "59677"); - Assert.assertEquals(out.get("ethdst"), "08:00:27:7F:93:2D"); - Assert.assertEquals(out.get("ip_src_addr"), "10.0.2.2"); - Assert.assertEquals(out.get("ttl"), "64"); - Assert.assertEquals(out.get("ethlen"), "0x4E"); - Assert.assertEquals(out.get("iplen"), "65536"); - Assert.assertEquals(out.get("icmptype"), ""); - Assert.assertEquals(out.get("protocol"), "TCP"); - Assert.assertEquals(out.get("ip_src_port"), "56642"); - Assert.assertEquals(out.get("tcpflags"), "***AP***"); - Assert.assertEquals(out.get("sig_id"), "12"); - Assert.assertEquals(out.get("sig_generator"), "129"); - Assert.assertEquals(out.get("is_alert"), "true"); - } - - @Test - public void testBadMessage() { - thrown.expect(IllegalStateException.class); - BasicSnortParser parser = new BasicSnortParser(); - parser.init(); - UnitTestHelper.setLog4jLevel(BasicSnortParser.class, Level.FATAL); - parser.parse("foo bar".getBytes()); - UnitTestHelper.setLog4jLevel(BasicSnortParser.class, Level.ERROR); - } - - @Test - public void parses_timestamp_as_local_zone_by_default() { - // test needs to be able to run from context of multiple timezones so we will set the default manually - TimeZone defaultTimeZone = TimeZone.getDefault(); - try { - TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("America/New_York"))); - BasicSnortParser parser = new BasicSnortParser(); - parser.configure(new HashMap()); - Map out = parser.parse(goodMessage.getBytes()).get(0); - Assert.assertEquals(out.get("timestamp"), 1453928464877L); - } finally { - // make sure we don't mess with other tests - TimeZone.setDefault(defaultTimeZone); - } - } - - /** - 01/27/2016-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments, exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,, - **/ - @Multiline - public static String dateFormattedMessage; - - @Test - public void uses_configuration_to_parse() { - Map<String, Object> parserConfig = new HashMap<>(); - parserConfig.put("dateFormat", "MM/dd/yyyy-HH:mm:ss.SSSSSS"); - parserConfig.put("timeZone", "America/New_York"); - BasicSnortParser parser = new BasicSnortParser(); - parser.configure(parserConfig); - Map result = parser.parse(dateFormattedMessage.getBytes()).get(0); - assertThat("timestamp should match", result.get(Constants.Fields.TIMESTAMP.getName()), equalTo(1453928464877L)); - } - - @Test - public void throws_exception_on_bad_config_timezone() { - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage(startsWith("Unable to find ZoneId")); - Map<String, Object> parserConfig = new HashMap<>(); - parserConfig.put("dateFormat", "MM/dd/yyyy-HH:mm:ss.SSSSSS"); - parserConfig.put("timeZone", "blahblahBADZONE"); - BasicSnortParser parser = new BasicSnortParser(); - parser.configure(parserConfig); - } - - @Test - public void throws_exception_on_bad_config_date_format() { - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage(startsWith("Unknown pattern letter:")); - Map<String, Object> parserConfig = new HashMap<>(); - parserConfig.put("dateFormat", "BADFORMAT"); - BasicSnortParser parser = new BasicSnortParser(); - parser.configure(parserConfig); - } - -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java deleted file mode 100644 index 93c8276..0000000 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.parsers; - -import org.adrianwalker.multilinestring.Multiline; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class SquidParserTest extends GrokParserTest { - - /** - * { - * "elapsed":161, - * "code":200, - * "ip_dst_addr":"199.27.79.73", - * "original_string":"1461576382.642 161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html", - * "method":"GET", - * "bytes":103701, - * "action":"TCP_MISS", - * "url":"http://www.cnn.com/";, - * "ip_src_addr":"127.0.0.1", - * "timestamp":1461576382642 - * } - */ - @Multiline - private String result1; - - /** - * { - * "elapsed":0, - * "code":403, - * "original_string":"1469539185.270 0 139.196.181.68 TCP_DENIED/403 3617 CONNECT search.yahoo.com:443 - NONE/- text/html", - * "method":"CONNECT", - * "bytes":3617, - * "action":"TCP_DENIED", - * "url":"search.yahoo.com:443", - * "ip_src_addr":"139.196.181.68", - * "timestamp":1469539185270, - * "ip_dst_addr": null - * } - */ - @Multiline - private String result2; - - @Override - public Map<String,String> getTestData() { - - String input1 = "1461576382.642 161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html"; - String input2 = "1469539185.270 0 139.196.181.68 TCP_DENIED/403 3617 CONNECT search.yahoo.com:443 - NONE/- text/html"; - - HashMap testData = new HashMap<String,String>(); - testData.put(input1,result1); - testData.put(input2,result2); - return testData; - - } - - - @Override - public String getGrokPath() { - return "../metron-parsers/src/main/resources/patterns/squid"; - } - - @Override - public String getGrokPatternLabel() { - return "SQUID_DELIMITED"; - } - - @Override - public List<String> getTimeFields() { - return new ArrayList<>(); - } - - @Override - public String getDateFormat() { - return null; - } - - @Override - public String getTimestampField() { - return "timestamp"; - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java deleted file mode 100644 index 8dd75a0..0000000 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.parsers; - -import org.adrianwalker.multilinestring.Multiline; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class YafParserTest extends GrokParserTest { - - /** - * { - "iflags": "AS", - "uflags": 0, - "isn": "22efa001", - "ip_dst_addr": "10.0.2.15", - "ip_dst_port": 39468, - "duration": "0.000", - "rpkt": 0, - "original_string": "2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle", - "pkt": 1, - "ruflags": 0, - "roct": 0, - "ip_src_addr": "216.21.170.221", - "tag": 0, - "rtag": 0, - "ip_src_port": 80, - "timestamp": 1453994988512, - "app": 0, - "oct": 44, - "end_reason": "idle", - "risn": 0, - "end_time": 1453994988512, - "start_time": 1453994988512, - "riflags": 0, - "rtt": "0.000", - "protocol": 6 - } - */ - @Multiline - public String result; - - @Override - public Map getTestData() { - - Map testData = new HashMap<String,String>(); - String input = "2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle"; - testData.put(input,result); - return testData; - - } - - @Override - public String getGrokPath() { - return "../metron-parsers/src/main/resources/patterns/yaf"; - } - - - - @Override - public String getGrokPatternLabel() { - return "YAF_DELIMITED"; - } - - @Override - public List<String> getTimeFields() { - return new ArrayList<String>() {{ - add("start_time"); - add("end_time"); - }}; - } - - @Override - public String getDateFormat() { - return "yyyy-MM-dd HH:mm:ss.S"; - } - - @Override - public String getTimestampField() { - return "start_time"; - } -}
