This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new bfe662d METRON-2128 LEEF config file is missing in RPM spec file
(simonellistonball via mmiklavc) closes apache/metron#1419
bfe662d is described below
commit bfe662d6e7b07f8340067d7aaeb976068617b6b1
Author: simonellistonball <si...@simonellistonball.com>
AuthorDate: Tue May 28 10:29:39 2019 -0600
METRON-2128 LEEF config file is missing in RPM spec file (simonellistonball
via mmiklavc) closes apache/metron#1419
---
.../packaging/docker/rpm-docker/SPECS/metron.spec | 1 +
.../src/main/config/zookeeper/parsers/leef.json | 0
.../org/apache/metron/parsers/leef/LEEFParser.java | 458 ++++++++++-----------
.../apache/metron/parsers/leef/LEEFParserTest.java | 396 +++++++++---------
4 files changed, 428 insertions(+), 427 deletions(-)
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index dc40967..8b68b6f 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -192,6 +192,7 @@ This package installs the Metron Bundled Parser files
%{metron_home}/config/zookeeper/parsers/websphere.json
%{metron_home}/config/zookeeper/parsers/yaf.json
%{metron_home}/config/zookeeper/parsers/asa.json
+%{metron_home}/config/zookeeper/parsers/leef.json
%{metron_home}/patterns/asa
%{metron_home}/patterns/fireeye
%{metron_home}/patterns/sourcefire
diff --git
a/metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/leef.json
b/metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/leef.json
similarity index 100%
rename from
metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/leef.json
rename to
metron-platform/metron-parsing/metron-parsers/src/main/config/zookeeper/parsers/leef.json
diff --git
a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
index db19ebd..ea09714 100644
---
a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
+++
b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
@@ -53,235 +53,235 @@ import org.slf4j.LoggerFactory;
*
*/
public class LEEFParser extends BasicParser {
- private static final long serialVersionUID = 1L;
-
- public enum HeaderFields {
- DEVICE_VENDOR("DeviceVendor"),
- DEVICE_PRODUCT("DeviceProduct"),
- DEVICE_VERSION("DeviceVersion"),
- DEVICE_EVENT("DeviceEvent"),
- DELIMITER("Delimiter"),
- VERSION("Version")
- ;
-
- private String name;
-
- HeaderFields(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
- }
-
- // Field name for custom device time in LEEF
- private static final String DEV_TIME = "devTime";
- private static final String DEV_TIME_FORMAT = "devTimeFormat";
-
- protected static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*";
- private static final Charset UTF_8 = StandardCharsets.UTF_8;
-
- private Pattern pattern;
-
- public void init() {
-
- // LEEF Headers: Version|Device Vendor|Device Product|Device
Version|Device Event|Delimiter
- String syslogTime =
"(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b
+(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
(?!<[0-9])(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))(?![0-9])?";
- String syslogTime5424 =
"(?:\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2}))";
- String syslogPriority = "<(?:[0-9]+)>";
- String syslogHost = "[a-z0-9\\.\\\\-_]+";
-
- StringBuilder sb = new StringBuilder("");
- sb.append("(?<syslogPriority>");
- sb.append(syslogPriority);
- sb.append(")?");
- sb.append("(?<syslogTime>");
- sb.append(syslogTime);
- sb.append("|");
- sb.append(syslogTime5424);
- sb.append(")?");
-
- sb.append("(?<syslogHost>");
- sb.append(syslogHost);
- sb.append(")?");
-
- sb.append(".*");
-
- sb.append("LEEF:(?<");
- sb.append(HeaderFields.VERSION.getName());
- sb.append(">1.0|2.0|0)?\\|");
-
- headerBlock(HeaderFields.DEVICE_VENDOR.getName(), sb);
- sb.append("\\|");
- headerBlock(HeaderFields.DEVICE_PRODUCT.getName(), sb);
- sb.append("\\|");
- headerBlock(HeaderFields.DEVICE_VERSION.getName(), sb);
- sb.append("\\|");
- headerBlock(HeaderFields.DEVICE_EVENT.getName(), sb);
- sb.append("\\|");
-
- // add optional delimiter header (only applicable for LEEF 2.0)
- sb.append("(");
- headerBlock(HeaderFields.DELIMITER.getName(), sb);
- sb.append("\\|");
- sb.append(")?");
-
- // extension capture:
- sb.append(" ?(?<extensions>.*)");
- pattern = Pattern.compile(sb.toString());
- }
-
- public Optional<MessageParserResult<JSONObject>>
parseOptionalResult(byte[] rawMessage) {
- List<JSONObject> messages = new ArrayList<>();
- Map<Object,Throwable> errors = new HashMap<>();
- String originalMessage = null;
-
- try (BufferedReader reader = new BufferedReader(new
StringReader(new String(rawMessage, StandardCharsets.UTF_8)))) {
- while ((originalMessage = reader.readLine()) != null) {
- Matcher matcher =
pattern.matcher(originalMessage);
- while (matcher.find()) {
- JSONObject obj = new JSONObject();
- if (!matcher.matches()) {
- break;
- }
- LOG.debug("Found %d groups",
matcher.groupCount());
-
obj.put(HeaderFields.DEVICE_VENDOR.getName(),
-
matcher.group(HeaderFields.DEVICE_VENDOR.getName()));
-
obj.put(HeaderFields.DEVICE_PRODUCT.getName(),
-
matcher.group(HeaderFields.DEVICE_PRODUCT.getName()));
-
obj.put(HeaderFields.DEVICE_VERSION.getName(),
-
matcher.group(HeaderFields.DEVICE_VERSION.getName()));
-
obj.put(HeaderFields.DEVICE_EVENT.getName(),
-
matcher.group(HeaderFields.DEVICE_EVENT.getName()));
-
- String ext =
matcher.group("extensions");
-
- // In LEEF 2.0 the delimiter can be
specified
- String version =
matcher.group(HeaderFields.VERSION.getName());
- if (version.equals("2.0")) {
- String delimiter =
matcher.group(HeaderFields.DELIMITER.getName());
- if (delimiter == null ||
delimiter.length() == 0) {
- delimiter = "\\t";
- }
- delimiter = "(?<!\\\\)[" +
delimiter.replace("^", "\\^").replace("\t", "\\t") + "]";
-
- String[] kvs =
ext.split(delimiter);
- for (String kv : kvs) {
- String[] a =
kv.split("=");
- obj.put(a[0], a[1]);
- }
- } else if (version.equals("1.0") ||
version.isEmpty()) {
- String delimiter = "\t";
- String[] kvs =
ext.split(delimiter);
- for (String kv : kvs) {
- String[] a =
kv.split("=");
- obj.put(a[0], a[1]);
- }
- } else {
- // Found in the wild examples
using CEF rules, which need to handle the processing per the CEFParser
- // Note that technically LEEF
does not support the CEF approach to numbered custom variables.
- // We however do here, due to
some found in the wild exceptions to the standard.
- CEFParser.parseExtensions(ext,
obj);
- }
-
- // Rename standard CEF fields to comply
with Metron standards
- obj = mutate(obj, "dst",
Fields.DST_ADDR.getName());
- obj = mutate(obj, "dstPort",
Fields.DST_PORT.getName());
- obj = convertToInt(obj,
Fields.DST_PORT.getName());
-
- obj = mutate(obj, "src",
Fields.SRC_ADDR.getName());
- obj = mutate(obj, "srcPort",
Fields.SRC_PORT.getName());
- obj = convertToInt(obj,
Fields.SRC_PORT.getName());
-
- obj.put(Fields.ORIGINAL.getName(),
originalMessage);
-
- // add the host
- String host =
matcher.group("syslogHost");
- if (!(host == null || host.isEmpty())) {
- obj.put("host", host);
- }
-
- // apply timestamp from message if
present, using devTime, syslog
- // timestamp,
- // default to current system time
- //devTime, devTimeFormat, calLanguage,
calCountryOrRegion
- if (obj.containsKey(DEV_TIME)) {
- String devTime = (String)
obj.get(DEV_TIME);
- try {
- // DateFormats allowed
in LEEF
- // epoch
- // MMM dd yyyy HH:mm:ss
- // MMM dd yyyy
HH:mm:ss.SSS
- // MMM dd yyyy
HH:mm:ss.SSS zzz
- // custom in
devTimeFormat field
- final String
devTimeFormat = (String) obj.get(DEV_TIME_FORMAT);
-
- List<SimpleDateFormat>
formats = (obj.containsKey(DEV_TIME_FORMAT)) ?
- new
ArrayList<SimpleDateFormat>() {{
-
add(new SimpleDateFormat(devTimeFormat));
- }} :
-
DateUtils.DATE_FORMATS_LEEF;
-
obj.put(Fields.TIMESTAMP.getName(), DateUtils.parseMultiformat(devTime,
formats));
- } catch
(java.text.ParseException e) {
-
errors.put(originalMessage,
- new
IllegalStateException("devTime field present in LEEF but cannot be parsed",
-
e));
- continue;
- }
- } else {
- String logTimestamp =
matcher.group("syslogTime");
- if (!(logTimestamp == null ||
logTimestamp.isEmpty())) {
- try {
-
obj.put(Fields.TIMESTAMP.getName(),
-
SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC()));
- } catch (ParseException
e) {
-
errors.put(originalMessage,
-
new IllegalStateException("Cannot parse syslog timestamp", e));
- continue;
- }
- } else {
-
obj.put(Fields.TIMESTAMP.getName(), System.currentTimeMillis());
- }
- }
- messages.add(obj);
- }
- }
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- Exception innerException = new
IllegalStateException("LEEF parser Error: "
- + e.getMessage()
- + " on "
- + originalMessage, e);
- return Optional.of(new
DefaultMessageParserResult<>(innerException));
- }
- return Optional.of(new DefaultMessageParserResult<>(messages,
errors));
- }
-
- @SuppressWarnings("unchecked")
- private JSONObject convertToInt(JSONObject obj, String key) {
- if (obj.containsKey(key)) {
- obj.put(key, Integer.valueOf((String) obj.get(key)));
- }
- return obj;
- }
-
- private void headerBlock(String name, StringBuilder sb) {
-
sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")");
- }
-
- @Override
- public void configure(Map<String, Object> config) {
- }
-
- @SuppressWarnings("unchecked")
- private JSONObject mutate(JSONObject json, String oldKey, String
newKey) {
- if (json.containsKey(oldKey)) {
- json.put(newKey, json.remove(oldKey));
- }
- return json;
- }
+ private static final long serialVersionUID = 1L;
+
+ public enum HeaderFields {
+ DEVICE_VENDOR("DeviceVendor"),
+ DEVICE_PRODUCT("DeviceProduct"),
+ DEVICE_VERSION("DeviceVersion"),
+ DEVICE_EVENT("DeviceEvent"),
+ DELIMITER("Delimiter"),
+ VERSION("Version")
+ ;
+
+ private String name;
+
+ HeaderFields(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ // Field name for custom device time in LEEF
+ private static final String DEV_TIME = "devTime";
+ private static final String DEV_TIME_FORMAT = "devTimeFormat";
+
+ protected static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*";
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
+
+ private Pattern pattern;
+
+ public void init() {
+
+ // LEEF Headers: Version|Device Vendor|Device Product|Device
Version|Device Event|Delimiter
+ String syslogTime =
"(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b
+(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
(?!<[0-9])(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))(?![0-9])?";
+ String syslogTime5424 =
"(?:\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2}))";
+ String syslogPriority = "<(?:[0-9]+)>";
+ String syslogHost = "[a-z0-9\\.\\\\-_]+";
+
+ StringBuilder sb = new StringBuilder("");
+ sb.append("(?<syslogPriority>");
+ sb.append(syslogPriority);
+ sb.append(")?");
+ sb.append("(?<syslogTime>");
+ sb.append(syslogTime);
+ sb.append("|");
+ sb.append(syslogTime5424);
+ sb.append(")?");
+
+ sb.append("(?<syslogHost>");
+ sb.append(syslogHost);
+ sb.append(")?");
+
+ sb.append(".*");
+
+ sb.append("LEEF:(?<");
+ sb.append(HeaderFields.VERSION.getName());
+ sb.append(">1.0|2.0|0)?\\|");
+
+ headerBlock(HeaderFields.DEVICE_VENDOR.getName(), sb);
+ sb.append("\\|");
+ headerBlock(HeaderFields.DEVICE_PRODUCT.getName(), sb);
+ sb.append("\\|");
+ headerBlock(HeaderFields.DEVICE_VERSION.getName(), sb);
+ sb.append("\\|");
+ headerBlock(HeaderFields.DEVICE_EVENT.getName(), sb);
+ sb.append("\\|");
+
+ // add optional delimiter header (only applicable for LEEF 2.0)
+ sb.append("(");
+ headerBlock(HeaderFields.DELIMITER.getName(), sb);
+ sb.append("\\|");
+ sb.append(")?");
+
+ // extension capture:
+ sb.append(" ?(?<extensions>.*)");
+ pattern = Pattern.compile(sb.toString());
+ }
+
+ public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[]
rawMessage) {
+ List<JSONObject> messages = new ArrayList<>();
+ Map<Object,Throwable> errors = new HashMap<>();
+ String originalMessage = null;
+
+ try (BufferedReader reader = new BufferedReader(new StringReader(new
String(rawMessage, StandardCharsets.UTF_8)))) {
+ while ((originalMessage = reader.readLine()) != null) {
+ Matcher matcher = pattern.matcher(originalMessage);
+ while (matcher.find()) {
+ JSONObject obj = new JSONObject();
+ if (!matcher.matches()) {
+ break;
+ }
+ LOG.debug("Found %d groups", matcher.groupCount());
+ obj.put(HeaderFields.DEVICE_VENDOR.getName(),
+ matcher.group(HeaderFields.DEVICE_VENDOR.getName()));
+ obj.put(HeaderFields.DEVICE_PRODUCT.getName(),
+ matcher.group(HeaderFields.DEVICE_PRODUCT.getName()));
+ obj.put(HeaderFields.DEVICE_VERSION.getName(),
+ matcher.group(HeaderFields.DEVICE_VERSION.getName()));
+ obj.put(HeaderFields.DEVICE_EVENT.getName(),
+ matcher.group(HeaderFields.DEVICE_EVENT.getName()));
+
+ String ext = matcher.group("extensions");
+
+ // In LEEF 2.0 the delimiter can be specified
+ String version = matcher.group(HeaderFields.VERSION.getName());
+ if (version.equals("2.0")) {
+ String delimiter = matcher.group(HeaderFields.DELIMITER.getName());
+ if (delimiter == null || delimiter.length() == 0) {
+ delimiter = "\\t";
+ }
+ delimiter = "(?<!\\\\)[" + delimiter.replace("^",
"\\^").replace("\t", "\\t") + "]";
+
+ String[] kvs = ext.split(delimiter);
+ for (String kv : kvs) {
+ String[] a = kv.split("=");
+ obj.put(a[0], a[1]);
+ }
+ } else if (version.equals("1.0") || version.isEmpty()) {
+ String delimiter = "\t";
+ String[] kvs = ext.split(delimiter);
+ for (String kv : kvs) {
+ String[] a = kv.split("=");
+ obj.put(a[0], a[1]);
+ }
+ } else {
+ // Found in the wild examples using CEF rules, which need to
handle the processing per the CEFParser
+ // Note that technically LEEF does not support the CEF approach to
numbered custom variables.
+ // We however do here, due to some found in the wild exceptions to
the standard.
+ CEFParser.parseExtensions(ext, obj);
+ }
+
+ // Rename standard CEF fields to comply with Metron standards
+ obj = mutate(obj, "dst", Fields.DST_ADDR.getName());
+ obj = mutate(obj, "dstPort", Fields.DST_PORT.getName());
+ obj = convertToInt(obj, Fields.DST_PORT.getName());
+
+ obj = mutate(obj, "src", Fields.SRC_ADDR.getName());
+ obj = mutate(obj, "srcPort", Fields.SRC_PORT.getName());
+ obj = convertToInt(obj, Fields.SRC_PORT.getName());
+
+ obj.put(Fields.ORIGINAL.getName(), originalMessage);
+
+ // add the host
+ String host = matcher.group("syslogHost");
+ if (!(host == null || host.isEmpty())) {
+ obj.put("host", host);
+ }
+
+ // apply timestamp from message if present, using devTime, syslog
+ // timestamp,
+ // default to current system time
+ //devTime, devTimeFormat, calLanguage, calCountryOrRegion
+ if (obj.containsKey(DEV_TIME)) {
+ String devTime = (String) obj.get(DEV_TIME);
+ try {
+ // DateFormats allowed in LEEF
+ // epoch
+ // MMM dd yyyy HH:mm:ss
+ // MMM dd yyyy HH:mm:ss.SSS
+ // MMM dd yyyy HH:mm:ss.SSS zzz
+ // custom in devTimeFormat field
+ final String devTimeFormat = (String) obj.get(DEV_TIME_FORMAT);
+
+ List<SimpleDateFormat> formats =
(obj.containsKey(DEV_TIME_FORMAT)) ?
+ new ArrayList<SimpleDateFormat>() {{
+ add(new SimpleDateFormat(devTimeFormat));
+ }} :
+ DateUtils.DATE_FORMATS_LEEF;
+ obj.put(Fields.TIMESTAMP.getName(),
DateUtils.parseMultiformat(devTime, formats));
+ } catch (java.text.ParseException e) {
+ errors.put(originalMessage,
+ new IllegalStateException("devTime field present in LEEF but
cannot be parsed",
+ e));
+ continue;
+ }
+ } else {
+ String logTimestamp = matcher.group("syslogTime");
+ if (!(logTimestamp == null || logTimestamp.isEmpty())) {
+ try {
+ obj.put(Fields.TIMESTAMP.getName(),
+ SyslogUtils.parseTimestampToEpochMillis(logTimestamp,
Clock.systemUTC()));
+ } catch (ParseException e) {
+ errors.put(originalMessage,
+ new IllegalStateException("Cannot parse syslog timestamp",
e));
+ continue;
+ }
+ } else {
+ obj.put(Fields.TIMESTAMP.getName(), System.currentTimeMillis());
+ }
+ }
+ messages.add(obj);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ Exception innerException = new IllegalStateException("LEEF parser Error:
"
+ + e.getMessage()
+ + " on "
+ + originalMessage, e);
+ return Optional.of(new DefaultMessageParserResult<>(innerException));
+ }
+ return Optional.of(new DefaultMessageParserResult<>(messages, errors));
+ }
+
+ @SuppressWarnings("unchecked")
+ private JSONObject convertToInt(JSONObject obj, String key) {
+ if (obj.containsKey(key)) {
+ obj.put(key, Integer.valueOf((String) obj.get(key)));
+ }
+ return obj;
+ }
+
+ private void headerBlock(String name, StringBuilder sb) {
+
sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")");
+ }
+
+ @Override
+ public void configure(Map<String, Object> config) {
+ }
+
+ @SuppressWarnings("unchecked")
+ private JSONObject mutate(JSONObject json, String oldKey, String newKey) {
+ if (json.containsKey(oldKey)) {
+ json.put(newKey, json.remove(oldKey));
+ }
+ return json;
+ }
}
diff --git
a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
index 704d748..2db3224 100644
---
a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
+++
b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
@@ -42,202 +42,202 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
public class LEEFParserTest {
- private static final Charset UTF_8 = StandardCharsets.UTF_8;
- private LEEFParser parser;
-
- @Before
- public void setUp() {
- parser = new LEEFParser();
- parser.init();
- }
-
- @Test
- public void testInvalid() {
- List<JSONObject> obj = parse("test test test nonsense\n");
- Assert.assertEquals(0, obj.size());
- }
-
- @Test
- public void testTimestampPriority() throws java.text.ParseException {
- long correctTime = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400")
- .getTime();
-
- SimpleDateFormat sdf = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz");
-
- for (JSONObject obj : parse(
-
"LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
- Assert.assertEquals(new Date(correctTime), new
Date((long) obj.get(Fields.TIMESTAMP.getName())));
- Assert.assertEquals(correctTime,
obj.get(Fields.TIMESTAMP.getName()));
- }
- for (JSONObject obj : parse(
- "2016-06-01T09:29:11.356-04:00 host
LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
- Assert.assertEquals(new Date(correctTime), new
Date((long) obj.get(Fields.TIMESTAMP.getName())));
- Assert.assertEquals(correctTime,
obj.get(Fields.TIMESTAMP.getName()));
- }
- for (JSONObject obj : parse(
- "2016-05-01T09:29:11.356-04:00 host
LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
- Assert.assertEquals(new Date(correctTime), new
Date((long) obj.get(Fields.TIMESTAMP.getName())));
- Assert.assertEquals(correctTime,
obj.get(Fields.TIMESTAMP.getName()));
- }
- for (JSONObject obj : parse(
-
"LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
-
Assert.assertNotNull(obj.get(Fields.TIMESTAMP.getName()));
- }
-
- }
-
- private void runMissingYear(Calendar expected, Calendar input) {
- SimpleDateFormat sdf = new SimpleDateFormat("MMM dd
HH:mm:ss.SSS");
- for (JSONObject obj :
parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|\t|src=10.0.0.1\tdevTime="
- + sdf.format(input.getTime()) +
- "\tdevTimeFormat=MMM dd HH:mm:ss.SSS" +
- "\tdst=2.1.2.2\tspt=1232")) {
- Assert.assertEquals(expected.getTime(), new Date((long)
obj.get(Fields.TIMESTAMP.getName())));
- Assert.assertEquals(expected.getTimeInMillis(),
obj.get(Fields.TIMESTAMP.getName()));
- }
- }
-
- @Test
- public void testMissingYearFromDate() throws java.text.ParseException {
- Calendar current = Calendar.getInstance();
- Calendar correct = Calendar.getInstance();
-
- correct.setTimeInMillis(current.getTimeInMillis());
-
- runMissingYear(correct, current);
- }
-
- @Test
- public void testFourDayFutureBecomesPast() {
- Calendar current = Calendar.getInstance();
- Calendar correct = Calendar.getInstance();
-
- current.add(Calendar.DAY_OF_MONTH, 5);
- // correct.setTime(current.getTime());
- correct.setTimeInMillis(current.getTimeInMillis());
- correct.add(Calendar.YEAR, -1);
-
- runMissingYear(correct, current);
- }
-
- /**
- * Sample from
https://docs.imperva.com/bundle/cloud-application-security/page/more/example-logs.htm#logEx2
- */
- @Test
- public void testLEEF_CEFlikeSample() {
- List<JSONObject> parse =
parse("LEEF:0|Incapsula|SIEMintegration|0|SQL Injection|
fileId=3412364560000000008 sourceServiceName=test56111115.incaptest.co
siteid=1333546 suid=300656 requestClientApplication=Mozilla/5.0 (Windows NT
6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0 popName=mia cs2=true
cs2Label=Javascript Support cs3=true cs3Label=CO Support cs1=NA cs1Label=Cap
Support cs4=936e64c2-bdd1-4719-9bd0-2d882a72f30d cs4Label=VID
cs5=bab1712be85b00ab21d20bf0d7b5db82701f27f53fbac19a [...]
- JSONObject obj = parse.get(0);
- Assert.assertNotNull(obj);
- Assert.assertEquals("3412364560000000008", obj.get("fileId"));
- Assert.assertEquals("Mozilla/5.0 (Windows NT 6.1; WOW64;
rv:38.0) Gecko/20100101 Firefox/38.0", obj.get("requestClientApplication"));
- Assert.assertTrue(obj.containsKey("longitude"));
- Assert.assertFalse(obj.containsKey("cs8"));
- Assert.assertFalse(obj.containsKey("cs8Label"));
- }
-
- @Test
- public void testLEEFParserSample() throws Exception {
- runTest("sample",
Resources.readLines(Resources.getResource(getClass(), "sample.leef"), UTF_8),
-
Resources.toString(Resources.getResource(getClass(), "sample.schema"), UTF_8));
- }
-
- private void runTest(String name, List<String> lines, String schema)
throws Exception {
- runTest(name, lines, schema, "");
- }
-
- private void runTest(String name, List<String> lines, String schema,
String targetJson) throws Exception {
- for (String inputString : lines) {
- JSONObject parsed = parse(inputString).get(0);
- Assert.assertNotNull(parsed);
-
Assert.assertNotNull(parsed.get(Fields.TIMESTAMP.getName()));
- Assert.assertTrue((long)
parsed.get(Fields.TIMESTAMP.getName()) > 0);
-
- JSONParser parser = new JSONParser();
-
- Map<?, ?> json = null;
- json = (Map<?, ?>) parser.parse(parsed.toJSONString());
- Assert.assertEquals(true, validateJsonData(schema,
json.toString()));
- }
- }
-
- private void assertSimpleSample(List<JSONObject> parse) {
- JSONObject obj = parse.get(0);
- Assert.assertNotNull(obj);
- Assert.assertTrue(obj.containsKey(Fields.SRC_ADDR.getName()));
- Assert.assertEquals("192.0.2.0",
obj.get(Fields.SRC_ADDR.getName()));
- }
-
- @Test
- public void testLEEF_1_0_versionIncluded() {
- List<JSONObject> parse =
parse("LEEF:1.0|Microsoft|MSExchange|4.0 SP1|15345|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
- assertSimpleSample(parse);
- }
-
- @Test
- public void testLEEF_2_0() {
- List<JSONObject> parse =
parse("LEEF:2.0|Vendor|Product|Version|EventID|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
- assertSimpleSample(parse);
- }
-
- @Test
- public void testLEEF_2_0_delimiterSpecified() {
- List<JSONObject> parse =
parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black");
- assertSimpleSample(parse);
- }
-
- @Test
- public void testLEEF_2_0_delimiterUsedIncorrectly() {
- List<JSONObject> parse =
parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
- assertFalse(parse.get(0).containsKey(Fields.DST_ADDR));
- }
-
- @Test
- public void testLEEFMultiLine() {
- List<JSONObject> parse =
parse("LEEF:2.0|Vendor|Product|Version|EventID|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=line1"
+
- "\nLEEF:2.0|Vendor|Product|Version|EventID|
src=192.0.2.1\tdst=172.50.123.2\tsev=6\tcat=anomaly\tsrcPort=82\tdstPort=22\tusrName=line2");
- assertSimpleSample(parse);
- assertEquals(2, parse.size());
- }
-
-
- @Test
- public void testLEEFcustomdevTimeFormat() {
- String customFormat = "yyyy-MM-dd HH:mm:ss.SSS zzz";
- Date customDate = new Date();
- DateFormat customFormatter = new SimpleDateFormat(customFormat);
-
- List<JSONObject> parse =
parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black^devTime="
+ customFormatter.format(customDate) + "^devTimeFormat=" + customFormat);
- JSONObject obj = parse.get(0);
- assertEquals(obj.get(Fields.TIMESTAMP.getName()),
customDate.getTime());
- }
-
- @Test
- public void testLEEFdevTimeWithNoCustomFormat() {
- String standardFormat = "MMM dd yyyy HH:mm:ss.SSS zzz";
- Date customDate = new Date();
- long expected = customDate.getTime();
- DateFormat customFormatter = new
SimpleDateFormat(standardFormat);
-
- List<JSONObject> parse =
parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black^devTime="
+ customFormatter.format(customDate));
- JSONObject obj = parse.get(0);
- assertEquals(obj.get(Fields.TIMESTAMP.getName()), expected);
- }
-
- protected boolean validateJsonData(final String jsonSchema, final
String jsonData) throws Exception {
- final JsonNode d = JsonLoader.fromString(jsonData);
- final JsonNode s = JsonLoader.fromString(jsonSchema);
-
- final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
- JsonValidator v = factory.getValidator();
-
- ProcessingReport report = v.validate(s, d);
-
- return report.toString().contains("success");
- }
-
- private List<JSONObject> parse(String string) {
- Optional<MessageParserResult<JSONObject>> parse =
parser.parseOptionalResult(string.getBytes(UTF_8));
- Assert.assertTrue(parse.isPresent());
- return parse.get().getMessages();
- }
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
+ private LEEFParser parser;
+
+ @Before
+ public void setUp() {
+ parser = new LEEFParser();
+ parser.init();
+ }
+
+ @Test
+ public void testInvalid() {
+ List<JSONObject> obj = parse("test test test nonsense\n");
+ Assert.assertEquals(0, obj.size());
+ }
+
+ @Test
+ public void testTimestampPriority() throws java.text.ParseException {
+ long correctTime = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400")
+ .getTime();
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz");
+
+ for (JSONObject obj : parse(
+ "LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(new Date(correctTime), new Date((long)
obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
+ }
+ for (JSONObject obj : parse(
+ "2016-06-01T09:29:11.356-04:00 host
LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(new Date(correctTime), new Date((long)
obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
+ }
+ for (JSONObject obj : parse(
+ "2016-05-01T09:29:11.356-04:00 host
LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(new Date(correctTime), new Date((long)
obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
+ }
+ for (JSONObject obj : parse(
+ "LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016
09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertNotNull(obj.get(Fields.TIMESTAMP.getName()));
+ }
+
+ }
+
+ private void runMissingYear(Calendar expected, Calendar input) {
+ SimpleDateFormat sdf = new SimpleDateFormat("MMM dd HH:mm:ss.SSS");
+ for (JSONObject obj :
parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|\t|src=10.0.0.1\tdevTime="
+ + sdf.format(input.getTime()) +
+ "\tdevTimeFormat=MMM dd HH:mm:ss.SSS" +
+ "\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(expected.getTime(), new Date((long)
obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(expected.getTimeInMillis(),
obj.get(Fields.TIMESTAMP.getName()));
+ }
+ }
+
+ @Test
+ public void testMissingYearFromDate() throws java.text.ParseException {
+ Calendar current = Calendar.getInstance();
+ Calendar correct = Calendar.getInstance();
+
+ correct.setTimeInMillis(current.getTimeInMillis());
+
+ runMissingYear(correct, current);
+ }
+
+ @Test
+ public void testFourDayFutureBecomesPast() {
+ Calendar current = Calendar.getInstance();
+ Calendar correct = Calendar.getInstance();
+
+ current.add(Calendar.DAY_OF_MONTH, 5);
+ // correct.setTime(current.getTime());
+ correct.setTimeInMillis(current.getTimeInMillis());
+ correct.add(Calendar.YEAR, -1);
+
+ runMissingYear(correct, current);
+ }
+
+ /**
+ * Sample from
https://docs.imperva.com/bundle/cloud-application-security/page/more/example-logs.htm#logEx2
+ */
+ @Test
+ public void testLEEF_CEFlikeSample() {
+ List<JSONObject> parse = parse("LEEF:0|Incapsula|SIEMintegration|0|SQL
Injection| fileId=3412364560000000008
sourceServiceName=test56111115.incaptest.co siteid=1333546 suid=300656
requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0)
Gecko/20100101 Firefox/38.0 popName=mia cs2=true cs2Label=Javascript Support
cs3=true cs3Label=CO Support cs1=NA cs1Label=Cap Support
cs4=936e64c2-bdd1-4719-9bd0-2d882a72f30d cs4Label=VID
cs5=bab1712be85b00ab21d20bf0d7b5db82701f27f53fbac1 [...]
+ JSONObject obj = parse.get(0);
+ Assert.assertNotNull(obj);
+ Assert.assertEquals("3412364560000000008", obj.get("fileId"));
+ Assert.assertEquals("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0)
Gecko/20100101 Firefox/38.0", obj.get("requestClientApplication"));
+ Assert.assertTrue(obj.containsKey("longitude"));
+ Assert.assertFalse(obj.containsKey("cs8"));
+ Assert.assertFalse(obj.containsKey("cs8Label"));
+ }
+
+ @Test
+ public void testLEEFParserSample() throws Exception {
+ runTest("sample", Resources.readLines(Resources.getResource(getClass(),
"sample.leef"), UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "sample.schema"),
UTF_8));
+ }
+
+ private void runTest(String name, List<String> lines, String schema) throws
Exception {
+ runTest(name, lines, schema, "");
+ }
+
+ private void runTest(String name, List<String> lines, String schema, String
targetJson) throws Exception {
+ for (String inputString : lines) {
+ JSONObject parsed = parse(inputString).get(0);
+ Assert.assertNotNull(parsed);
+ Assert.assertNotNull(parsed.get(Fields.TIMESTAMP.getName()));
+ Assert.assertTrue((long) parsed.get(Fields.TIMESTAMP.getName()) > 0);
+
+ JSONParser parser = new JSONParser();
+
+ Map<?, ?> json = null;
+ json = (Map<?, ?>) parser.parse(parsed.toJSONString());
+ Assert.assertEquals(true, validateJsonData(schema, json.toString()));
+ }
+ }
+
+ private void assertSimpleSample(List<JSONObject> parse) {
+ JSONObject obj = parse.get(0);
+ Assert.assertNotNull(obj);
+ Assert.assertTrue(obj.containsKey(Fields.SRC_ADDR.getName()));
+ Assert.assertEquals("192.0.2.0", obj.get(Fields.SRC_ADDR.getName()));
+ }
+
+ @Test
+ public void testLEEF_1_0_versionIncluded() {
+ List<JSONObject> parse = parse("LEEF:1.0|Microsoft|MSExchange|4.0
SP1|15345|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
+ assertSimpleSample(parse);
+ }
+
+ @Test
+ public void testLEEF_2_0() {
+ List<JSONObject> parse = parse("LEEF:2.0|Vendor|Product|Version|EventID|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
+ assertSimpleSample(parse);
+ }
+
+ @Test
+ public void testLEEF_2_0_delimiterSpecified() {
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black");
+ assertSimpleSample(parse);
+ }
+
+ @Test
+ public void testLEEF_2_0_delimiterUsedIncorrectly() {
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
+ assertFalse(parse.get(0).containsKey(Fields.DST_ADDR));
+ }
+
+ @Test
+ public void testLEEFMultiLine() {
+ List<JSONObject> parse = parse("LEEF:2.0|Vendor|Product|Version|EventID|
src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=line1"
+
+ "\nLEEF:2.0|Vendor|Product|Version|EventID|
src=192.0.2.1\tdst=172.50.123.2\tsev=6\tcat=anomaly\tsrcPort=82\tdstPort=22\tusrName=line2");
+ assertSimpleSample(parse);
+ assertEquals(2, parse.size());
+ }
+
+
+ @Test
+ public void testLEEFcustomdevTimeFormat() {
+ String customFormat = "yyyy-MM-dd HH:mm:ss.SSS zzz";
+ Date customDate = new Date();
+ DateFormat customFormatter = new SimpleDateFormat(customFormat);
+
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black^devTime="
+ customFormatter.format(customDate) + "^devTimeFormat=" + customFormat);
+ JSONObject obj = parse.get(0);
+ assertEquals(obj.get(Fields.TIMESTAMP.getName()), customDate.getTime());
+ }
+
+ @Test
+ public void testLEEFdevTimeWithNoCustomFormat() {
+ String standardFormat = "MMM dd yyyy HH:mm:ss.SSS zzz";
+ Date customDate = new Date();
+ long expected = customDate.getTime();
+ DateFormat customFormatter = new SimpleDateFormat(standardFormat);
+
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^|
src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black^devTime="
+ customFormatter.format(customDate));
+ JSONObject obj = parse.get(0);
+ assertEquals(obj.get(Fields.TIMESTAMP.getName()), expected);
+ }
+
+ protected boolean validateJsonData(final String jsonSchema, final String
jsonData) throws Exception {
+ final JsonNode d = JsonLoader.fromString(jsonData);
+ final JsonNode s = JsonLoader.fromString(jsonSchema);
+
+ final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+ JsonValidator v = factory.getValidator();
+
+ ProcessingReport report = v.validate(s, d);
+
+ return report.toString().contains("success");
+ }
+
+ private List<JSONObject> parse(String string) {
+ Optional<MessageParserResult<JSONObject>> parse =
parser.parseOptionalResult(string.getBytes(UTF_8));
+ Assert.assertTrue(parse.isPresent());
+ return parse.get().getMessages();
+ }
}
