simonellistonball commented on a change in pull request #1408: METRON-2118: Added a LEEF parser URL: https://github.com/apache/metron/pull/1408#discussion_r285264301
########## File path: metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java ########## @@ -0,0 +1,252 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.leef; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.fge.jackson.JsonLoader; +import com.github.fge.jsonschema.core.report.ProcessingReport; +import com.github.fge.jsonschema.main.JsonSchemaFactory; +import com.github.fge.jsonschema.main.JsonValidator; +import com.google.common.io.Resources; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class LEEFParserTest { + private static final Charset UTF_8 = Charset.forName("utf-8"); + private LEEFParser parser; + + @Before + public void setUp() { + parser = new LEEFParser(); + parser.init(); + } + + @Test + public void testInvalid() { + List<JSONObject> obj = parse("test test test nonsense\n"); + Assert.assertEquals(0, obj.size()); + } + + @Test + public void testTimestampPriority() throws java.text.ParseException { + long correctTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400") + .getTime(); + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz"); + + for (JSONObject obj : parse( + "LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) { + Assert.assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp"))); + Assert.assertEquals(correctTime, obj.get("timestamp")); + } + for (JSONObject obj : parse( + "2016-06-01T09:29:11.356-04:00 host LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) { + Assert.assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp"))); + Assert.assertEquals(correctTime, obj.get("timestamp")); + } + for (JSONObject obj : parse( + "2016-05-01T09:29:11.356-04:00 host LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) { + Assert.assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp"))); + Assert.assertEquals(correctTime, obj.get("timestamp")); + } + for (JSONObject obj : parse( + "LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) { + Assert.assertNotNull(obj.get("timestamp")); + } + + } + + private void runMissingYear(Calendar expected, Calendar input) { + SimpleDateFormat sdf = new SimpleDateFormat("MMM dd HH:mm:ss.SSS"); + for (JSONObject obj : parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|\t|src=10.0.0.1\tdevTime=" + + sdf.format(input.getTime()) + + "\tdevTimeFormat=MMM dd HH:mm:ss.SSS" + + "\tdst=2.1.2.2\tspt=1232")) { + Assert.assertEquals(expected.getTime(), new Date((long) obj.get("timestamp"))); + Assert.assertEquals(expected.getTimeInMillis(), obj.get("timestamp")); + } + } + + @Test + public void testMissingYearFromDate() throws java.text.ParseException { + Calendar current = Calendar.getInstance(); + Calendar correct = Calendar.getInstance(); + + correct.setTimeInMillis(current.getTimeInMillis()); + + runMissingYear(correct, current); + } + + @Test + public void testFourDayFutureBecomesPast() { + Calendar current = Calendar.getInstance(); + Calendar correct = Calendar.getInstance(); + + current.add(Calendar.DAY_OF_MONTH, 5); + // correct.setTime(current.getTime()); + correct.setTimeInMillis(current.getTimeInMillis()); + correct.add(Calendar.YEAR, -1); + + runMissingYear(correct, current); + } + + /** + * Sample from https://docs.imperva.com/bundle/cloud-application-security/page/more/example-logs.htm#logEx2 + */ + @Test + public void testLEEF_CEFlikeSample() { + List<JSONObject> parse = parse("LEEF:0|Incapsula|SIEMintegration|0|SQL Injection| fileId=3412364560000000008 sourceServiceName=test56111115.incaptest.co siteid=1333546 suid=300656 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0 popName=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support cs1=NA cs1Label=Cap Support cs4=936e64c2-bdd1-4719-9bd0-2d882a72f30d cs4Label=VID cs5=bab1712be85b00ab21d20bf0d7b5db82701f27f53fbac19a4252efc722ac9131fdc60c0da620282b02dfb8051e7a60f9 cs5Label=clappsig dproc=Browser cs6=Firefox cs6Label=clapp calCountryOrRegion=IL cicode=Rehovot cs7=31.8969 cs7Label=latitude cs8=34.8186 cs8Label=longitude Customer=siemtest start=1460303291788 url=test56111115.incaptest.co/ requestMethod=GET qstr=keywords\\=3%29%29%29%20AND%203434%3d%28%27%3amvc%3a%27%7c%7c%28SELECT%20CASE%203434%20WHEN%203434%20THEN%201%20ELSE%200%20END%20FROM%20RDB%24DATABASE%29%7c%7c%27%3aqvi%3a%27%29%20AND%20%28%28%283793%3d3793 cn1=200 proto=HTTP cat=REQ_PASSED deviceExternalId=2323800832649 dst=54.195.35.43 dstPort=80 in=406 xff=127.0.0.1 srcPort=443 src=127.0.0.1 protoVer=TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 fileType=12999,50999,50037,50044, filePermission=37,20,1,1, cs9=,High Risk SQL Expressions,,SQL SELECT Expression, cs9Label=Rule name"); + JSONObject obj = parse.get(0); + Assert.assertNotNull(obj); + Assert.assertEquals("3412364560000000008", obj.get("fileId")); + Assert.assertEquals("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0", obj.get("requestClientApplication")); + Assert.assertTrue(obj.containsKey("longitude")); + Assert.assertFalse(obj.containsKey("cs8")); + Assert.assertFalse(obj.containsKey("cs8Label")); + } + + @Test + public void testLEEFParserSample() throws Exception { + runTest("sample", Resources.readLines(Resources.getResource(getClass(), "sample.leef"), UTF_8), + Resources.toString(Resources.getResource(getClass(), "sample.schema"), UTF_8)); + } + + private void runTest(String name, List<String> lines, String schema) throws Exception { + runTest(name, lines, schema, ""); + } + + private void runTest(String name, List<String> lines, String schema, String targetJson) throws Exception { + for (String inputString : lines) { + JSONObject parsed = parse(inputString).get(0); + Assert.assertNotNull(parsed); + Assert.assertNotNull(parsed.get("timestamp")); + Assert.assertTrue((long) parsed.get("timestamp") > 0); + + JSONParser parser = new JSONParser(); + + Map<?, ?> json = null; + try { + json = (Map<?, ?>) parser.parse(parsed.toJSONString()); + Assert.assertEquals(true, validateJsonData(schema, json.toString())); + } catch (ParseException e) { + e.printStackTrace(); + } + + // test against an explicit json example + if (!targetJson.isEmpty()) { + Review comment: Not really, this is a left over from a half complete idea in the current product's CEF Tests ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
