http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt b/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt deleted file mode 100644 index 0e4bf74..0000000 --- a/metron-platform/metron-parsers/src/test/resources/logData/LancopeParserTest.txt +++ /dev/null @@ -1 +0,0 @@ -{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt b/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt deleted file mode 100644 index c58bcc8..0000000 --- a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt +++ /dev/null @@ -1,2 +0,0 @@ -<11>Jan 5 05:38:59 PAN1.exampleCustomer.com 1,2015/01/05 05:38:58,0006C110285,THREAT,vulnerability,1,2015/01/05 05:38:58,10.0.0.115,216.0.10.198,0.0.0.0,0.0.0.0,EX-Allow,example\\user.name,,web-browsing,vsys1,internal,external,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 05:38:58,12031,1,54180,80,0,0,0x80004000,tcp,reset-both,\"ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9\",HTTP: IIS Denial Of Service Attempt(40019),any,high,client-to-server,347368099,0x0,10.0.0.0-10.255.255.255,US,0,,1200568889751109656,, -<14>Jan 5 12:51:34 PAN1.exampleCustomer.com 1,2015/01/05 12:51:33,0011C103117,TRAFFIC,end,1,2015/01/05 12:51:33,10.0.0.39,10.1.0.163,0.0.0.0,0.0.0.0,EX-Allow,,example\\user.name,ms-ds-smb,vsys1,v_external,v_internal,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 12:51:33,33760927,1,52688,445,0,0,0x401a,tcp,allow,2229,1287,942,10,2015/01/05 12:51:01,30,any,0,17754932062,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,4 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt b/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt deleted file mode 100644 index af257aa..0000000 --- a/metron-platform/metron-parsers/src/test/resources/logData/SourcefireParserTest.txt +++ /dev/null @@ -1,3 +0,0 @@ -SFIMS: [Primary Detection Engine (a7213248-6423-11e3-8537-fac6a92b7d9d)][MTD Access Control] Connection Type: Start, User: Unknown, Client: Unknown, Application Protocol: Unknown, Web App: Unknown, Firewall Rule Name: MTD Access Control, Firewall Rule Action: Allow, Firewall Rule Reasons: Unknown, URL Category: Unknown, URL_Reputation: Risk unknown, URL: Unknown, Interface Ingress: s1p1, Interface Egress: N/A, Security Zone Ingress: Unknown, Security Zone Egress: N/A, Security Intelligence Matching IP: None, Security Intelligence Category: None, {TCP} 72.163.0.129:60517 -> 10.1.128.236:443 -snort: [1:3192:2] WEB-CLIENT Windows Media Player directory traversal via Content-Disposition attempt [Classification: Attempted User Privilege Gain] [Priority: 1] {TCP} 46.149.110.103:80 -> 192.168.56.102:1073 -SFIMS: Correlation Event: Open Soc Log Forwarding/Opensoc Log Forwarding at Thu Oct 23 04:55:39 2014 UTC: [1:19123:7] \"MALWARE-CNC Dropper Win.Trojan.Cefyns.A variant outbound connection\" [Impact: Unknown] From \"172.19.50.7\" at Thu Oct 23 04:55:38 2014 UTC [Classification: A Network Trojan was Detected] [Priority: 1] {tcp} 139.230.245.23:52078->72.52.4.91:80 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef deleted file mode 100644 index a35f354..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef +++ /dev/null @@ -1 +0,0 @@ -2016-04-01T09:29:11.356-0400 CEF:0|Adallom|Adallom|1.0|56fe779ee4b0459f4e9a484a|ALERT_CABINET_EVENT_MATCH_AUDIT|0|msg=Activity policy 'User download/view file' was triggered by 'per...@example.com' suser=au...@example.com start=1459517280810 end=1459517280810 audits=["AVPR-4oIPeFmuZ3CKKrg","AVPR-wx80cd9PUpAu2aj","AVPR-6XGPeFmuZ3CKKvx","AVPSALn_qE4Kgs_8_yK9","AVPSASW3gw_f3aEvgEmi"] services=["APPID_SXC"] users=["anot...@example.com"] cs6=https://abcd-remote.console.arc.com/#/alerts/56fe779ee4b0459f4e9a484a cs6Label=consoleUrl \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema deleted file mode 100644 index a91cce0..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema +++ /dev/null @@ -1,37 +0,0 @@ -{ - "title": "Adallom Schema", - "type": "object", - "properties": { - "original_string": { - "type": "string" - }, - "timestamp": { - "type": "integer" - }, - "DeviceVendor": { - "type": "string" - }, - "DeviceProduct": { - "type": "string" - }, - "DeviceVersion": { - "type": "string" - }, - "DeviceEvent": { - "type": "string" - }, - "Name": { - "type": "string" - }, - "Severity": { - "type": "integer" - }, - "consoleUrl": { - "type": "string" - } - }, - "required": [ - "original_string", "timestamp", - "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity", - "consoleUrl"] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef deleted file mode 100644 index 9d4fe6f..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef +++ /dev/null @@ -1 +0,0 @@ -Mar 21 14:05:02 HHHPVATN1 CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve password|5|act=Retrieve password suser=spilgrim fname=Root\ABC phobos3 - COMP dvc=120.99.70.3 shost=10.44.134.78 dhost= duser= externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2=Security Vulnerability Mgmt cs3Label="Device Type" cs3= cs4Label="Database" cs4= cs5Label="Other info" cs5=101.198.70.93 cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2=Needed to verify config files being pulled msg=Needed to verify config files being pulled \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json deleted file mode 100644 index e900a9a..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "\"Other info\"": "101.198.70.93", - "\"Safe Name\"": "Security Vulnerability Mgmt", - "\"Ticket Id\"": "Needed to verify config files being pulled ", - "deviceAction": "Retrieve password", - "deviceAddress": "120.99.70.3", - "device_product": "Vault", - "device_vendor": "Cyber-Ark", - "device_version": "7.20.0091", - "event_class_id": "295", - "event_name": "Retrieve password", - "fileName": "Root\\ABC phobos3 - COMP", - "header": "Mar 21 14:05:02 HHHPVATN1 CEF:0", - "message": "Needed to verify config files being pulled", - "original_string": "Mar 21 14:05:02 HHHPVATN1 CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve password|5|act=Retrieve password suser=spilgrim fname=Root\\ABC phobos3 - COMP dvc=120.99.70.3 shost=10.44.134.78 dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2=Security Vulnerability Mgmt cs3Label=\"Device Type\" cs3= cs4Label=\"Database\" cs4= cs5Label=\"Other info\" cs5=101.198.70.93 cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2=Needed to verify config files being pulled msg=Needed to verify config files being pulled", - "severity": "5", - "source.type": "cyberark", - "src_hostname": "10.44.134.78", - "src_username": "spilgrim", - "timestamp": 1458569102000 -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema deleted file mode 100644 index 5bd1021..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema +++ /dev/null @@ -1,38 +0,0 @@ -{ - "title": "CyberArk Schema", - "type": "object", - "properties": { - "ip_src_addr": { - "type": "string" - }, - "ip_dst_addr": { - "type": "string" - }, - "original_string": { - "type": "string" - }, - "timestamp": { - "type": "integer" - }, - "DeviceVendor": { - "type": "string" - }, - "DeviceProduct": { - "type": "string" - }, - "DeviceVersion": { - "type": "string" - }, - "DeviceEvent": { - "type": "string" - }, - "Name": { - "type": "string" - }, - "Severity": { - "type": "integer" - } - }, - "required": ["original_string", "timestamp", - "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity"] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef deleted file mode 100644 index ab9b830..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.cef +++ /dev/null @@ -1 +0,0 @@ -<14>Apr 7 10:10:10 hostname CEF: 0|Palo Alto Networks|PAN-OS|6.1.3|url|THREAT|1|rt=Apr 07 2017 00:10:10 GMT deviceExternalId=00000000 src=10.10.10.10 dst=20.20.20.20 sourceTranslatedAddress=0.0.0.0 destinationTranslatedAddress=0.0.0.0 cs1Label=Rule cs1=Trusted-to-Untrusted suser= duser= app=ssl cs3Label=Virtual Sys cs3=vsys2 cs4Label=Src Zone cs4=Trusted cs5Label=Dst Zone cs5=Untrusted deviceInboundInterface=ethernet1/12.345 deviceOutboundInterface=ethernet1/12.345 cs6Label=LogProfile cs6=Log_Profile cn1Label=SessionID cn1=123456 cnt=1 spt=18371 dpt=443 sourceTranslatedPort=0 destinationTranslatedPort=0 flexString1Label=Flags flexString1=0x8000 proto=tcp act=alert request=\"www.example.com/\" cs2Label=URL Cat cs2=gambling flexString2Label=Direction flexString2=client-to-server externalId=123456789 requestContext= cat=(9999) filePath= fileId=0 fileHash= deviceProcessName=Device.Process.Name \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema deleted file mode 100644 index 7135634..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/palo.schema +++ /dev/null @@ -1,38 +0,0 @@ -{ - "title": "PaloAlto Schema", - "type": "object", - "properties": { - "ip_src_addr": { - "type": "string" - }, - "ip_dst_addr": { - "type": "string" - }, - "original_string": { - "type": "string" - }, - "timestamp": { - "type": "integer" - }, - "DeviceVendor": { - "type": "string" - }, - "DeviceProduct": { - "type": "string" - }, - "DeviceVersion": { - "type": "string" - }, - "DeviceEvent": { - "type": "string" - }, - "Name": { - "type": "string" - }, - "Severity": { - "type": "integer" - } - }, - "required": ["original_string", "timestamp", - "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity"] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef deleted file mode 100644 index 86e1d6b..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef +++ /dev/null @@ -1 +0,0 @@ -<14>CEF:0|Imperva Inc.|SecureSphere|10.0.0.4_16|ABC - Secure Login.vm Page Rate Limit UK - Source IP||High|act=alert dst=17.43.200.42 dpt=88 duser=${Alert.username} src=10.31.45.69 spt=34435 proto=TCP rt=31 March 2016 13:04:55 cat=Alert cs1= cs1Label=Policy cs2=ABC-Secure cs2Label=ServerGroup cs3=servers_svc cs3Label=ServiceName cs4=server_app cs4Label=ApplicationName cs5=QA cs5Label=Description \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema deleted file mode 100644 index b38485c..0000000 --- a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema +++ /dev/null @@ -1,67 +0,0 @@ -{ - "title": "WAF CEF Schema", - "type": "object", - "properties": { - "ip_src_addr": { - "type": "string" - }, - "ip_src_port": { - "type": "integer" - }, - "ip_dst_addr": { - "type": "string" - }, - "ip_dst_port": { - "type": "integer" - }, - "original_string": { - "type": "string" - }, - "@version": { - "type": "string" - }, - "timestamp": { - "type": "integer" - }, - "type": { - "type": "string" - }, - "DeviceVendor": { - "type": "string" - }, - "DeviceProduct": { - "type": "string" - }, - "DeviceVersion": { - "type": "string" - }, - "DeviceEvent": { - "type": "string" - }, - "Name": { - "type": "string" - }, - "Severity": { - "type": "integer" - }, - "cat": { - "type": "string" - }, - "ServerGroup": { - "type": "string" - }, - "ServiceName": { - "type": "string" - }, - "ApplicationName": { - "type": "string" - }, - "Description": { - "type": "string" - } - }, - "required": ["ip_src_addr", "ip_dst_addr", "ip_src_port", "ip_dst_port", "original_string", "timestamp", - "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity", - "cat", - "ServerGroup", "ServiceName", "ApplicationName", "Description"] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/otherPatterns/common ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/otherPatterns/common b/metron-platform/metron-parsers/src/test/resources/otherPatterns/common new file mode 100644 index 0000000..10c72dc --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/otherPatterns/common @@ -0,0 +1,96 @@ +# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME:UNWANTED} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM:UNWANTED}) +BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+)) +BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b + +POSINT \b(?:[1-9][0-9]*)\b +NONNEGINT \b(?:[0-9]+)\b +WORD \b\w+\b +NOTSPACE \S+ +SPACE \s* +DATA .*? +GREEDYDATA .* +#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`))) +QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) +IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED}) +HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b) +HOST %{HOSTNAME:UNWANTED} +IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED}) +HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT}) + +# paths +PATH (?:%{UNIXPATH}|%{WINPATH}) +UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+ +#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+ +TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) +WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +# Time: HH:MM:SS +#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? +# I'm still on the fence about using grok to perform the time match, +# since it's probably slower. +# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +GREEDYDATA .* + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG (?:[\w._/%-]+) +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING:UNWANTED} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-parsers/src/test/resources/patterns/test ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/patterns/test b/metron-platform/metron-parsers/src/test/resources/patterns/test new file mode 100644 index 0000000..a88a255 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/patterns/test @@ -0,0 +1,2 @@ +YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED} +YAF_DELIMITED %{NUMBER:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:protocol}\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\|%{SPACE:UNWANTED}%{INT:ip_src_port}\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java index 2b39f9d..0994361 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/TestConstants.java @@ -20,7 +20,11 @@ package org.apache.metron; public class TestConstants { public final static String SAMPLE_CONFIG_PATH = "../metron-integration-test/src/main/config/zookeeper/"; + public final static String SAMPLE_EXTENSIONS_CONFIG_PATH = "../metron-integration-test/src/main/config/zookeeper/extensions"; + public final static String SAMPLE_EXTENSIONS_PARSER_CONFIG_PATH = "../metron-integration-test/src/main/config/zookeeper/extensions/parsers"; public final static String PARSER_CONFIGS_PATH = "../metron-parsers/src/main/config/zookeeper/"; + public final static String A_PARSER_CONFIGS_PATH_FMT = "../metron-extensions/metron-parser-extensions/metron-parser-%s-extension/metron-parser-%s/src/main/config/zookeeper/"; + public final static String THIS_PARSER_CONFIGS_PATH = "src/main/config/zookeeper/"; public final static String ENRICHMENTS_CONFIGS_PATH = "../metron-enrichment/src/main/config/zookeeper/"; public final static String SAMPLE_DATA_PATH = "../metron-integration-test/src/main/sample/data/"; public final static String SAMPLE_DATA_INPUT_PATH = "../metron-integration-test/src/main/sample/data/yaf/raw/"; http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ResourceCopier.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ResourceCopier.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ResourceCopier.java new file mode 100644 index 0000000..2e85b88 --- /dev/null +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/ResourceCopier.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.test.utils; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +/** + * Utility class to copy a whole directory. This is useful for copying files from + * src/test/resources to target/testbed for example + */ +public class ResourceCopier { + + /** + * Copy the resources from sourcePath to targetPath. If a resource exists, it will be overwritten. + * This is the equivolent of calling <code>copyResources(source,target,true)</code> + * + * @param sourcePath source {@link Path} + * @param targetPath target {@link Path} + */ + public static void copyResources(final Path sourcePath, final Path targetPath) + throws IOException { + synchronized (ResourceCopier.class) { + copyResources(sourcePath, targetPath, true); + } + } + + /** + * Copy the resources from sourcePath to targetPath. The overwrite flag determines if existing + * resources are overwritten or not. + * + * @param sourcePath source {@link Path} + * @param targetPath target {@link Path} + * @param overwrite if true, the the target will be overwritten with {@link + * java.nio.file.StandardCopyOption#REPLACE_EXISTING} + */ + public static void copyResources(final Path sourcePath, final Path targetPath, + final boolean overwrite) throws IOException { + synchronized (ResourceCopier.class) { + Files.walkFileTree(sourcePath, new SimpleFileVisitor<Path>() { + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + + Path relativeSource = sourcePath.relativize(dir); + Path target = targetPath.resolve(relativeSource); + + Files.createDirectories(target); + + return FileVisitResult.CONTINUE; + + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + + Path relativeSource = sourcePath.relativize(file); + Path target = targetPath.resolve(relativeSource); + + if (overwrite) { + Files.copy(file, target, REPLACE_EXISTING); + } else { + if (!target.toFile().exists()) { + Files.copy(file, target); + } + } + + return FileVisitResult.CONTINUE; + } + }); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java index 0e3e4e6..c6153d3 100644 --- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java +++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/SampleDataUtils.java @@ -24,9 +24,22 @@ import java.io.File; import java.io.FileNotFoundException; public class SampleDataUtils { - + public static final String UP_LEVEL = "../"; public static String getSampleDataPath(String sensorType, TestDataType testDataType) throws FileNotFoundException { - File sensorSampleDataPath = new File(TestConstants.SAMPLE_DATA_PATH, sensorType); + return getSampleDataPath(0,sensorType,testDataType); + } + + public static String getSampleDataPath(int level, String sensorType, TestDataType testDataType) throws FileNotFoundException { + String path = TestConstants.SAMPLE_DATA_PATH; + + if(level > 0){ + StringBuilder sb = new StringBuilder(); + for(int i = 0; i < level; i++){ + sb.append(UP_LEVEL); + } + path = sb.append(path).toString(); + } + File sensorSampleDataPath = new File(path, sensorType); if (sensorSampleDataPath.exists() && sensorSampleDataPath.isDirectory()) { File sampleDataPath = new File(sensorSampleDataPath, testDataType.getDirectoryName()); if (sampleDataPath.exists() && sampleDataPath.isDirectory()) { http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml index 93ced81..fee13fc 100644 --- a/metron-platform/pom.xml +++ b/metron-platform/pom.xml @@ -50,6 +50,7 @@ <module>metron-data-management</module> <module>metron-pcap</module> <module>metron-integration-test</module> + <module>metron-extensions</module> <module>metron-test-utilities</module> <module>metron-api</module> <module>metron-indexing</module> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ad97f58..9a86314 100644 --- a/pom.xml +++ b/pom.xml @@ -29,13 +29,16 @@ <url>https://www.apache.org</url> </organization> <modules> - <module>metron-analytics</module> - <module>metron-platform</module> - <module>metron-deployment</module> - <module>metron-contrib</module> - <module>metron-interface</module> - <module>site-book</module> - <module>metron-stellar</module> + <module>bundles-lib</module> + <module>metron-analytics</module> + <module>metron-platform</module> + <module>metron-maven-archetypes</module> + <module>metron-deployment</module> + <module>metron-contrib</module> + <module>metron-interface</module> + <module>site-book</module> + <module>metron-stellar</module> + </modules> <repositories> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/site/current-book/metron-analytics/metron-maas-service/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-analytics/metron-maas-service/index.html b/site/current-book/metron-analytics/metron-maas-service/index.html index b0f9162..f3bab6b 100644 --- a/site/current-book/metron-analytics/metron-maas-service/index.html +++ b/site/current-book/metron-analytics/metron-maas-service/index.html @@ -381,7 +381,7 @@ usage: ModelSubmission <div class="source"> <div class="source"> <pre>{ - "parserClassName": "org.apache.metron.parsers.GrokParser", + "parserClassName": "org.apache.metron.parsers.grok.GrokParser", "sensorTopic": "squid", "parserConfig": { "grokPath": "/patterns/squid", http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/site/current-book/metron-platform/metron-management/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-management/index.html b/site/current-book/metron-platform/metron-management/index.html index 6efed91..57050f2 100644 --- a/site/current-book/metron-platform/metron-management/index.html +++ b/site/current-book/metron-platform/metron-management/index.html @@ -997,7 +997,7 @@ Functions loaded, you may refer to functions now... [Stellar]>>> #Just to make sure it looks right, we can view the JSON [Stellar]>>> squid_parser_config { - "parserClassName": "org.apache.metron.parsers.GrokParser", + "parserClassName": "org.apache.metron.parsers.grok.GrokParser", "sensorTopic": "squid", "parserConfig": { "grokPath": "/patterns/squid", @@ -1027,7 +1027,7 @@ Functions loaded, you may refer to functions now... ╠═══════════════════════════╪═══════════════════════════════════════════╪═════════════════════════════════════╣ ║ squid_parser_config │ { │ CONFIG_GET('PARSER', 'squid') ║ ║ │ "parserClassName": │ ║ -║ │ "org.apache.metron.parsers.GrokParser", │ ║ +║ │ "org.apache.metron.parsers.grok.GrokParser", │ ║ ║ │ │ ║ ║ │ "sensorTopic": "squid", │ ║ ║ │ │ ║ @@ -1136,7 +1136,7 @@ Returns: The String representation of the config in zookeeper [Stellar]>>> #It should be just as we started the exercise [Stellar]>>> CONFIG_GET('PARSER', 'squid') { - "parserClassName" : "org.apache.metron.parsers.GrokParser", + "parserClassName" : "org.apache.metron.parsers.grok.GrokParser", "sensorTopic" : "squid", "parserConfig" : { "grokPath" : "/patterns/squid", http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/site/current-book/metron-platform/metron-parsers/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-parsers/index.html b/site/current-book/metron-platform/metron-parsers/index.html index f7d13a6..8cc6b49 100644 --- a/site/current-book/metron-platform/metron-parsers/index.html +++ b/site/current-book/metron-platform/metron-parsers/index.html @@ -238,7 +238,7 @@ <ul> -<li>Grok parser: <tt>org.apache.metron.parsers.GrokParser</tt> with possible <tt>parserConfig</tt> entries of +<li>Grok parser: <tt>org.apache.metron.parsers.grok.GrokParser</tt> with possible <tt>parserConfig</tt> entries of <ul> @@ -482,7 +482,7 @@ HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )" <div class="source"> <div class="source"> <pre>{ - "parserClassName":"org.apache.metron.parsers.GrokParser", + "parserClassName":"org.apache.metron.parsers.grok.GrokParser", "sensorTopic":"yaf", "fieldTransformations" : [ { @@ -523,7 +523,7 @@ HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )" <ul> -<li>org.apache.metron.parsers.GrokParser</li> +<li>org.apache.metron.parsers.grok.GrokParser</li> </ul> <p>For more information on the Grok project please refer to the following link:</p> <p><a class="externalLink" href="https://github.com/thekrakken/java-grok";>https://github.com/thekrakken/java-grok</a></p>
