http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed new file mode 100644 index 0000000..b1d3102 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/parsed/test.parsed @@ -0,0 +1,27 @@ +{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"CTo78A11g7CYbbOHvj","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CTo78A11g7CYbbOHvj resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15l xUn5ngPfd"],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"TTLs":[3600.0,289.0,14.0],"qclass_name":"C_INTERNET","bro_timestamp":"1402308259.609","qtype_name":"AAAA","ip_dst_port":53,"qtype":28,"rejected":false,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"trans_id":62418,"uid":"CuJT272SKaJSuqO0Ia","protocol":"dns","original_string":"DNS | AA:true TTLs:[3600.0,289.0,14.0] qclass_name:C_INTERNET id.orig_p:33976 qtype_name:AAAA qtype:28 rejected:false id.resp_p:53 query:www.cisco.com answers:[\"www.cisco.com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"] trans_id:62418 rcode:0 rcode_name:NOERROR TC:false RA:true uid:CuJT272SKaJSuqO0Ia RD:true proto:udp id.orig_h:10.122.196.204 Z:0 qclass:1 ts:1402308259.609 id.resp_h:144.254.71.184","ip_dst_addr":"144.254.71.184","Z":0,"ip_src_addr":"10.122.196.204","qclass":1,"timestamp":1402308259609,"AA":true,"query":"www.cisco.com","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","RD":true,"ip_src_port":33976,"proto":"udp","gui d":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:KIRAN resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:10.122.196.204 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"10.122.196.204","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"],"timestamp":14 02307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN12312312","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:KIRAN12312312 resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd" ],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN12312312","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:KIRAN12312312 resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd" ],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"CTo78A11g7CYbbOHvj","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CTo78A11g7CYbbOHvj resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:gabacentre.pw status_msg:OK id.orig_h:10.122.196.204 response_body_len:25523 email:abul...@mail.csuchico.edu user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"gabacentre.pw","status_msg":"OK","response_body_len":25523,"ip_src_addr":"10.122.196.204","email":"abul...@mail.csuchico.edu","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0 .1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"TTLs":[3600.0,289.0,14.0],"qclass_name":"C_INTERNET","bro_timestamp":"1402308259.609","qtype_name":"AAAA","ip_dst_port":53,"qtype":28,"rejected":false,"answers":["gabacentre.pw","www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"trans_id":62418,"uid":"CYbbOHvj","protocol":"dns","original_string":"DNS | AA:true TTLs:[3600.0,289.0,14.0] qclass_name:C_INTERNET id.orig_p:33976 qtype_name:AAAA qtype:28 rejected:false id.resp_p:53 query:www.cisco.com answers:[\"gabacentre.pw\",\"www.cisco.com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"] trans_id:62418 rcode:0 rcode_name:NOERROR TC:false RA:true uid:CYbbOHvj RD:true proto:udp id.orig_h:93.188.160.43 Z:0 qclass:1 ts:1402308259.609 id.resp_h:144.254.71.184","ip_dst_addr":"144.254.71.184","Z":0,"ip_src_addr":"93.188.160.43","qclass":1,"timestamp":1402308259609,"AA":true,"query":"www.cisco.com","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","RD":true,"ip_src_port":33976,"proto ":"udp","guid":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"CTo78A11g7CYbbOHvj","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CTo78A11g7CYbbOHvj resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:192.249.113.37 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"192.249.113.37","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15l xUn5ngPfd"],"timestamp":1402307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"TTLs":[3600.0,289.0,14.0],"qclass_name":"C_INTERNET","bro_timestamp":"1402308259.609","qtype_name":"AAAA","ip_dst_port":53,"qtype":28,"rejected":false,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"trans_id":62418,"uid":"CuJT272SKaJSuqO0Ia","protocol":"dns","original_string":"DNS | AA:true TTLs:[3600.0,289.0,14.0] qclass_name:C_INTERNET id.orig_p:33976 qtype_name:AAAA qtype:28 rejected:false id.resp_p:53 query:www.cisco.com answers:[\"www.cisco.com.akadns.net\",\"origin-www.cisco.com\",\"2001:420:1201:2::a\"] trans_id:62418 rcode:0 rcode_name:NOERROR TC:false RA:true uid:CuJT272SKaJSuqO0Ia RD:true proto:udp id.orig_h:10.122.196.204 Z:0 qclass:1 ts:1402308259.609 id.resp_h:144.254.71.184","ip_dst_addr":"144.254.71.184","Z":0,"ip_src_addr":"10.122.196.204","qclass":1,"timestamp":1402308259609,"AA":true,"query":"www.cisco.com","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","RD":true,"ip_src_port":33976,"proto":"udp","gui d":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1402307733.473","status_code":200,"method":"GET","ip_dst_port":80,"request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","uid":"KIRAN","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:58808 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:KIRAN resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:www.cisco.com status_msg:OK id.orig_h:10.122.196.204 response_body_len:25523 user_agent:curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3 ts:1402307733.473 id.resp_h:72.163.4.161 resp_fuids:[\"FJDyMC15lxUn5ngPfd\"]","ip_dst_addr":"72.163.4.161","ip_src_port":58808,"host":"www.cisco.com","status_msg":"OK","response_body_len":25523,"ip_src_addr":"10.122.196.204","user_agent":"curl\/7.22.0 (x86_64-pc-linux-gnu) libcurl\/7.22.0 OpenSSL\/1.0.1 zlib\/1.2.3.4 libidn\/1.23 librtmp\/2.3","resp_fuids":["FJDyMC15lxUn5ngPfd"],"timestamp":14 02307733473,"guid":"this-is-random-uuid-will-be-36-chars"} +{"bro_timestamp":"1440447880.931272","resp_pkts":1,"resp_ip_bytes":48,"ip_dst_port":1812,"orig_bytes":75,"orig_ip_bytes":103,"orig_pkts":1,"missed_bytes":0,"history":"Dd","tunnel_parents":[],"source.type":"bro","duration":1.001459,"uid":"CWxtRHnBTbldHnmGh","protocol":"conn","resp_bytes":20,"original_string":"CONN | id.orig_p:52178 resp_pkts:1 resp_ip_bytes:48 orig_bytes:75 id.resp_p:1812 orig_ip_bytes:103 orig_pkts:1 missed_bytes:0 history:Dd tunnel_parents:[] duration:1.001459 uid:CWxtRHnBTbldHnmGh resp_bytes:20 service:radius conn_state:SF proto:udp id.orig_h:127.0.0.1 ts:1440447880.931272 id.resp_h:127.0.0.1","ip_dst_addr":"127.0.0.1","ip_src_port":52178,"service":"radius","conn_state":"SF","proto":"udp","guid":"4a92fe07-8f9d-4092-83c3-0d4e37c92d29","ip_src_addr":"127.0.0.1","timestamp":1440447880931} +{"bro_timestamp":"1440447904.122012","resp_pkts":0,"resp_ip_bytes":0,"ip_dst_port":1812,"orig_bytes":225,"orig_ip_bytes":309,"orig_pkts":3,"missed_bytes":0,"history":"D","tunnel_parents":[],"source.type":"bro","duration":10.008839,"uid":"CK2Oivhlh0ovRcYx","protocol":"conn","resp_bytes":0,"original_string":"CONN | id.orig_p:62956 resp_pkts:0 resp_ip_bytes:0 orig_bytes:225 id.resp_p:1812 orig_ip_bytes:309 orig_pkts:3 missed_bytes:0 history:D tunnel_parents:[] duration:10.008839 uid:CK2Oivhlh0ovRcYx resp_bytes:0 service:radius conn_state:S0 proto:udp id.orig_h:127.0.0.1 ts:1440447904.122012 id.resp_h:127.0.0.1","ip_dst_addr":"127.0.0.1","ip_src_port":62956,"service":"radius","conn_state":"S0","proto":"udp","guid":"9e4952e0-6dd3-4487-b5fa-299b9433c381","ip_src_addr":"127.0.0.1","timestamp":1440447904122} +{"bro_timestamp":"1440448190.335333","resp_pkts":1,"resp_ip_bytes":99,"ip_dst_port":1812,"orig_bytes":75,"orig_ip_bytes":103,"orig_pkts":1,"missed_bytes":0,"history":"Dd","tunnel_parents":[],"source.type":"bro","duration":5.17E-4,"uid":"CX6mcO38sO7dkDxK55","protocol":"conn","resp_bytes":71,"original_string":"CONN | id.orig_p:53127 resp_pkts:1 resp_ip_bytes:99 orig_bytes:75 id.resp_p:1812 orig_ip_bytes:103 orig_pkts:1 missed_bytes:0 history:Dd tunnel_parents:[] duration:0.000517 uid:CX6mcO38sO7dkDxK55 resp_bytes:71 service:radius conn_state:SF proto:udp id.orig_h:127.0.0.1 ts:1440448190.335333 id.resp_h:127.0.0.1","ip_dst_addr":"127.0.0.1","ip_src_port":53127,"service":"radius","conn_state":"SF","proto":"udp","guid":"bc1af1bf-5b1c-4829-b574-3243670fd448","ip_src_addr":"127.0.0.1","timestamp":1440448190335} +{"bro_timestamp":"1216702277.477596","ip_dst_port":80,"failure_reason":"not a http reply line","source.type":"bro","uid":"C4O50B3WAUCb2Yw29j","protocol":"dpd","original_string":"DPD | uid:C4O50B3WAUCb2Yw29j id.orig_p:33348 analyzer:HTTP id.resp_p:80 proto:tcp id.orig_h:192.168.15.4 failure_reason:not a http reply line ts:1216702277.477596 id.resp_h:66.33.212.43","ip_dst_addr":"66.33.212.43","ip_src_port":33348,"analyzer":"HTTP","proto":"tcp","guid":"b03d9d34-4a39-4e68-8b21-08bdd532ae07","ip_src_addr":"192.168.15.4","timestamp":1216702277477} +{"bro_timestamp":"1166289883.160785","ip_dst_port":21,"reply_msg":"Entering Passive Mode (192,168,0,193,28,86)","data_channel.orig_h":"192.168.0.114","data_channel.passive":true,"data_channel.resp_p":7254,"command":"PASV","source.type":"bro","uid":"ClOsCM3BUs3saPsD2c","password":"<hidden>","protocol":"ftp","original_string":"FTP | id.orig_p:1137 id.resp_p:21 reply_msg:Entering Passive Mode (192,168,0,193,28,86) data_channel.orig_h:192.168.0.114 data_channel.passive:true data_channel.resp_p:7254 command:PASV uid:ClOsCM3BUs3saPsD2c password:<hidden> data_channel.resp_h:192.168.0.193 id.orig_h:192.168.0.114 user:csanders reply_code:227 ts:1166289883.160785 id.resp_h:192.168.0.193","ip_dst_addr":"192.168.0.193","ip_src_port":1137,"data_channel.resp_h":"192.168.0.193","guid":"4b0c4cda-28ee-404e-b966-036bc7f638ff","user":"csanders","ip_src_addr":"192.168.0.114","reply_code":227,"timestamp":1166289883160} +{"bro_timestamp":"1216706983.387664","timedout":true,"source":"HTTP","is_orig":false,"overflow_bytes":0,"source.type":"bro","duration":30.701792,"protocol":"files","depth":0,"original_string":"FILES | timedout:true rx_hosts:[\"192.168.15.4\"] source:HTTP is_orig:false tx_hosts:[\"216.113.185.92\"] overflow_bytes:0 duration:30.701792 depth:0 analyzers:[\"MD5\",\"SHA1\"] fuid:FnEYba9VPOcC41c1 conn_uids:[\"CLWqoN1IA9MB8Ru9i3\"] seen_bytes:0 missing_bytes:3384 ts:1216706983.387664","ip_dst_addr":"192.168.15.4","analyzers":["MD5","SHA1"],"guid":"7b7148a0-f484-4450-97a3-29493e1c7360","fuid":"FnEYba9VPOcC41c1","conn_uids":["CLWqoN1IA9MB8Ru9i3"],"seen_bytes":0,"missing_bytes":3384,"ip_src_addr":"216.113.185.92","timestamp":1216706983387} +{"bro_timestamp":"1216706999.34818","protocol":"known_certs","original_string":"KNOWN_CERTS | issuer_subject:CN=VeriSign Class 3 Secure Server CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)05,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US serial:24A2DD82DC52358E7F0C6AF6135F3B32 subject:CN=nexus.passport.com,OU=MSN Passport,O=Microsoft,L=Redmond,ST=Washington,C=US port_num:443 host:65.54.179.216 ts:1216706999.34818","issuer_subject":"CN=VeriSign Class 3 Secure Server CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)05,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US","serial":"24A2DD82DC52358E7F0C6AF6135F3B32","subject":"CN=nexus.passport.com,OU=MSN Passport,O=Microsoft,L=Redmond,ST=Washington,C=US","port_num":443,"host":"65.54.179.216","guid":"76fe881c-3ed7-4477-a870-f5381577e4ae","timestamp":1216706999348,"source.type":"bro"} +{"bro_timestamp":"1258568036.57884","ip_dst_port":25,"source.type":"bro","helo":"M57Terry","uid":"ChR6254RrWbrxiGsd7","path":["192.168.1.1","192.168.1.105"],"trans_depth":1,"protocol":"smtp","original_string":"SMTP | id.orig_p:49353 id.resp_p:25 helo:M57Terry uid:ChR6254RrWbrxiGsd7 path:[\"192.168.1.1\",\"192.168.1.105\"] trans_depth:1 is_webmail:false last_reply:220 2.0.0 Ready to start TLS id.orig_h:192.168.1.105 tls:true fuids:[] ts:1258568036.57884 id.resp_h:192.168.1.1","ip_dst_addr":"192.168.1.1","ip_src_port":49353,"is_webmail":false,"last_reply":"220 2.0.0 Ready to start TLS","guid":"9a3d1e86-7d25-4426-b2af-6ab5be1e607f","tls":true,"fuids":[],"ip_src_addr":"192.168.1.105","timestamp":1258568036578} +{"cipher":"TLS_RSA_WITH_RC4_128_MD5","established":true,"server_name":"login.live.com","bro_timestamp":"1216706999.444925","client_cert_chain_fuids":[],"ip_dst_port":443,"subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","cert_chain_fuids":["FkYBO41LPAXxh44KFk","FPrzYN1SuBqHflXZId","FZ71xF13r5XVSam1z1"],"version":"TLSv10","issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US","source.type":"bro","uid":"CVrS2IBW8gukBClA8","protocol":"ssl","original_string":"SSL | cipher:TLS_RSA_WITH_RC4_128_MD5 established:true server_name:login.live.com id.orig_p:36532 client_cert_chain_fuids:[] subject:CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporatio n,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553 id.resp_p:443 cert_chain_fuids:[\"FkYBO41LPAXxh44KFk\",\"FPrzYN1SuBqHflXZId\",\"FZ71xF13r5XVSam1z1\"] version:TLSv10 issuer:CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US uid:CVrS2IBW8gukBClA8 id.orig_h:192.168.15.4 validation_status:unable to get local issuer certificate resumed:false ts:1216706999.444925 id.resp_h:65.54.186.47","ip_dst_addr":"65.54.186.47","ip_src_port":36532,"guid":"1bff79d0-7b86-43de-b5ec-132bb62f4339","validation_status":"unable to get local issuer certificate","resumed":false,"ip_src_addr":"192.168.15.4","timestamp":1216706999444} +{"bro_timestamp":"1216706981.177382","ip_dst_port":80,"source.type":"bro","uid":"Cfxxnt3m0v9SEf5XQ7","protocol":"weird","original_string":"WEIRD | uid:Cfxxnt3m0v9SEf5XQ7 id.orig_p:36446 peer:bro id.resp_p:80 name:unescaped_special_URI_char id.orig_h:192.168.15.4 ts:1216706981.177382 id.resp_h:66.151.146.194 notice:false","ip_dst_addr":"66.151.146.194","ip_src_port":36446,"peer":"bro","name":"unescaped_special_URI_char","guid":"fa2d1068-ca33-4962-b9ab-902605ea3e14","ip_src_addr":"192.168.15.4","notice":false,"timestamp":1216706981177} +{"msg":"SSL certificate validation failed with (unable to get local issuer certificate)","suppress_for":3600.0,"note":"SSL::Invalid_Server_Cert","sub":"CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US","bro_timestamp":"1216706377.196728","dst":"74.125.19.104","ip_dst_port":443,"src":"192.168.15.4","dropped":false,"peer_descr":"bro","source.type":"bro","p":443,"uid":"CNHQmp1mNiZHdAf5Ce","protocol":"notice","original_string":"NOTICE | msg:SSL certificate validation failed with (unable to get local issuer certificate) suppress_for:3600.0 note:SSL::Invalid_Server_Cert sub:CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US id.orig_p:35736 dst:74.125.19.104 src:192.168.15.4 id.resp_p:443 dropped:false peer_descr:bro p:443 uid:CNHQmp1mNiZHdAf5Ce proto:tcp id.orig_h:192.168.15.4 actions:[\"Notice::ACTION_LOG\"] ts:1216706377.196728 id.resp_h:74.125.19.104","ip_dst_addr":"74.125.19.104","ip_src_port":35736,"proto":"tcp","guid":"31e56b6a-48fd-4605-81ec-b0586006 f7d7","actions":["Notice::ACTION_LOG"],"ip_src_addr":"192.168.15.4","timestamp":1216706377196} +{"bro_timestamp":"1258567562.944638","ip_dst_port":67,"trans_id":418901490,"assigned_ip":"192.168.1.103","mac":"00:0b:db:63:5b:d4","source.type":"bro","uid":"CSiO9f3y8Uyu0XprAi","protocol":"dhcp","original_string":"DHCP | uid:CSiO9f3y8Uyu0XprAi id.orig_p:68 lease_time:3564.0 id.resp_p:67 id.orig_h:192.168.1.103 trans_id:418901490 assigned_ip:192.168.1.103 mac:00:0b:db:63:5b:d4 ts:1258567562.944638 id.resp_h:192.168.1.1","ip_dst_addr":"192.168.1.1","ip_src_port":68,"lease_time":3564.0,"guid":"0d2ed5dc-f44c-4d37-b286-7b9f40da420a","ip_src_addr":"192.168.1.103","timestamp":1258567562944} +{"kex_alg":"diffie-hellman-group-exchange-sha256","server":"SSH-2.0-OpenSSH_5.8p1 Debian-7ubuntu1","mac_alg":"hmac-md5","bro_timestamp":"1320435930.914196","auth_success":false,"ip_dst_port":22,"host_key_alg":"ssh-rsa","compression_alg":"none","version":2,"source.type":"bro","uid":"CyrWKo1E1rRywjbOAk","host_key":"87:11:46:da:89:c5:2b:d9:6b:ee:e0:44:7e:73:80:f8","protocol":"ssh","original_string":"SSH | kex_alg:diffie-hellman-group-exchange-sha256 server:SSH-2.0-OpenSSH_5.8p1 Debian-7ubuntu1 mac_alg:hmac-md5 id.orig_p:58435 auth_success:false id.resp_p:22 host_key_alg:ssh-rsa compression_alg:none version:2 uid:CyrWKo1E1rRywjbOAk host_key:87:11:46:da:89:c5:2b:d9:6b:ee:e0:44:7e:73:80:f8 cipher_alg:aes128-ctr client:SSH-2.0-OpenSSH_5.6 id.orig_h:172.16.238.1 ts:1320435930.914196 id.resp_h:172.16.238.136","ip_dst_addr":"172.16.238.136","ip_src_port":58435,"cipher_alg":"aes128-ctr","client":"SSH-2.0-OpenSSH_5.6","guid":"8aebc887-4090-4807-8d65-e841f52b6177","ip_src_addr":"172.16.238.1","t imestamp":1320435930914} +{"bro_timestamp":"1320435464.768382","software_type":"SSH::SERVER","source.type":"bro","unparsed_version":"OpenSSH_5.3","protocol":"software","host_p":22,"original_string":"SOFTWARE | unparsed_version:OpenSSH_5.3 host_p:22 host:172.16.238.168 name:OpenSSH software_type:SSH::SERVER version.major:5 version.minor:3 ts:1320435464.768382","host":"172.16.238.168","name":"OpenSSH","guid":"ad3d1b4b-ffad-4416-be0f-7df08587ccb5","version.major":5,"version.minor":3,"timestamp":1320435464768} +{"bro_timestamp":"1440447766.441298","ip_dst_port":1812,"source.type":"bro","result":"failed","uid":"CqF4zGzBOXFjTWqHh","protocol":"radius","original_string":"RADIUS | result:failed uid:CqF4zGzBOXFjTWqHh id.orig_p:53031 id.resp_p:1812 id.orig_h:127.0.0.1 ts:1440447766.441298 id.resp_h:127.0.0.1 username:steve","ip_dst_addr":"127.0.0.1","ip_src_port":53031,"guid":"b029735a-3e98-45a0-b8da-232967a34085","ip_src_addr":"127.0.0.1","username":"steve","timestamp":1440447766441} +{"certificate.key_length":1024,"bro_timestamp":"1216706999.661483","certificate.sig_alg":"sha1WithRSAEncryption","certificate.not_valid_before":1.2138336E9,"certificate.key_type":"rsa","basic_constraints.ca":false,"certificate.key_alg":"rsaEncryption","certificate.exponent":"65537","source.type":"bro","protocol":"x509","original_string":"X509 | certificate.key_length:1024 certificate.sig_alg:sha1WithRSAEncryption certificate.not_valid_before:1213833600.0 certificate.key_type:rsa basic_constraints.ca:false certificate.key_alg:rsaEncryption certificate.exponent:65537 certificate.version:3 certificate.subject:CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553 id:FkYBO41LPAXxh44KFk certificate.not_valid_after:1248134399.0 certificate.serial:6905C4A47CFDBF9DBC98DACE3 8835FB8 certificate.issuer:CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US ts:1216706999.661483","certificate.version":3,"certificate.subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\\, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","guid":"578eac04-9024-49ab-828d-e25f01c33c82","id":"FkYBO41LPAXxh44KFk","certificate.not_valid_after":1.248134399E9,"certificate.serial":"6905C4A47CFDBF9DBC98DACE38835FB8","certificate.issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https:\/\/www.verisign.com\/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\\, Inc.,C=US","timestamp":1216706999661} +{"bro_timestamp":"1258531221.486539","protocol":"known_devices","original_string":"KNOWN_DEVICES | dhcp_host_name:m57-jo mac:00:0b:db:63:58:a6 ts:1258531221.486539","dhcp_host_name":"m57-jo","guid":"e7a216d8-3623-4dea-af78-01da8c5e0bc5","mac":"00:0b:db:63:58:a6","timestamp":1258531221486,"source.type":"bro"}
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/raw/test.raw ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/raw/test.raw b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/raw/test.raw new file mode 100644 index 0000000..5c88714 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/raw/test.raw @@ -0,0 +1,27 @@ +{"http":{"ts":1402307733.473,"uid":"CTo78A11g7CYbbOHvj","id.orig_h":"192.249.113.37","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"dns":{"ts":1402308259.609,"uid":"CuJT272SKaJSuqO0Ia","id.orig_h":"10.122.196.204","id.orig_p":33976,"id.resp_h":"144.254.71.184","id.resp_p":53,"proto":"udp","trans_id":62418,"query":"www.cisco.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"TTLs":[3600.0,289.0,14.0],"rejected":false}} +{"http":{"ts":1402307733.473,"uid":"KIRAN","id.orig_h":"10.122.196.204","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"http":{"ts":1402307733.473,"uid":"KIRAN12312312","id.orig_h":"192.249.113.37","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"http":{"ts":1402307733.473,"uid":"KIRAN12312312","id.orig_h":"192.249.113.37","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"http":{"ts":1402307733.473,"uid":"CTo78A11g7CYbbOHvj","id.orig_h":"10.122.196.204","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"email":"abul...@mail.csuchico.edu","method":"GET","host":"gabacentre.pw","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"dns":{"ts":1402308259.609,"uid":"CYbbOHvj","id.orig_h":"93.188.160.43","id.orig_p":33976,"id.resp_h":"144.254.71.184","id.resp_p":53,"proto":"udp","trans_id":62418,"query":"www.cisco.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["gabacentre.pw","www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"TTLs":[3600.0,289.0,14.0],"rejected":false}} +{"http":{"ts":1402307733.473,"uid":"CTo78A11g7CYbbOHvj","id.orig_h":"192.249.113.37","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"dns":{"ts":1402308259.609,"uid":"CuJT272SKaJSuqO0Ia","id.orig_h":"10.122.196.204","id.orig_p":33976,"id.resp_h":"144.254.71.184","id.resp_p":53,"proto":"udp","trans_id":62418,"query":"www.cisco.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["www.cisco.com.akadns.net","origin-www.cisco.com","2001:420:1201:2::a"],"TTLs":[3600.0,289.0,14.0],"rejected":false}} +{"http":{"ts":1402307733.473,"uid":"KIRAN","id.orig_h":"10.122.196.204","id.orig_p":58808,"id.resp_h":"72.163.4.161","id.resp_p":80,"trans_depth":1,"method":"GET","host":"www.cisco.com","uri":"/","user_agent":"curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3","request_body_len":0,"response_body_len":25523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJDyMC15lxUn5ngPfd"],"resp_mime_types":["text/html"]}} +{"conn": {"ts":1440447880.931272,"uid":"CWxtRHnBTbldHnmGh","id.orig_h":"127.0.0.1","id.orig_p":52178,"id.resp_h":"127.0.0.1","id.resp_p":1812,"proto":"udp","service":"radius","duration":1.001459,"orig_bytes":75,"resp_bytes":20,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":48,"tunnel_parents":[]}} +{"conn": {"ts":1440447904.122012,"uid":"CK2Oivhlh0ovRcYx","id.orig_h":"127.0.0.1","id.orig_p":62956,"id.resp_h":"127.0.0.1","id.resp_p":1812,"proto":"udp","service":"radius","duration":10.008839,"orig_bytes":225,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":3,"orig_ip_bytes":309,"resp_pkts":0,"resp_ip_bytes":0,"tunnel_parents":[]}} +{"conn": {"ts":1440448190.335333,"uid":"CX6mcO38sO7dkDxK55","id.orig_h":"127.0.0.1","id.orig_p":53127,"id.resp_h":"127.0.0.1","id.resp_p":1812,"proto":"udp","service":"radius","duration":0.000517,"orig_bytes":75,"resp_bytes":71,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":1,"resp_ip_bytes":99,"tunnel_parents":[]}} +{"dpd": {"ts":1216702277.477596,"uid":"C4O50B3WAUCb2Yw29j","id.orig_h":"192.168.15.4","id.orig_p":33348,"id.resp_h":"66.33.212.43","id.resp_p":80,"proto":"tcp","analyzer":"HTTP","failure_reason":"not a http reply line"}} +{"ftp": {"ts":1166289883.160785,"uid":"ClOsCM3BUs3saPsD2c","id.orig_h":"192.168.0.114","id.orig_p":1137,"id.resp_h":"192.168.0.193","id.resp_p":21,"user":"csanders","password":"<hidden>","command":"PASV","reply_code":227,"reply_msg":"Entering Passive Mode (192,168,0,193,28,86)","data_channel.passive":true,"data_channel.orig_h":"192.168.0.114","data_channel.resp_h":"192.168.0.193","data_channel.resp_p":7254}} +{"files": {"ts":1216706983.387664,"fuid":"FnEYba9VPOcC41c1","tx_hosts":["216.113.185.92"],"rx_hosts":["192.168.15.4"],"conn_uids":["CLWqoN1IA9MB8Ru9i3"],"source":"HTTP","depth":0,"analyzers":["MD5","SHA1"],"duration":30.701792,"is_orig":false,"seen_bytes":0,"missing_bytes":3384,"overflow_bytes":0,"timedout":true}} +{"known_certs": {"ts":1216706999.34818,"host":"65.54.179.216","port_num":443,"subject":"CN=nexus.passport.com,OU=MSN Passport,O=Microsoft,L=Redmond,ST=Washington,C=US","issuer_subject":"CN=VeriSign Class 3 Secure Server CA,OU=Terms of use at https://www.verisign.com/rpa (c)05,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US","serial":"24A2DD82DC52358E7F0C6AF6135F3B32"}} +{"smtp": {"ts":1258568036.57884,"uid":"ChR6254RrWbrxiGsd7","id.orig_h":"192.168.1.105","id.orig_p":49353,"id.resp_h":"192.168.1.1","id.resp_p":25,"trans_depth":1,"helo":"M57Terry","last_reply":"220 2.0.0 Ready to start TLS","path":["192.168.1.1","192.168.1.105"],"tls":true,"fuids":[],"is_webmail":false}} +{"ssl": {"ts":1216706999.444925,"uid":"CVrS2IBW8gukBClA8","id.orig_h":"192.168.15.4","id.orig_p":36532,"id.resp_h":"65.54.186.47","id.resp_p":443,"version":"TLSv10","cipher":"TLS_RSA_WITH_RC4_128_MD5","server_name":"login.live.com","resumed":false,"established":true,"cert_chain_fuids":["FkYBO41LPAXxh44KFk","FPrzYN1SuBqHflXZId","FZ71xF13r5XVSam1z1"],"client_cert_chain_fuids":[],"subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\u005c, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US","validation_status":"unable to get local issuer certificate"}} +{"weird": {"ts":1216706981.177382,"uid":"Cfxxnt3m0v9SEf5XQ7","id.orig_h":"192.168.15.4","id.orig_p":36446,"id.resp_h":"66.151.146.194","id.resp_p":80,"name":"unescaped_special_URI_char","notice":false,"peer":"bro"}} +{"notice": {"ts":1216706377.196728,"uid":"CNHQmp1mNiZHdAf5Ce","id.orig_h":"192.168.15.4","id.orig_p":35736,"id.resp_h":"74.125.19.104","id.resp_p":443,"proto":"tcp","note":"SSL::Invalid_Server_Cert","msg":"SSL certificate validation failed with (unable to get local issuer certificate)","sub":"CN=www.google.com,O=Google Inc,L=Mountain View,ST=California,C=US","src":"192.168.15.4","dst":"74.125.19.104","p":443,"peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false}} +{"dhcp": {"ts":1258567562.944638,"uid":"CSiO9f3y8Uyu0XprAi","id.orig_h":"192.168.1.103","id.orig_p":68,"id.resp_h":"192.168.1.1","id.resp_p":67,"mac":"00:0b:db:63:5b:d4","assigned_ip":"192.168.1.103","lease_time":3564.0,"trans_id":418901490}} +{"ssh": {"ts":1320435930.914196,"uid":"CyrWKo1E1rRywjbOAk","id.orig_h":"172.16.238.1","id.orig_p":58435,"id.resp_h":"172.16.238.136","id.resp_p":22,"version":2,"auth_success":false,"client":"SSH-2.0-OpenSSH_5.6","server":"SSH-2.0-OpenSSH_5.8p1 Debian-7ubuntu1","cipher_alg":"aes128-ctr","mac_alg":"hmac-md5","compression_alg":"none","kex_alg":"diffie-hellman-group-exchange-sha256","host_key_alg":"ssh-rsa","host_key":"87:11:46:da:89:c5:2b:d9:6b:ee:e0:44:7e:73:80:f8"}} +{"software": {"ts":1320435464.768382,"host":"172.16.238.168","host_p":22,"software_type":"SSH::SERVER","name":"OpenSSH","version.major":5,"version.minor":3,"unparsed_version":"OpenSSH_5.3"}} +{"radius": {"ts":1440447766.441298,"uid":"CqF4zGzBOXFjTWqHh","id.orig_h":"127.0.0.1","id.orig_p":53031,"id.resp_h":"127.0.0.1","id.resp_p":1812,"username":"steve","result":"failed"}} +{"x509": {"ts":1216706999.661483,"id":"FkYBO41LPAXxh44KFk","certificate.version":3,"certificate.serial":"6905C4A47CFDBF9DBC98DACE38835FB8","certificate.subject":"CN=login.live.com,OU=MSN-Passport,O=Microsoft Corporation,street=One Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=V1.0\u005c, Clause 5.(b),1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553","certificate.issuer":"CN=VeriSign Class 3 Extended Validation SSL CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\u005c, Inc.,C=US","certificate.not_valid_before":1213833600.0,"certificate.not_valid_after":1248134399.0,"certificate.key_alg":"rsaEncryption","certificate.sig_alg":"sha1WithRSAEncryption","certificate.key_type":"rsa","certificate.key_length":1024,"certificate.exponent":"65537","basic_constraints.ca":false}} +{"known_devices": {"ts":1258531221.486539,"mac":"00:0b:db:63:58:a6","dhcp_host_name":"m57-jo"}} http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/log4j.properties b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/log4j.properties new file mode 100644 index 0000000..27263f7 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/log4j.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root logger option +log4j.rootLogger=ERROR, stdout +log4j.logger.org.apache.storm.daemon=FATAL, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +log4j.appender.stdout.filter.1=org.apache.log4j.varia.StringMatchFilter +log4j.appender.stdout.filter.1.StringToMatch=Connection timed out +log4j.appender.stdout.filter.1.AcceptOnMatch=false +log4j.appender.stdout.filter.2=org.apache.log4j.varia.StringMatchFilter +log4j.appender.stdout.filter.2.StringToMatch=Background +log4j.appender.stdout.filter.2.AcceptOnMatch=false +log4j.appender.stdout.filter.3=org.apache.log4j.varia.StringMatchFilter +log4j.appender.stdout.filter.3.StringToMatch=Error when handling request +log4j.appender.stdout.filter.3.AcceptOnMatch=false http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/pom.xml new file mode 100644 index 0000000..09ed864 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/pom.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions</artifactId> + <version>0.4.1</version> + </parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-bro-extension</artifactId> + <name>metron-parser-bro-extension</name> + <version>0.4.1</version> + <packaging>pom</packaging> + + + <description>Bro Parser Extension for Metron</description> + <modules> + <module>metron-parser-bro</module> + <module>metron-parser-bro-bundle</module> + <module>metron-parser-bro-assembly</module> + </modules> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/pom.xml new file mode 100644 index 0000000..cc5bcfc --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/pom.xml @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions</artifactId> + <version>0.4.1</version> + </parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-bundle-tests</artifactId> + <packaging>jar</packaging> + <name>metron-parser-bundle-tests</name> + <version>0.4.1</version> + <description>Parser Extension bundle tests</description> + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parsers</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>bundles-lib</artifactId> + <version>0.4.1</version> + </dependency> + <!-- testing --> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions-testing</artifactId> + <version>${project.parent.version}</version> + <type>pom</type> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/java/org/apache/metron/parsers/ASABundleHDFSIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/java/org/apache/metron/parsers/ASABundleHDFSIntegrationTest.java b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/java/org/apache/metron/parsers/ASABundleHDFSIntegrationTest.java new file mode 100644 index 0000000..a6bb440 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/java/org/apache/metron/parsers/ASABundleHDFSIntegrationTest.java @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers; + + +import com.google.common.base.Function; +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.metron.TestConstants; +import org.apache.metron.bundles.BundleClassLoaders; +import org.apache.metron.bundles.util.BundleProperties; +import org.apache.metron.common.Constants; +import org.apache.metron.enrichment.integration.components.ConfigUploadComponent; +import org.apache.metron.integration.BaseIntegrationTest; +import org.apache.metron.integration.ComponentRunner; +import org.apache.metron.integration.ProcessorResult; +import org.apache.metron.integration.components.KafkaComponent; +import org.apache.metron.integration.components.MRComponent; +import org.apache.metron.integration.components.ZKServerComponent; +import org.apache.metron.integration.processors.KafkaMessageSet; +import org.apache.metron.integration.processors.KafkaProcessor; +import org.apache.metron.integration.utils.TestUtils; +import org.apache.metron.parsers.integration.ParserValidation; +import org.apache.metron.parsers.integration.components.ParserTopologyComponent; +import org.apache.metron.parsers.integration.validation.PathedSampleDataValidation; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +public class ASABundleHDFSIntegrationTest extends BaseIntegrationTest { + static final Map<String,String> EMPTY_MAP = new HashMap<String,String>(); + static final String sensorType = "asa"; + static final String ERROR_TOPIC = "parser_error"; + protected List<byte[]> inputMessages; + @AfterClass + public static void after(){ + try { + RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/"), true); + System.out.println("==============(AFTER)=============="); + while (files.hasNext()) { + LocatedFileStatus fileStat = files.next(); + System.out.println(fileStat.getPath().toString()); + } + }catch(Exception e){} + mrComponent.stop(); + BundleClassLoaders.reset(); + } + + static MRComponent mrComponent; + static Configuration configuration; + static FileSystem fileSystem; + @BeforeClass + public static void setup() { + mrComponent = new MRComponent().withBasePath("target/hdfs"); + mrComponent.start(); + configuration = mrComponent.getConfiguration(); + + try { + + // copy the correct things in + copyResources("./src/test/resources","./target/remote"); + + // we need to patch the properties file + BundleProperties properties = BundleProperties.createBasicBundleProperties("./target/remote/zookeeper/bundle.properties",new HashMap<>()); + String hdfsPrefix = configuration.get("fs.defaultFS"); + properties.setProperty(BundleProperties.BUNDLE_LIBRARY_DIRECTORY, hdfsPrefix + "/extension_lib/"); + properties.setProperty(BundleProperties.BUNDLE_LIBRARY_DIRECTORY_PREFIX + "alt", hdfsPrefix + "/extension_contrib_lib/"); + FileOutputStream fso = new FileOutputStream("./target/remote/zookeeper/bundle.properties"); + properties.storeProperties(fso,"HDFS UPDATE"); + fso.flush(); + fso.close(); + + fileSystem = FileSystem.newInstance(configuration); + if(!fileSystem.mkdirs(new Path("/work/"),new FsPermission(FsAction.READ_WRITE,FsAction.READ_WRITE,FsAction.READ_WRITE))){ + System.out.println("FAILED MAKE DIR"); + } + fileSystem.copyFromLocalFile(new Path("./target/remote/metron/extension_contrib_lib/"), new Path("/")); + fileSystem.copyFromLocalFile(new Path("./target/remote/metron/extension_lib/"), new Path("/")); + fileSystem.copyFromLocalFile(new Path("./target/remote/zookeeper/bundle.properties"), new Path("/work/")); + + RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/"),true); + System.out.println("==============(BEFORE)=============="); + while (files.hasNext()){ + LocatedFileStatus fileStat = files.next(); + System.out.println(fileStat.getPath().toString()); + } + + } catch (IOException e) { + throw new RuntimeException("Unable to start cluster", e); + } + + } + public static void copyResources(String source, String target) throws IOException { + final java.nio.file.Path sourcePath = Paths.get(source); + final java.nio.file.Path targetPath = Paths.get(target); + + Files.walkFileTree(sourcePath, new SimpleFileVisitor<java.nio.file.Path>() { + + @Override + public FileVisitResult preVisitDirectory(java.nio.file.Path dir, BasicFileAttributes attrs) + throws IOException { + + java.nio.file.Path relativeSource = sourcePath.relativize(dir); + java.nio.file.Path target = targetPath.resolve(relativeSource); + + if(!Files.exists(target)) { + Files.createDirectories(target); + } + return FileVisitResult.CONTINUE; + + } + + @Override + public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs) + throws IOException { + + java.nio.file.Path relativeSource = sourcePath.relativize(file); + java.nio.file.Path target = targetPath.resolve(relativeSource); + + Files.copy(file, target, REPLACE_EXISTING); + + return FileVisitResult.CONTINUE; + } + }); + } + + @Test + public void testHDFS() throws Exception{ + final Properties topologyProperties = new Properties(); + inputMessages = TestUtils.readSampleData(getSampleDataPath()); + final KafkaComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaComponent.Topic>() {{ + add(new KafkaComponent.Topic(sensorType, 1)); + add(new KafkaComponent.Topic(Constants.ENRICHMENT_TOPIC, 1)); + add(new KafkaComponent.Topic(ERROR_TOPIC,1)); + }}); + topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList()); + + ZKServerComponent zkServerComponent = getZKServerComponent(topologyProperties); + + ConfigUploadComponent configUploadComponent = new ConfigUploadComponent() + .withTopologyProperties(topologyProperties) + .withGlobalConfigsPath("./target/remote/zookeeper/") + .withParserConfigsPath("../metron-parser-asa-extension/metron-parser-asa/" + TestConstants.THIS_PARSER_CONFIGS_PATH); + + ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder() + .withSensorType(sensorType) + .withTopologyProperties(topologyProperties) + .withBrokerUrl(kafkaComponent.getBrokerList()).build(); + + //UnitTestHelper.verboseLogging(); + ComponentRunner runner = new ComponentRunner.Builder() + .withComponent("zk", zkServerComponent) + .withComponent("kafka", kafkaComponent) + .withComponent("config", configUploadComponent) + .withComponent("org/apache/storm", parserTopologyComponent) + .withMillisecondsBetweenAttempts(5000) + .withNumRetries(10) + .withCustomShutdownOrder(new String[] {"org/apache/storm","config","kafka","zk"}) + .build(); + runner.start(); + + try { + kafkaComponent.writeMessages(sensorType, inputMessages); + ProcessorResult<List<byte[]>> result = runner.process(getProcessor()); + List<byte[]> outputMessages = result.getResult(); + StringBuffer buffer = new StringBuffer(); + if (result.failed()){ + result.getBadResults(buffer); + buffer.append(String.format("%d Valid Messages Processed", outputMessages.size())).append("\n"); + dumpParsedMessages(outputMessages,buffer); + Assert.fail(buffer.toString()); + } else { + List<ParserValidation> validations = getValidations(); + if (validations == null || validations.isEmpty()) { + buffer.append("No validations configured for sensorType " + sensorType + ". Dumping parsed messages").append("\n"); + dumpParsedMessages(outputMessages,buffer); + Assert.fail(buffer.toString()); + } else { + for (ParserValidation validation : validations) { + System.out.println("Running " + validation.getName() + " on sensorType " + sensorType); + validation.validate(sensorType, outputMessages); + } + } + } + } finally { + runner.stop(); + } + } + + public void dumpParsedMessages(List<byte[]> outputMessages, StringBuffer buffer) { + for (byte[] outputMessage : outputMessages) { + buffer.append(new String(outputMessage)).append("\n"); + } + } + + @SuppressWarnings("unchecked") + private KafkaProcessor<List<byte[]>> getProcessor(){ + + return new KafkaProcessor<>() + .withKafkaComponentName("kafka") + .withReadTopic(Constants.ENRICHMENT_TOPIC) + .withErrorTopic(ERROR_TOPIC) + .withValidateReadMessages(new Function<KafkaMessageSet, Boolean>() { + @Nullable + @Override + public Boolean apply(@Nullable KafkaMessageSet messageSet) { + return (messageSet.getMessages().size() + messageSet.getErrors().size() == inputMessages.size()); + } + }) + .withProvideResult(new Function<KafkaMessageSet,List<byte[]>>(){ + @Nullable + @Override + public List<byte[]> apply(@Nullable KafkaMessageSet messageSet) { + return messageSet.getMessages(); + } + }); + } + + public List<ParserValidation> getValidations() { + return new ArrayList<ParserValidation>() {{ + add(new PathedSampleDataValidation("../metron-parser-asa-extension/metron-parser-asa/src/test/resources/data/parsed/test.parsed")); + }}; + } + + protected String getGlobalConfigPath() throws Exception{ + return "../../../../metron-integration-test/src/main/config/zookeeper/"; + } + + protected String getSampleDataPath() throws Exception { + return "../metron-parser-asa-extension/metron-parser-asa/src/test/resources/data/raw/test.raw"; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_contrib_lib/metron-parser-test-bundle-1.0-SNAPSHOT.bundle ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_contrib_lib/metron-parser-test-bundle-1.0-SNAPSHOT.bundle b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_contrib_lib/metron-parser-test-bundle-1.0-SNAPSHOT.bundle new file mode 100644 index 0000000..3ebb8f7 Binary files /dev/null and b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_contrib_lib/metron-parser-test-bundle-1.0-SNAPSHOT.bundle differ http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_lib/metron-parser-asa-bundle-0.4.0.bundle ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_lib/metron-parser-asa-bundle-0.4.0.bundle b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_lib/metron-parser-asa-bundle-0.4.0.bundle new file mode 100644 index 0000000..306bb44 Binary files /dev/null and b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/metron/extension_lib/metron-parser-asa-bundle-0.4.0.bundle differ http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/bundle.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/bundle.properties b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/bundle.properties new file mode 100644 index 0000000..3b34f54 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/bundle.properties @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +bundle.library.directory=./target/local/metron/extension_lib/ +bundle.library.directory.alt=./target/local/metron/extension_contrib_lib/ +bundle.working.directory=./target/local/metron/work/bundle/ +bundle.documentation.working.directory=./target/local/metron/work/docs/components/ +bundle.archive.extension=bundle +bundle.meta.id.prefix=Bundle +bundle.extension.type.MessageParser=org.apache.metron.parsers.interfaces.MessageParser http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/enrichments/test.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/enrichments/test.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/enrichments/test.json new file mode 100644 index 0000000..1037b69 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/enrichments/test.json @@ -0,0 +1,75 @@ +{ + "enrichment": { + "fieldMap": { + "geo": [ + "ip_src_addr", + "ip_dst_addr" + ], + "host": [ + "ip_src_addr", + "ip_dst_addr" + ], + "hbaseEnrichment": [ + "ip_src_addr", + "ip_dst_addr" + ], + "stellar" : { + "config" : { + "numeric" : { + "foo": "1 + 1" + } + ,"ALL_CAPS" : "TO_UPPER(source.type)" + ,"src_enrichment" : { + "src_classification" : "ENRICHMENT_GET('playful_classification', ip_src_addr, 'enrichments', 'cf')" + } + ,"dst_enrichment" : { + "dst_classification" : "ENRICHMENT_GET('playful_classification', ip_dst_addr, 'enrichments', 'cf')" + } + } + } + } + ,"fieldToTypeMap": { + "ip_src_addr": [ + "playful_classification" + ], + "ip_dst_addr": [ + "playful_classification" + ] + } + }, + "threatIntel": { + "fieldMap": { + "hbaseThreatIntel": [ + "ip_src_addr", + "ip_dst_addr" + ], + "stellar" : { + "config" : { + "bar" : "TO_UPPER(source.type)" + ,"is_src_malicious" : "ENRICHMENT_EXISTS('malicious_ip', ip_src_addr, 'threat_intel', 'cf')" + } + } + }, + "fieldToTypeMap": { + "ip_src_addr": [ + "malicious_ip" + ], + "ip_dst_addr": [ + "malicious_ip" + ] + }, + "triageConfig" : { + "riskLevelRules" : [ + { + "name" : "The name of the triage rule", + "comment" : "A description of the triage rule", + "rule" : "ip_src_addr == '10.0.2.3' or ip_dst_addr == '10.0.2.3'", + "score": 10, + "reason": "'Reason field'" + } + ], + "aggregator" : "MAX" + } + } +} + http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/global.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/global.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/global.json new file mode 100644 index 0000000..8d3005f --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/global.json @@ -0,0 +1,27 @@ +{ + "es.clustername": "metron", + "es.ip": "localhost", + "es.port": 9300, + "es.date.format": "yyyy.MM.dd.HH", + + "solr.zookeeper": "localhost:2181", + "solr.collection": "metron", + "solr.numShards": 1, + "solr.replicationFactor": 1, + + "fieldValidations" : [ + { + "input" : [ "ip_src_addr", "ip_dst_addr"], + "validation" : "IP" + } + ], + + "profiler.client.period.duration": "15", + "profiler.client.period.duration.units": "MINUTES", + "profiler.client.hbase.table": "profiler", + "profiler.client.hbase.column.family": "P", + "profiler.client.salt.divisor": "1000", + "hbase.provider.impl": "org.apache.metron.hbase.HTableProvider", + + "geo.hdfs.file": "src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/indexing/test.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/indexing/test.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/indexing/test.json new file mode 100644 index 0000000..0197f0c --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bundle-tests/src/test/resources/zookeeper/indexing/test.json @@ -0,0 +1,18 @@ +{ + "hdfs" : { + "index": "yaf", + "batchSize": 5, + "enabled" : true + }, + "elasticsearch" : { + "index": "yaf", + "batchSize": 5, + "enabled" : true + }, + "solr" : { + "index": "yaf", + "batchSize": 5, + "enabled" : true + } +} + http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/pom.xml new file mode 100644 index 0000000..5c2e2ed --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/pom.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-cef-extension</artifactId> + <version>0.4.1</version> + </parent> + + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-cef-assembly</artifactId> + <version>0.4.1</version> + <packaging>pom</packaging> + <name>metron-parser-cef-assembly</name> + + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptor>src/main/assembly/assembly.xml</descriptor> + </configuration> + <executions> + <execution> + <id>make-assembly</id> <!-- this is used for inheritance merges --> + <phase>package</phase> <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/src/main/assembly/assembly.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/src/main/assembly/assembly.xml new file mode 100644 index 0000000..9171424 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-assembly/src/main/assembly/assembly.xml @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<assembly> + <id>archive</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${project.basedir}/../metron-parser-cef/src/main/config</directory> + <outputDirectory>/config</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + <excludes> + <exclude>**/*.formatted</exclude> + <exclude>**/*.filtered</exclude> + </excludes> + <fileMode>0644</fileMode> + <lineEnding>unix</lineEnding> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${project.basedir}/../metron-parser-cef-bundle/target</directory> + <includes> + <include>metron-parser-cef-bundle-${project.version}.bundle</include> + </includes> + <outputDirectory>/lib</outputDirectory> + <useDefaultExcludes>true</useDefaultExcludes> + </fileSet> + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-bundle/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-bundle/pom.xml new file mode 100644 index 0000000..9912928 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef-bundle/pom.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-cef-extension</artifactId> + <version>0.4.1</version> + </parent> + + <artifactId>metron-parser-cef-bundle</artifactId> + <version>0.4.1</version> + <name>metron-parser-cef-bundle</name> + <packaging>bundle</packaging> + <properties> + <maven.javadoc.skip>true</maven.javadoc.skip> + <source.skip>false</source.skip> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-cef</artifactId> + <version>0.4.1</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/README.md b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/README.md new file mode 100644 index 0000000..c2d0135 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/README.md @@ -0,0 +1,3 @@ +# CEF Parser + +This is the CEF parser http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/pom.xml new file mode 100644 index 0000000..038ef50 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/pom.xml @@ -0,0 +1,89 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-cef-extension</artifactId> + <version>0.4.1</version> + </parent> + <artifactId>metron-parser-cef</artifactId> + <version>0.4.1</version> + <name>metron-parser-cef</name> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parsers</artifactId> + <version>${project.parent.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.atteo.classindex</groupId> + <artifactId>classindex</artifactId> + <version>${global_classindex_version}</version> + <scope>provided</scope> + </dependency> + <!-- testing --> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions-testing</artifactId> + <version>${project.parent.version}</version> + <type>pom</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.github.fge</groupId> + <artifactId>json-schema-validator</artifactId> + <version>2.2.6</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>${global_jar_version}</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + <resource> + <directory>src/main/patterns</directory> + </resource> + <resource> + <directory>src/test/resources</directory> + </resource> + </resources> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/enrichments/cef.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/enrichments/cef.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/enrichments/cef.json new file mode 100644 index 0000000..54fde2b --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/enrichments/cef.json @@ -0,0 +1,19 @@ +{ + "enrichment" : { + "fieldMap": { + "geo": [ + "ip_dst_addr", + "ip_src_addr" + ] + } + }, + "threatIntel": { + "fieldMap": { + "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"] + }, + "fieldToTypeMap": { + "ip_src_addr" : ["malicious_ip"], + "ip_dst_addr" : ["malicious_ip"] + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/indexing/cef.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/indexing/cef.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/indexing/cef.json new file mode 100644 index 0000000..e1e9053 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/indexing/cef.json @@ -0,0 +1,18 @@ +{ + "hdfs" : { + "index": "cef", + "batchSize": 5, + "enabled" : true + }, + "elasticsearch" : { + "index": "cef", + "batchSize": 5, + "enabled" : true + }, + "solr" : { + "index": "cef", + "batchSize": 5, + "enabled" : true + } +} + http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/parsers/cef.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/parsers/cef.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/parsers/cef.json new file mode 100644 index 0000000..8bacbbd --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-cef-extension/metron-parser-cef/src/main/config/zookeeper/parsers/cef.json @@ -0,0 +1,7 @@ +{ + "parserClassName": "org.apache.metron.parsers.cef.CEFParser", + "sensorTopic": "cef", + "parserConfig": { + "deviceTimeZone": "UTC" + } +}
