http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/indexed/test.indexed ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/indexed/test.indexed b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/indexed/test.indexed new file mode 100644 index 0000000..d48fa46 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/indexed/test.indexed @@ -0,0 +1,10 @@ +{"adapter.threatinteladapter.end.ts":"1457102731219","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa001","index.elasticsearchwriter.ts":"1457102731220","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731206","adapter.hostfromjsonlistadapter.begin.ts":"1457102731185","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":44,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731185","threatintelsplitterbolt.splitter.ts":"1457102731207","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512, "adapter.threatinteladapter.begin.ts":"1457102731210","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AS","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731220","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.ho st.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":10000000,"index.elasticsearchwriter.ts":"1457102731221","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731208","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitt er.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988502,"adapter.threatinteladapter.begin.ts":"1457102731219","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731221","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":37299,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latit ude":"test latitude","timestamp":1453994988502,"risn":0,"end_time":1453994988502,"is_alert":"true","source.type":"yaf","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":37299,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":312,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter .threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988504,"enrichments.host.dip.known_i nfo.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts": "1457102731211","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":56303,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"t est latitude","timestamp":1453994988504,"risn":0,"end_time":1453994988504,"is_alert":"true","source.type":"yaf","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":56303,"rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":84,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988506,"adapter. threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988506,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988506,"enrichments.host.dip.known_in fo.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fca","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":60,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbol t.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988508,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"S","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":145399 4988508,"risn":0,"end_time":1453994988508,"source.type":"yaf","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterb olt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453 994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":148,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitter bolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":14 53994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731225","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":145399498851 2,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.h ost.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"} +{"adapter.threatinteladapter.end.ts":"1457102731226","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":604,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731213","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988562 ,"adapter.threatinteladapter.begin.ts":"1457102731226","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731226","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988562,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988562,"enrichments.h ost.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/parsed/test.parsed ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/parsed/test.parsed b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/parsed/test.parsed new file mode 100644 index 0000000..6ee2b2f --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/parsed/test.parsed @@ -0,0 +1,10 @@ +{"iflags":"AS","uflags":0,"isn":"22efa001","ip_dst_addr":"10.0.2.15","ip_dst_port":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"216.21.170.221","tag":0,"rtag":0,"ip_src_port":80,"timestamp":1453994988512,"app":0,"oct":44,"end_reason":"idle","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":"TCP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"A","uflags":0,"isn":10000000,"ip_dst_addr":"10.0.2.3","ip_dst_port":53,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":37299,"timestamp":1453994988502,"app":0,"oct":56,"end_reason":"idle","risn":0,"end_time":1453994988502,"source.type":"yaf","start_time":1453994988502,"riflags":0,"rtt":"0.000","protocol":"UDP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"A","uflags":0,"isn":0,"ip_dst_addr":"10.0.2.15","ip_dst_port":37299,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.3","tag":0,"rtag":0,"ip_src_port":53,"timestamp":1453994988504,"app":0,"oct":312,"end_reason":"idle","risn":0,"end_time":1453994988504,"source.type":"yaf","start_time":1453994988504,"riflags":0,"rtt":"0.000","protocol":"UDP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"A","uflags":0,"isn":0,"ip_dst_addr":"10.0.2.3","ip_dst_port":53,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":56303,"timestamp":1453994988504,"app":0,"oct":56,"end_reason":"idle","risn":0,"end_time":1453994988504,"source.type":"yaf","start_time":1453994988504,"riflags":0,"rtt":"0.000","protocol":"UDP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"A","uflags":0,"isn":0,"ip_dst_addr":"10.0.2.15","ip_dst_port":56303,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.3","tag":0,"rtag":0,"ip_src_port":53,"timestamp":1453994988506,"app":0,"oct":84,"end_reason":"idle","risn":0,"end_time":1453994988506,"source.type":"yaf","start_time":1453994988506,"riflags":0,"rtt":"0.000","protocol":"UDP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"S","uflags":0,"isn":"58c52fca","ip_dst_addr":"216.21.170.221","ip_dst_port":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":39468,"timestamp":1453994988508,"app":0,"oct":60,"end_reason":"idle","risn":0,"end_time":1453994988508,"source.type":"yaf","start_time":1453994988508,"riflags":0,"rtt":"0.000","protocol":"TCP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"A","uflags":0,"isn":"58c52fcb","ip_dst_addr":"216.21.170.221","ip_dst_port":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":39468,"timestamp":1453994988512,"app":0,"oct":40,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":"TCP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"AP","uflags":0,"isn":"58c52fcb","ip_dst_addr":"216.21.170.221","ip_dst_port":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":39468,"timestamp":1453994988512,"app":0,"oct":148,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":"TCP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"A","uflags":0,"isn":"22efa002","ip_dst_addr":"10.0.2.15","ip_dst_port":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"216.21.170.221","tag":0,"rtag":0,"ip_src_port":80,"timestamp":1453994988512,"app":0,"oct":40,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":"TCP","guid":"this-is-random-uuid-will-be-36-chars"} +{"iflags":"AP","uflags":0,"isn":"22efa002","ip_dst_addr":"10.0.2.15","ip_dst_port":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"216.21.170.221","tag":0,"rtag":0,"ip_src_port":80,"timestamp":1453994988562,"app":0,"oct":604,"end_reason":"idle","risn":0,"end_time":1453994988562,"source.type":"yaf","start_time":1453994988562,"riflags":0,"rtt":"0.000","protocol":"TCP","guid":"this-is-random-uuid-will-be-36-chars"} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/raw/test.raw ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/raw/test.raw b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/raw/test.raw new file mode 100644 index 0000000..8f3ff44 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/data/raw/test.raw @@ -0,0 +1,10 @@ +2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle +2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle +2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle +2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle +2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle +2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle +2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle +2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle +2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle +2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/log4j.properties b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/log4j.properties new file mode 100644 index 0000000..27263f7 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/metron-parser-yaf/src/test/resources/log4j.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root logger option +log4j.rootLogger=ERROR, stdout +log4j.logger.org.apache.storm.daemon=FATAL, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +log4j.appender.stdout.filter.1=org.apache.log4j.varia.StringMatchFilter +log4j.appender.stdout.filter.1.StringToMatch=Connection timed out +log4j.appender.stdout.filter.1.AcceptOnMatch=false +log4j.appender.stdout.filter.2=org.apache.log4j.varia.StringMatchFilter +log4j.appender.stdout.filter.2.StringToMatch=Background +log4j.appender.stdout.filter.2.AcceptOnMatch=false +log4j.appender.stdout.filter.3=org.apache.log4j.varia.StringMatchFilter +log4j.appender.stdout.filter.3.StringToMatch=Error when handling request +log4j.appender.stdout.filter.3.AcceptOnMatch=false http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/pom.xml new file mode 100644 index 0000000..26c1509 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-yaf-extension/pom.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions</artifactId> + <version>0.4.1</version> + </parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-yaf-extension</artifactId> + <name>metron-parser-yaf-extension</name> + <version>0.4.1</version> + <packaging>pom</packaging> + + + <description>Yaf Parser Extension for Metron</description> + <modules> + <module>metron-parser-yaf</module> + <!-- YAF IS CONFIGURATION ONLY AT THIS TIME + <module>metron-parser-yaf-bundle</module> + --> + <module>metron-parser-yaf-assembly</module> + </modules> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_deployment.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_deployment.md b/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_deployment.md new file mode 100644 index 0000000..4ff4ea9 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_deployment.md @@ -0,0 +1,240 @@ +# Metron Parser Extension Deployment + +Metron Parser Extensions have the common extension deployment of the [Metron Bundles](../../bundles-lib) containing the runtime +library and parser dependencies. On top of this, parser extensions deploy the parser configurations required +to integrate into the Metron system. + +## Bundle Deployment + +These Bundles are deployed to HDFS under /apps/metron/extension_lib for system parsers ( parsers that are built and deployed with Metron itself). +Parser extensions created and managed outside the project have their bundles deployed to extension_alt_lib. + +The bundles are loaded by Apache VFS as a composite file system, at which time they will be cached locally. + +> NOTE: Bundles may also be deployed locally on the cluster under /usr/metron/VERSION/ + +```bash + drwxrwxr-x - metron hadoop 0 2017-06-27 16:15 /apps/metron/extension_lib + drwxrwxr-x - metron hadoop 0 2017-06-27 16:15 /apps/metron/extension_alt_lib +``` +The system parsers bundles being deployed as such: +```bash + [root@node1 0.4.0]# hadoop fs -ls /apps/metron/extension_lib +Found 11 items +-rwxr-xr-x 1 hdfs hdfs 39809 2017-06-27 16:15 /apps/metron/extension_lib/metron-parser-asa-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 27377 2017-06-27 16:15 /apps/metron/extension_lib/metron-parser-bro-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 26947 2017-06-27 16:15 /apps/metron/extension_lib/metron-parser-cef-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 24758 2017-06-27 16:14 /apps/metron/extension_lib/metron-parser-fireeye-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 76150 2017-06-27 16:14 /apps/metron/extension_lib/metron-parser-ise-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 21904 2017-06-27 16:14 /apps/metron/extension_lib/metron-parser-lancope-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 20624 2017-06-27 16:15 /apps/metron/extension_lib/metron-parser-logstash-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 23546 2017-06-27 16:14 /apps/metron/extension_lib/metron-parser-paloalto-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 23972 2017-06-27 16:14 /apps/metron/extension_lib/metron-parser-snort-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 24230 2017-06-27 16:14 /apps/metron/extension_lib/metron-parser-sourcefire-bundle-0.4.0.bundle +-rwxr-xr-x 1 hdfs hdfs 23790 2017-06-27 16:14 /apps/metron/extension_lib/metron-parser-websphere-bundle-0.4.0.bundle +``` + +## Configuration Deployment + +- System Parser configurations are deployed to disk on Metron Parser cluster nodes, and are then deployed +to Zookeeper by the Ambari service installation. + +Configurations for each parser are deployed to /usr/metron/VERSION/extension_etc/PARSERNAME. +The configurations include + +- zookeeper +Configurations to be loaded into zookeeper, including configurations for enrichments (if present), indexing and parsers. + +- patterns +GROK patterns by the parser + +### On disk +```bash +extension_etc +âââ parsers + âââ asa + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ asa.json + â  â  âââ indexing + â  â  â  âââ asa.json + â  â  âââ parsers + â  â  âââ asa.json + â  âââ patterns + â  âââ asa + â  âââ common + âââ bro + â  âââ config + â  â  âââ elasticsearch + â  â  â  âââ bro_index.template + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ bro.json + â  â  âââ indexing + â  â  â  âââ bro.json + â  â  âââ parsers + â  â  âââ bro.json + âââ cef + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ cef.json + â  â  âââ indexing + â  â  â  âââ cef.json + â  â  âââ parsers + â  â  âââ cef.json + âââ fireeye + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ fireeye.json + â  â  âââ indexing + â  â  â  âââ fireeye.json + â  â  âââ parsers + â  â  âââ fireeye.json + â  âââ patterns + â  âââ common + â  âââ fireeye + âââ ise + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ ise.json + â  â  âââ indexing + â  â  â  âââ ise.json + â  â  âââ parsers + â  â  âââ ise.json + âââ lancope + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ lancope.json + â  â  âââ indexing + â  â  â  âââ lancope.json + â  â  âââ parsers + â  â  âââ lancope.json + âââ logstash + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ logstash.json + â  â  âââ indexing + â  â  â  âââ logstash.json + â  â  âââ parsers + â  â  âââ logstash.json + âââ paloalto + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ paloalto.json + â  â  âââ indexing + â  â  â  âââ paloalto.json + â  â  âââ parsers + â  â  âââ paloalto.json + âââ snort + â  âââ config + â  â  âââ elasticsearch + â  â  â  âââ snort_index.template + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ snort.json + â  â  âââ indexing + â  â  â  âââ snort.json + â  â  âââ parsers + â  â  âââ snort.json + âââ sourcefire + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ sourcefire.json + â  â  âââ indexing + â  â  â  âââ sourcefire.json + â  â  âââ parsers + â  â  âââ sourcefire.json + â  âââ patterns + â  âââ common + â  âââ sourcefire + âââ squid + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ squid.json + â  â  âââ indexing + â  â  â  âââ squid.json + â  â  âââ parsers + â  â  âââ squid.json + â  âââ patterns + â  âââ common + â  âââ squid + âââ websphere + â  âââ config + â  â  âââ zookeeper + â  â  âââ enrichments + â  â  â  âââ websphere.json + â  â  âââ indexing + â  â  â  âââ websphere.json + â  â  âââ parsers + â  â  âââ websphere.json + â  âââ patterns + â  âââ common + â  âââ websphere + âââ yaf + âââ config + â  âââ elasticsearch + â  â  âââ yaf_index.template + â  âââ zookeeper + â  âââ enrichments + â  â  âââ yaf.json + â  âââ indexing + â  â  âââ yaf.json + â  âââ parsers + â  âââ yaf.json + âââ patterns + âââ common + âââ yaf +``` +### In Zookeeper + +```bash + +[zk: localhost(CONNECTED) 4] ls /metron/topology/parsers +[websphere, cef, fireeye, asa, paloalto, logstash, jsonMap, lancope, sourcefire, ise, squid, bro, snort, yaf] + +[zk: localhost(CONNECTED) 5] ls /metron/topology/indexing +[websphere, cef, fireeye, error, asa, paloalto, logstash, lancope, sourcefire, ise, squid, bro, snort, yaf] + +[zk: localhost(CONNECTED) 7] ls /metron/topology/enrichments +[websphere, cef, fireeye, asa, paloalto, logstash, lancope, sourcefire, ise, squid, bro, snort, yaf] +``` +Patterns are also deployed to HDFS: + +```bash +[root@node1 0.4.0]# hadoop fs -ls /apps/metron/patterns +Found 14 items +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/asa +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/bro +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/cef +-rwxr-xr-x 1 hdfs hdfs 5202 2017-06-27 16:14 /apps/metron/patterns/common +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/fireeye +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/ise +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/lancope +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/logstash +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/paloalto +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/snort +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/sourcefire +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/squid +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/websphere +drwxrwxr-x - metron hadoop 0 2017-06-27 16:14 /apps/metron/patterns/yaf +``` + +An example for a specific parser being: + +```bash +root@node1 0.4.0]# hadoop fs -ls /apps/metron/patterns/asa +Found 2 items +-rwxr-xr-x 1 hdfs hadoop 13748 2017-06-27 16:14 /apps/metron/patterns/asa/asa +-rwxr-xr-x 1 hdfs hadoop 5202 2017-06-27 16:14 /apps/metron/patterns/asa/common + +``` + http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_packaging.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_packaging.md b/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_packaging.md new file mode 100644 index 0000000..81315e9 --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/parser_extension_packaging.md @@ -0,0 +1,84 @@ +# Metron Parser Extension Packaging + +Parser Extensions packages should contain all the executable code and configurations required for one or more parsers. + +## The Package + +The package itself is a tar.gz file created at build time. The configuration for this packaging +in the project ( as produced by the [Metron Maven Parser Extension Archetype](../../../metron-maven-archetypes/metron-parser-extension-archetype)) by the +XXXX-assembly module, specifically in the src/main/assembly/assembly.xml file. + +## The Package Contents + +### config +The config directory includes the json configurations for the parsers, enrichment, and indexing. Of these, the enrichment configuration +is optional, as some parsers will not provide default enrichment. + +The directory may also include default configurations for elasticsearch and solr. + +> While this provides the means for managing and versioning these configurations, they are not used +> in the deployment of the extensions at this time. In the future, these will be the default configurations +> deployed during installation or instantiation of a parser + +### lib +The lib directory contains the [Metron Bundle](../../../bundles-lib), which itself contains the jars and dependencies +for the one or more parsers within the extension. The lib directory may not be present. It is possible to define a parser extension +solely by configuration. For an example of this, see the [Metron Yaf Parser Extension](metron-parser-yaf-extension/metron-parser-yaf). + +### patterns +Many Metron Parsers are based on the [GROK](https://github.com/thekrakken/java-grok) log parsing library, and may include rules. The rules for the one or more parsers +within the extension are in this directory. This directory may not be present if the parser does not use GROK rules. + +## Example: The Metron ASA Parser Extension + +metron-parser-asa-assembly-0.4.0-archive.tar.gz + +``` + +âââ config +â  âââ zookeeper +â  âââ enrichments +â  â  âââ asa.json +â  âââ indexing +â  â  âââ asa.json +â  âââ parsers +â  âââ asa.json +âââ lib +â  âââ metron-parser-asa-bundle-0.4.0.bundle +âââ patterns + âââ asa + âââ common +``` + +### Multiple Parsers in one extension +Parser extensions support packaging multiple parsers in one extension. This could be multiple configurations of a single +parser class, or multiple configurations of multiple parser classes + +for example: + +``` +âââ config +â  âââ zookeeper +â  âââ enrichments +â  â  âââ nice.json +â  â  âââ nice_other_config.json +â  â  âââ amazing.json +â  â  âââ amazing_other_config.json +â  âââ indexing +â  â  âââ nice.json +â  â  âââ nice_other_config.json +â  â  âââ amazing.json +â  â  âââ amazing_other_config.json +â  âââ parsers +â  â  âââ nice.json +â  â  âââ nice_other_config.json +â  â  âââ amazing.json +â  â  âââ amazing_other_config.json +âââ lib +â  âââ metron-parser-amazinglynice-bundle-0.4.0.bundle +âââ patterns + âââ ancommon + âââ nice + âââ amazing + âââ common +``` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/metron-parser-extensions/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/metron-parser-extensions/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/pom.xml new file mode 100644 index 0000000..e1aaf9f --- /dev/null +++ b/metron-platform/metron-extensions/metron-parser-extensions/pom.xml @@ -0,0 +1,97 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --><project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-extensions</artifactId> + <version>0.4.1</version> + </parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parser-extensions</artifactId> + <packaging>pom</packaging> + <name>metron-parser-extensions</name> + <version>0.4.1</version> + + <description>Parser Extensions for Metron</description> + <url>https://metron.apache.org/</url> + <scm> + <connection>scm:git:https://git-wip-us.apache.org/repos/asf/metron.git</connection> + <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/metron.git</developerConnection> + <tag>HEAD</tag> + <url>https://git-wip-us.apache.org/repos/asf/metron</url> + </scm> + + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + </license> + </licenses> + <modules> + <module>metron-parser-extensions-testing</module> + <module>metron-parser-asa-extension</module> + <module>metron-parser-bro-extension</module> + <module>metron-parser-cef-extension</module> + <module>metron-parser-fireeye-extension</module> + <module>metron-parser-ise-extension</module> + <module>metron-parser-lancope-extension</module> + <module>metron-parser-logstash-extension</module> + <module>metron-parser-paloalto-extension</module> + <module>metron-parser-snort-extension</module> + <module>metron-parser-sourcefire-extension</module> + <module>metron-parser-websphere-extension</module> + <module>metron-parser-squid-extension</module> + <module>metron-parser-yaf-extension</module> + <module>metron-parser-bundle-tests</module> + </modules> + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${global_slf4j_version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>${global_slf4j_version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${global_junit_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>1.6.6</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-api-mockito</artifactId> + <version>1.6.6</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.adrianwalker</groupId> + <artifactId>multiline-string</artifactId> + <version>0.1.2</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-extensions/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-extensions/pom.xml b/metron-platform/metron-extensions/pom.xml new file mode 100644 index 0000000..e9e199c --- /dev/null +++ b/metron-platform/metron-extensions/pom.xml @@ -0,0 +1,60 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-platform</artifactId> + <version>0.4.1</version> + </parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-extensions</artifactId> + <packaging>pom</packaging> + <name>metron-extensions</name> + + <description>Extensions for Metron</description> + <url>https://metron.apache.org/</url> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <commons.config.version>1.10</commons.config.version> + </properties> + <modules> + <module>metron-parser-extensions</module> + </modules> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.metron</groupId> + <artifactId>bundles-maven-plugin</artifactId> + <version>0.4.1</version> + <extensions>true</extensions> + <configuration> + </configuration> + </plugin> + </plugins> + </pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.metron</groupId> + <artifactId>bundles-maven-plugin</artifactId> + <version>0.4.1</version> + <extensions>true</extensions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/asa.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/asa.json b/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/asa.json deleted file mode 100644 index 153ccff..0000000 --- a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/asa.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "hdfs" : { - "index": "asa", - "batchSize": 5, - "enabled" : true - }, - "elasticsearch" : { - "index": "asa", - "batchSize": 5, - "enabled" : true - }, - "solr" : { - "index": "asa", - "batchSize": 5, - "enabled" : true - } -} - http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/bro.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/bro.json b/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/bro.json deleted file mode 100644 index b0aa8e4..0000000 --- a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/bro.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "hdfs" : { - "index": "bro", - "batchSize": 5, - "enabled" : true - }, - "elasticsearch" : { - "index": "bro", - "batchSize": 5, - "enabled" : true - }, - "solr" : { - "index": "bro", - "batchSize": 5, - "enabled" : true - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/snort.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/snort.json b/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/snort.json deleted file mode 100644 index f6112a8..0000000 --- a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/snort.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "hdfs" : { - "index": "snort", - "batchSize": 1, - "enabled" : true - }, - "elasticsearch" : { - "index": "snort", - "batchSize": 1, - "enabled" : true - }, - "solr" : { - "index": "snort", - "batchSize": 1, - "enabled" : true - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/websphere.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/websphere.json b/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/websphere.json deleted file mode 100644 index 1a2b53d..0000000 --- a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/websphere.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "hdfs" : { - "index": "websphere", - "batchSize": 5, - "enabled" : true - }, - "elasticsearch" : { - "index": "websphere", - "batchSize": 5, - "enabled" : true - }, - "solr" : { - "index": "websphere", - "batchSize": 5, - "enabled" : true - } -} - http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/yaf.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/yaf.json b/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/yaf.json deleted file mode 100644 index 0586497..0000000 --- a/metron-platform/metron-indexing/src/main/config/zookeeper/indexing/yaf.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "hdfs" : { - "index": "yaf", - "batchSize": 5, - "enabled" : true - }, - "elasticsearch" : { - "index": "yaf", - "batchSize": 5, - "enabled" : true - }, - "solr" : { - "index": "yaf", - "batchSize": 5, - "enabled" : true - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-integration-test/src/main/config/zookeeper/bundle.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/bundle.properties b/metron-platform/metron-integration-test/src/main/config/zookeeper/bundle.properties new file mode 100644 index 0000000..7c4fc23 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/bundle.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +bundle.library.directory=. +bundle.archive.extension=bundle +bundle.meta.id.prefix=Bundle +bundle.extension.type.MessageParser=org.apache.metron.parsers.interfaces.MessageParser http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-integration-test/src/main/config/zookeeper/extensions/parsers/metron-test-parsers.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/extensions/parsers/metron-test-parsers.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/extensions/parsers/metron-test-parsers.json new file mode 100644 index 0000000..adc0609 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/extensions/parsers/metron-test-parsers.json @@ -0,0 +1,7 @@ +{ + "extensionAssemblyName" : "test.fool-1.0.tar.gz", + "extensionBundleName" : "metron-test-parsers-1.0.bundle", + "extensionsBundleID" : "metron-test-parsers", + "extensionsBundleVersion" : "1.0.0", + "parserExtensionParserName" : [ "test2", "test" ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json index 9292f72..7bc5b90 100644 --- a/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json +++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json @@ -25,5 +25,6 @@ "geo.hdfs.file": "src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz", "update.hbase.table" : "updates", - "update.hbase.cf" : "t" + "update.hbase.cf" : "t", + "metron.apps.hdfs.dir": "./target/" } \ No newline at end of file
