http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-stellar/stellar-zeppelin/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-stellar/stellar-zeppelin/index.html b/site/current-book/metron-stellar/stellar-zeppelin/index.html new file mode 100644 index 0000000..54bc800 --- /dev/null +++ b/site/current-book/metron-stellar/stellar-zeppelin/index.html @@ -0,0 +1,239 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-zeppelin/index.md at 2018-06-07 + | Rendered using Apache Maven Fluido Skin 1.7 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Stellar Interpreter for Apache Zeppelin</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active ">Stellar Interpreter for Apache Zeppelin</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> + <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + </ul> + </div> + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> + <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li> + <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li class="active"><a href="#"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> + </div> + </div> + </div> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<h1>Stellar Interpreter for Apache Zeppelin</h1> +<p><a class="externalLink" href="https://zeppelin.apache.org/";>Apache Zeppelin</a> is a web-based notebook that enables data-driven, interactive data analytics and collaborative documents with SQL, Scala and more. This project provides a means to run the Stellar REPL directly within a Zeppelin Notebook.</p> +<ul> + +<li><a href="#Prerequisites">Prerequisites</a></li> +<li><a href="#Installation">Installation</a></li> +<li><a href="#Usage">Usage</a></li> +</ul> +<div class="section"> +<h2><a name="Prerequisites"></a>Prerequisites</h2> +<ul> + +<li><a class="externalLink" href="https://zeppelin.apache.org/";>Apache Zeppelin</a> 0.7.3 +<p>This is tested with version 0.7.3. Other versions may work, but are not supported.</p></li> +</ul></div> +<div class="section"> +<h2><a name="Installation"></a>Installation</h2> +<p>Currently, you need to manually install the Stellar Interpreter in Zeppelin. In the future this step could be automated by the Metron Mpack.</p> +<p>To install the Stellar Interpreter in your Apache Zeppelin installation, follow these instructions. This is paraphrased from the <a class="externalLink" href="https://zeppelin.apache.org/docs/latest/development/writingzeppelininterpreter.html#install-your-interpreter-binary";>Zeppelin docs</a>.</p> +<ol style="list-style-type: decimal"> + +<li> + +<p>Build and install Metron. Metron and its dependencies will be retrieved from your local Maven repository.</p> + +<div> +<div> +<pre class="source">cd $METRON_HOME +mvn clean install -DskipTests +</pre></div></div> +</li> +<li> + +<p>If you do not already have Zeppelin installed, <a class="externalLink" href="https://zeppelin.apache.org/download.html";>download and unpack Apache Zeppelin</a>. Then change directories to the root of your Zeppelin download.</p> + +<div> +<div> +<pre class="source">cd $ZEPPELIN_HOME +</pre></div></div> +</li> +<li> + +<p>Use Zeppelin’s installation utility to install the Stellar Interpreter.</p> +<p>If Zeppelin was already installed, make sure that it is stopped before running this command. Update the version, ‘0.5.0’ in the example below, to whatever is appropriate for your environment.</p> + +<div> +<div> +<pre class="source">bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.5.0 +</pre></div></div> +</li> +<li> + +<p>Start Zeppelin.</p> + +<div> +<div> +<pre class="source">bin/zeppelin-daemon.sh start +</pre></div></div> +</li> +<li> + +<p>Navigate to Zeppelin running at <a class="externalLink" href="http://localhost:8080/";>http://localhost:8080/</a>. The Stellar Interpreter should be ready for use with a basic set of functions.</p> +</li> +</ol></div> +<div class="section"> +<h2><a name="Usage"></a>Usage</h2> +<ol style="list-style-type: decimal"> + +<li> + +<p>Create a new notebook.</p> +<ol style="list-style-type: decimal"> + +<li> + +<p>Click on “Notebook” > “Create new note”.</p> +</li> +<li> + +<p>Set the default Interpreter to <tt>stellar</tt>.</p> +<p>When creating the notebook, if you define <tt>stellar</tt> as the default interpreter, then there is no need to enter <tt>%stellar</tt> at the top of each code block.</p> +<p>If <tt>stellar</tt> is not the default interpreter, then you must enter <tt>%stellar</tt> at the top of a code block containing Stellar code.</p> +</li> +</ol> +</li> +<li> + +<p>In the first block, add the following Stellar, then click Run.</p> + +<div> +<div> +<pre class="source">2 in [2,3,4] +</pre></div></div> +</li> +<li> + +<p>In the next block, check which functions are available to you.</p> +<p>When executing Stellar’s magic functions, you must explicitly define which interpreter should be used in the code block. If you define ‘stellar’ as the default interpreter when creating a notebook, then this is only required when using Stellar’s magic functions.</p> + +<div> +<div> +<pre class="source">%stellar + +%functions +</pre></div></div> + +<p>You will <b>only</b> ‘see’ the functions defined within <tt>stellar-common</tt> since that is the only library that we added to the interpreter.</p> +</li> +<li> + +<p>Add additional Stellar functions to your session.</p> +<ol style="list-style-type: decimal"> + +<li> + +<p>Go back to the Stellar interpreter configuration and add another dependency as follows.</p> + +<div> +<div> +<pre class="source">org.apache.metron:metron-statistics:0.5.0 +</pre></div></div> +</li> +<li> + +<p>Go back to your notebook and run <tt>%functions</tt> again. You will now see the additional functions defined within the <tt>metron-statistics</tt> project.</p> +</li> +</ol> +</li> +<li> + +<p>Auto-completion is also available for Stellar expressions.</p> +<p>In another block, type ‘TO_’ then press the <kbd>CTRL</kbd> + <kbd>PERIOD</kbd> keys. This will trigger the auto-complete mechanism in Stellar and display a list of matching functions or variables.</p> +</li> +</ol></div> + </div> + </div> + </div> + <hr/> + <footer> + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> + </div> + </footer> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/use-cases/forensic_clustering/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/use-cases/forensic_clustering/index.html b/site/current-book/use-cases/forensic_clustering/index.html index 73a087e..f67a830 100644 --- a/site/current-book/use-cases/forensic_clustering/index.html +++ b/site/current-book/use-cases/forensic_clustering/index.html @@ -1,218 +1,111 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2018-01-03 - | Rendered using Apache Maven Fluido Skin 1.3.0 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/forensic_clustering/index.md at 2018-06-07 + | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180103" /> + <meta name="Date-Revision-yyyymmdd" content="20180607" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Problem Statement</title> - <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> <link rel="stylesheet" href="../../css/site.css" /> <link rel="stylesheet" href="../../css/print.css" media="print" /> - - - <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> - - - -<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> - - </head> - <body class="topBarDisabled"> - - - - - <div class="container-fluid"> - <div id="banner"> - <div class="pull-left"> - <a href="http://metron.apache.org/"; id="bannerLeft"> - <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> - </a> - </div> - <div class="pull-right"> </div> + <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> <div class="clear"><hr/></div> </div> <div id="breadcrumbs"> <ul class="breadcrumb"> - - - <li class=""> - <a href="http://www.apache.org"; class="externalLink" title="Apache"> - Apache</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> - Metron</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="../../index.html" title="Documentation"> - Documentation</a> - </li> - <li class="divider ">/</li> - <li class="">Problem Statement</li> - - - - <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.2</li> - - </ul> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active ">Problem Statement</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> + <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + </ul> </div> - - <div class="row-fluid"> - <div id="leftColumn" class="span3"> + <div id="leftColumn" class="span2"> <div class="well sidebar-nav"> - - - <ul class="nav nav-list"> - <li class="nav-header">User Documentation</li> - - <li> - - <a href="../../index.html" title="Metron"> - <i class="icon-chevron-down"></i> - Metron</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../Upgrading.html" title="Upgrading"> - <i class="none"></i> - Upgrading</a> - </li> - - <li> - - <a href="../../metron-analytics/index.html" title="Analytics"> - <i class="icon-chevron-right"></i> - Analytics</a> - </li> - - <li> - - <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> - <i class="none"></i> - Docker</a> - </li> - - <li> - - <a href="../../metron-deployment/index.html" title="Deployment"> - <i class="icon-chevron-right"></i> - Deployment</a> - </li> - - <li> - - <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> - <i class="none"></i> - Alerts</a> - </li> - - <li> - - <a href="../../metron-interface/metron-config/index.html" title="Config"> - <i class="none"></i> - Config</a> - </li> - - <li> - - <a href="../../metron-interface/metron-rest/index.html" title="Rest"> - <i class="none"></i> - Rest</a> - </li> - - <li> - - <a href="../../metron-platform/index.html" title="Platform"> - <i class="icon-chevron-right"></i> - Platform</a> - </li> - - <li> - - <a href="../../metron-sensors/index.html" title="Sensors"> - <i class="icon-chevron-right"></i> - Sensors</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> - <i class="none"></i> - Stellar-3rd-party-example</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> - <i class="icon-chevron-right"></i> - Stellar-common</a> - </li> - - <li> - - <a href="../../use-cases/index.html" title="Use-cases"> - <i class="icon-chevron-down"></i> - Use-cases</a> - <ul class="nav nav-list"> - - <li class="active"> - - <a href="#"><i class="none"></i>Forensic_clustering</a> - </li> - - <li> - - <a href="../../use-cases/geographic_login_outliers/index.html" title="Geographic_login_outliers"> - <i class="none"></i> - Geographic_login_outliers</a> - </li> - </ul> - </li> - </ul> - </li> - </ul> - - - - <hr class="divider" /> - - <div id="poweredBy"> - <div class="clear"></div> - <div class="clear"></div> - <div class="clear"></div> - <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> - <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> - </a> - </div> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> + <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li> + <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-down"></span>Use-cases</a> + <ul class="nav nav-list"> + <li class="active"><a href="#"><span class="none"></span>Forensic_clustering</a></li> + <li><a href="../../use-cases/geographic_login_outliers/index.html" title="Geographic_login_outliers"><span class="none"></span>Geographic_login_outliers</a></li> + <li><a href="../../use-cases/typosquat_detection/index.html" title="Typosquat_detection"><span class="none"></span>Typosquat_detection</a></li> + </ul> +</li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> + </div> </div> </div> - - - <div id="bodyColumn" class="span9" > - - <h1>Problem Statement</h1> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<h1>Problem Statement</h1> <p><a name="Problem_Statement"></a></p> -<p>Having a forensic hash, such as <a class="externalLink" href="https://github.com/trendmicro/tlsh";>TLSH</a>, is a useful tool in cybersecurity. In short, the notion is that semantically similar documents should hash to a value which also similar. Contrast this with your standard cryptographic hashes, such as SHA and MD, where small deviations in the input data will yield large deviations in the hashes.</p> -<p>The traditional use-case is to hash input documents or binaries and compare against a known blacklist of malicious hashes. A sufficiently similar hash will indicate a match. This will avoid malicious parties fuzzing input data to avoid detection.</p> -<p>While this is interesting, it still requires metric-space searches in a blacklist. I envisioned a slightly more interesting streaming use-case of on-the-fly clustering of data. While the TLSH hashes created do not necessarily hash to precisely the same value on similar documents, more traditional non-forensic hashes <i>do</i> collide when sufficiently similar. Namely, the Hamming distance <a class="externalLink" href="https://en.wikipedia.org/wiki/Locality-sensitive_hashing#Bit_sampling_for_Hamming_distance";>LSH</a> applied to the TLSH hash would give us a way to bin semantic hashes such that similar hashes (by hamming distance) have the same hash.</p> +<p>Having a forensic hash, such as <a class="externalLink" href="https://github.com/trendmicro/tlsh";>TLSH</a>, is a useful tool in cybersecurity. In short, the notion is that semantically similar documents should hash to a value which also similar. Contrast this with your standard cryptographic hashes, such as SHA and MD, where small deviations in the input data will yield large deviations in the hashes.</p> +<p>The traditional use-case is to hash input documents or binaries and compare against a known blacklist of malicious hashes. A sufficiently similar hash will indicate a match. This will avoid malicious parties fuzzing input data to avoid detection.</p> +<p>While this is interesting, it still requires metric-space searches in a blacklist. I envisioned a slightly more interesting streaming use-case of on-the-fly clustering of data. While the TLSH hashes created do not necessarily hash to precisely the same value on similar documents, more traditional non-forensic hashes <i>do</i> collide when sufficiently similar. Namely, the Hamming distance <a class="externalLink" href="https://en.wikipedia.org/wiki/Locality-sensitive_hashing#Bit_sampling_for_Hamming_distance";>LSH</a> applied to the TLSH hash would give us a way to bin semantic hashes such that similar hashes (by hamming distance) have the same hash.</p> <p>Inspired by a good <a class="externalLink" href="https://github.com/fluenda/dataworks_summit_iot_botnet/blob/master/dws-fucs-lopresto.pdf";>talk</a> by Andy LoPresto and Andre Fucs de Miranda from Apache NiFi, we will proceed to take logs from the Cowrie honeypot and compute TLSH hashes and semantic bins so that users can easily find similarly malicious activity to known threats in logs.</p> <p>Consider the following excerpts from the Cowrie logs the authors above have shared:</p> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "eventid": "cowrie.command.success" , "timestamp": "2017-09-18T11:45:25.028091Z" , "message": "Command found: /bin/busybox LSUCT" @@ -224,11 +117,12 @@ , "sensor": "a927e8b28666" } </pre></div></div> + <p>and</p> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "eventid": "cowrie.command.success" , "timestamp": "2017-09-17T04:06:39.673206Z" , "message": "Command found: /bin/busybox XUSRH" @@ -240,11 +134,13 @@ , "sensor": "a927e8b28666" } </pre></div></div> -<p>You will note the <tt>/bin/busybox</tt> call with a random selection afterwards.<br />Excerpting from an analysis of an IOT exploit <a class="externalLink" href="https://isc.sans.edu/diary/21543";>here</a>:</p> -<div class="source"> -<div class="source"> -<pre>The use of the command "busybox ECCHI" appears to have two functions. +<p>You will note the <tt>/bin/busybox</tt> call with a random selection afterwards.<br /> +Excerpting from an analysis of an IOT exploit <a class="externalLink" href="https://isc.sans.edu/diary/21543";>here</a>:</p> + +<div> +<div> +<pre class="source">The use of the command "busybox ECCHI" appears to have two functions. First of all, cowrie, and more "complete" Linux distrubtions then commonly found on DVRs will respond with a help screen if a wrong module is used. So this way, "ECCHI" can be used to detect honeypots and @@ -253,77 +149,67 @@ Secondly, the command is used as a market to indicate that the prior command finished. Later, the attacker adds "/bin/busybox ECCHI" at the end of each line, following the actual command to be executed. </pre></div></div> -<p>We have a few options at our disposal:</p> +<p>We have a few options at our disposal:</p> <ul> - + <li>If we were merely filtering and alerting on the execution of <tt>/bin/busybox</tt> we would include false positives.</li> - <li>If we looked at <tt>/bin/busybox XUSRH</tt>, we’d miss many attempts with a <i>different</i> value as <tt>XUSRH</tt> is able to be swapped out for another random sequence to foil overly strict rules.</li> - <li>If we looked for <tt>/bin/busybox *</tt> then we’d capture this scenario well, but it’d be nice to be able to not be specific to detecting the <tt>/bin/busybox</tt> style of exploits.</li> </ul> -<p>Indeed, this is precisely what semantic hashing and binning allows us, the ability to group by semantic similarity without being too specific about what we mean of as “semantic” or “similar”. We want to cast a wide net, but not pull back every fish in the sea.</p> -<p>For this demonstration, we will </p> - +<p>Indeed, this is precisely what semantic hashing and binning allows us, the ability to group by semantic similarity without being too specific about what we mean of as “semantic” or “similar”. We want to cast a wide net, but not pull back every fish in the sea.</p> +<p>For this demonstration, we will</p> <ul> - + <li>ingest some 400 cowrie records</li> - <li>tag records from an IP blacklist for known malicious actors</li> - <li>use the alerts UI to investigate and find similar attacks.</li> </ul> <div class="section"> <h2><a name="Preliminaries"></a>Preliminaries</h2> <p>We assume that the following environment variables are set:</p> - <ul> - + <li><tt>METRON_HOME</tt> - the home directory for metron</li> - <li><tt>ZOOKEEPER</tt> - The zookeeper quorum (comma separated with port specified: e.g. <tt>node1:2181</tt> for full-dev)</li> - <li><tt>BROKERLIST</tt> - The Kafka broker list (comma separated with port specified: e.g. <tt>node1:6667</tt> for full-dev)</li> - <li><tt>ES_HOST</tt> - The elasticsearch master (and port) e.g. <tt>node1:9200</tt> for full-dev.</li> </ul> -<p>Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.</p> +<p>Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.</p> <p>Before editing configurations, be sure to pull the configs from zookeeper locally via</p> -<div class="source"> -<div class="source"> -<pre>$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f -</pre></div></div></div> +<div> +<div> +<pre class="source">$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f +</pre></div></div> +</div> <div class="section"> <h2><a name="Setting_up_the_Data"></a>Setting up the Data</h2> <p>First we must set up the cowrie log data in our cluster’s access node.</p> - <ul> - + <li>Download the data from the github repository for the talk mentioned above <a class="externalLink" href="https://github.com/fluenda/dataworks_summit_iot_botnet/blob/master/180424243034750.tar.gz";>here</a>. Ensure that’s moved into your home directory on the metron node.</li> - -<li>Create a directory called <tt>cowrie</tt> in ~ and untar the tarball into that directory via:</li> +<li>Create a directory called <tt>cowrie</tt> in ~ and untar the tarball into that directory via:</li> </ul> -<div class="source"> -<div class="source"> -<pre>mkdir ~/cowrie +<div> +<div> +<pre class="source">mkdir ~/cowrie cd ~/cowrie tar xzvf ~/180424243034750.tar.gz -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h2><a name="Configuring_the_Parser"></a>Configuring the Parser</h2> -<p>The Cowrie data is coming in as simple JSON blobs, so it’s easy to parse. We really just need to adjust the timestamp and a few fields and we have valid data.</p> - +<p>The Cowrie data is coming in as simple JSON blobs, so it’s easy to parse. We really just need to adjust the timestamp and a few fields and we have valid data.</p> <ul> - + <li>Create <tt>$METRON_HOME/config/zookeeper/parsers/cowrie.json</tt> with the following content:</li> </ul> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "parserClassName":"org.apache.metron.parsers.json.JSONMapParser", "sensorTopic":"cowrie", "fieldTransformations" : [ @@ -339,11 +225,12 @@ tar xzvf ~/180424243034750.tar.gz } </pre></div></div> + <p>Before we start, we will want to install ES mappings so ES knows how to interpret our fields:</p> -<div class="source"> -<div class="source"> -<pre>curl -XPUT 'http://$ES_HOST/cowrie*/_mapping/cowrie_doc' -d ' +<div> +<div> +<pre class="source">curl -XPUT 'http://$ES_HOST/cowrie*/_mapping/cowrie_doc' -d ' { "properties" : { "adapter:stellaradapter:begin:ts" : { @@ -487,36 +374,36 @@ tar xzvf ~/180424243034750.tar.gz </pre></div></div> <ul> - + <li>Create the <tt>cowrie</tt> kafka topic via:</li> </ul> -<div class="source"> -<div class="source"> -<pre>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic cowrie --partitions 1 --replication-factor 1 -</pre></div></div></div> +<div> +<div> +<pre class="source">/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic cowrie --partitions 1 --replication-factor 1 +</pre></div></div> +</div> <div class="section"> <h2><a name="Import_the_Blacklist"></a>Import the Blacklist</h2> -<p>Here, to build out a scenario, we will assume that we have a blacklist of known malicious hosts. For our purposes, we’ll choose one particular host IP to be malicious.</p> - +<p>Here, to build out a scenario, we will assume that we have a blacklist of known malicious hosts. For our purposes, we’ll choose one particular host IP to be malicious.</p> <ul> - + <li>Create <tt>~/blacklist.csv</tt> to contain the following:</li> </ul> -<div class="source"> -<div class="source"> -<pre>94.51.110.74 +<div> +<div> +<pre class="source">94.51.110.74 </pre></div></div> <ul> - + <li>Create <tt>~/blacklist_extractor.json</tt> to contain the following:</li> </ul> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "config" : { "columns" : { "ip" : 0 @@ -530,50 +417,43 @@ tar xzvf ~/180424243034750.tar.gz </pre></div></div> <ul> - + <li>Import the data <tt>$METRON_HOME/bin/flatfile_loader.sh -i ~/blacklist.csv -t threatintel -c t -e ~/blacklist_extractor.json</tt></li> </ul> <p>This will create a new enrichment type “blacklist” with a single entry “94.51.110.74”.</p></div> <div class="section"> <h2><a name="Configure_Enrichments"></a>Configure Enrichments</h2> <p>We will want to do the following:</p> - <ul> - + <li>Add enrichments to faciliate binning - <ul> - -<li>Construct what we consider to be a sufficient representation of the thing we want to cluster. For our purposes, this is centered around the input command, so that would be: - + +<li>Construct what we consider to be a sufficient representation of the thing we want to cluster. For our purposes, this is centered around the input command, so that would be: <ul> - + <li>The <tt>message</tt> field</li> - <li>The <tt>input</tt> field</li> - <li>The <tt>isError</tt> field</li> - </ul></li> - +</ul> +</li> <li>Compute the TLSH hash of this representation, called <tt>tlsh</tt></li> - <li>Compute the locality sensitive hash of the TLSH hash suitable for binning, called <tt>similarity_bin</tt></li> - </ul></li> - +</ul> +</li> <li>Set up the threat intelligence to use the blacklist - <ul> - + <li>Set an alert if the message is from an IP address in the threat intelligence blacklist.</li> - -<li>Score blacklisted messages with <tt>10</tt>. In production, this would be more complex.</li> - </ul></li> +<li>Score blacklisted messages with <tt>10</tt>. In production, this would be more complex.</li> +</ul> +</li> </ul> <p>Now, we can create the enrichments thusly by creating <tt>$METRON_HOME/config/zookeeper/enrichments/cowrie.json</tt> with the following content:</p> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "enrichment": { "fieldMap": { "stellar" : { @@ -615,23 +495,23 @@ tar xzvf ~/180424243034750.tar.gz } } </pre></div></div> + <div class="section"> <h3><a name="A_Note_About_Similarity_Hashes_and_TLSH"></a>A Note About Similarity Hashes and TLSH</h3> -<p>Notice that we have specified a number of hash functions of <tt>16</tt> when constructing the similarity bin.<br />I arrived at that by trial and error, which is not always tenable, frankly. What is more sensible is likely to construct <i>multiple</i> similarity bins of size <tt>8</tt>, <tt>16</tt>, <tt>32</tt> at minimum.</p> - +<p>Notice that we have specified a number of hash functions of <tt>16</tt> when constructing the similarity bin.<br /> +I arrived at that by trial and error, which is not always tenable, frankly. What is more sensible is likely to construct <i>multiple</i> similarity bins of size <tt>8</tt>, <tt>16</tt>, <tt>32</tt> at minimum.</p> <ul> - + <li>The smaller the number of hashes, the more loose the notion of similarity (more possibly dissimilar things would get grouped together).</li> - <li>The larger the number of hashes, the more strict (similar things may not be grouped together).</li> </ul></div></div> <div class="section"> <h2><a name="Create_the_Data_Loader"></a>Create the Data Loader</h2> <p>We want to pull a snapshot of the cowrie logs, so create <tt>~/load_data.sh</tt> with the following content:</p> -<div class="source"> -<div class="source"> -<pre>COWRIE_HOME=~/cowrie +<div> +<div> +<pre class="source">COWRIE_HOME=~/cowrie for i in cowrie.1626302-1636522.json cowrie.16879981-16892488.json cowrie.21312194-21331475.json cowrie.698260-710913.json cowrie.762933-772239.json cowrie.929866-939552.json cowrie.1246880-1248235.json cowrie.19285959-19295444.json cowrie.16542668-16581213.json cowrie.5849832-5871517.json cowrie.6607473-6609163.json;do echo $i cat $COWRIE_HOME/$i | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic cowrie @@ -640,87 +520,80 @@ done </pre></div></div> <ul> - + <li>Set the <tt>+x</tt> bit on the executable via:</li> </ul> -<div class="source"> -<div class="source"> -<pre>chmod +x ~/load_data.sh -</pre></div></div></div> +<div> +<div> +<pre class="source">chmod +x ~/load_data.sh +</pre></div></div> +</div> <div class="section"> <h2><a name="Execute_Demonstration"></a>Execute Demonstration</h2> <p>From here, we’ve set up our configuration and can push the configs:</p> - <ul> - + <li>Push the configs to zookeeper via</li> </ul> -<div class="source"> -<div class="source"> -<pre>$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z $ZOOKEEPER -i $METRON_HOME/config/zookeeper/ +<div> +<div> +<pre class="source">$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z $ZOOKEEPER -i $METRON_HOME/config/zookeeper/ </pre></div></div> <ul> - + <li>Start the parser via:</li> </ul> -<div class="source"> -<div class="source"> -<pre>$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s cowrie +<div> +<div> +<pre class="source">$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s cowrie </pre></div></div> <ul> - + <li>Push cowrie data into the <tt>cowrie</tt> topic via</li> </ul> -<div class="source"> -<div class="source"> -<pre>~/load_data.sh +<div> +<div> +<pre class="source">~/load_data.sh </pre></div></div> -<p>Once this data is loaded, we can use the Alerts UI, starting from known malicious actors, to find others doing similar things.</p> +<p>Once this data is loaded, we can use the Alerts UI, starting from known malicious actors, to find others doing similar things.</p> <ul> - + <li> -<p>First we can look at the alerts directly and find an instance of our <tt>/bin/busybox</tt> activity: <img src="find_alerts.png" alt="Alerts" /></p></li> - + +<p>First we can look at the alerts directly and find an instance of our <tt>/bin/busybox</tt> activity: <img src="find_alerts.png" alt="Alerts" /></p> +</li> <li> -<p>We can now pivot and look for instances of messages with the same <tt>semantic_hash</tt> but who are <i>not</i> alerts: <img src="clustered.png" alt="Pivot" /></p></li> + +<p>We can now pivot and look for instances of messages with the same <tt>semantic_hash</tt> but who are <i>not</i> alerts: <img src="clustered.png" alt="Pivot" /></p> +</li> </ul> <p>As you can see, we have found a few more malicious actors:</p> - <ul> - + <li>177.239.192.172</li> - <li>180.110.69.182</li> - <li>177.238.236.21</li> - <li>94.78.80.45</li> </ul> -<p>Now we can look at <i>other</i> things that they’re doing to build and refine our definition of what an alert is without resorting to hard-coding of rules. Note that nothing in our enrichments actually used the string <tt>busybox</tt>, so this is a more general purpose way of navigating similar things.</p></div> - </div> - </div> - </div> - +<p>Now we can look at <i>other</i> things that they’re doing to build and refine our definition of what an alert is without resorting to hard-coding of rules. Note that nothing in our enrichments actually used the string <tt>busybox</tt>, so this is a more general purpose way of navigating similar things.</p></div> + </div> + </div> + </div> <hr/> - <footer> - <div class="container-fluid"> - <div class="row span12">Copyright © 2018 - <a href="https://www.apache.org";>The Apache Software Foundation</a>. - All Rights Reserved. - + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> </div> - - - - </div> </footer> </body> </html> http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/use-cases/geographic_login_outliers/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/use-cases/geographic_login_outliers/index.html b/site/current-book/use-cases/geographic_login_outliers/index.html index 4d0ff74..2c0441a 100644 --- a/site/current-book/use-cases/geographic_login_outliers/index.html +++ b/site/current-book/use-cases/geographic_login_outliers/index.html @@ -1,259 +1,148 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2018-01-03 - | Rendered using Apache Maven Fluido Skin 1.3.0 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/geographic_login_outliers/index.md at 2018-06-07 + | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180103" /> + <meta name="Date-Revision-yyyymmdd" content="20180607" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Problem Statement</title> - <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> <link rel="stylesheet" href="../../css/site.css" /> <link rel="stylesheet" href="../../css/print.css" media="print" /> - - - <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> - - - -<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> - - </head> - <body class="topBarDisabled"> - - - - - <div class="container-fluid"> - <div id="banner"> - <div class="pull-left"> - <a href="http://metron.apache.org/"; id="bannerLeft"> - <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> - </a> - </div> - <div class="pull-right"> </div> + <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> <div class="clear"><hr/></div> </div> <div id="breadcrumbs"> <ul class="breadcrumb"> - - - <li class=""> - <a href="http://www.apache.org"; class="externalLink" title="Apache"> - Apache</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> - Metron</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="../../index.html" title="Documentation"> - Documentation</a> - </li> - <li class="divider ">/</li> - <li class="">Problem Statement</li> - - - - <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.2</li> - - </ul> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active ">Problem Statement</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> + <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + </ul> </div> - - <div class="row-fluid"> - <div id="leftColumn" class="span3"> + <div id="leftColumn" class="span2"> <div class="well sidebar-nav"> - - - <ul class="nav nav-list"> - <li class="nav-header">User Documentation</li> - - <li> - - <a href="../../index.html" title="Metron"> - <i class="icon-chevron-down"></i> - Metron</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../Upgrading.html" title="Upgrading"> - <i class="none"></i> - Upgrading</a> - </li> - - <li> - - <a href="../../metron-analytics/index.html" title="Analytics"> - <i class="icon-chevron-right"></i> - Analytics</a> - </li> - - <li> - - <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> - <i class="none"></i> - Docker</a> - </li> - - <li> - - <a href="../../metron-deployment/index.html" title="Deployment"> - <i class="icon-chevron-right"></i> - Deployment</a> - </li> - - <li> - - <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> - <i class="none"></i> - Alerts</a> - </li> - - <li> - - <a href="../../metron-interface/metron-config/index.html" title="Config"> - <i class="none"></i> - Config</a> - </li> - - <li> - - <a href="../../metron-interface/metron-rest/index.html" title="Rest"> - <i class="none"></i> - Rest</a> - </li> - - <li> - - <a href="../../metron-platform/index.html" title="Platform"> - <i class="icon-chevron-right"></i> - Platform</a> - </li> - - <li> - - <a href="../../metron-sensors/index.html" title="Sensors"> - <i class="icon-chevron-right"></i> - Sensors</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> - <i class="none"></i> - Stellar-3rd-party-example</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> - <i class="icon-chevron-right"></i> - Stellar-common</a> - </li> - - <li> - - <a href="../../use-cases/index.html" title="Use-cases"> - <i class="icon-chevron-down"></i> - Use-cases</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../use-cases/forensic_clustering/index.html" title="Forensic_clustering"> - <i class="none"></i> - Forensic_clustering</a> - </li> - - <li class="active"> - - <a href="#"><i class="none"></i>Geographic_login_outliers</a> - </li> - </ul> - </li> - </ul> - </li> - </ul> - - - - <hr class="divider" /> - - <div id="poweredBy"> - <div class="clear"></div> - <div class="clear"></div> - <div class="clear"></div> - <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> - <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> - </a> - </div> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> + <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li> + <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-down"></span>Use-cases</a> + <ul class="nav nav-list"> + <li><a href="../../use-cases/forensic_clustering/index.html" title="Forensic_clustering"><span class="none"></span>Forensic_clustering</a></li> + <li class="active"><a href="#"><span class="none"></span>Geographic_login_outliers</a></li> + <li><a href="../../use-cases/typosquat_detection/index.html" title="Typosquat_detection"><span class="none"></span>Typosquat_detection</a></li> + </ul> +</li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> + </div> </div> </div> - - - <div id="bodyColumn" class="span9" > - - <h1>Problem Statement</h1> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<h1>Problem Statement</h1> <p><a name="Problem_Statement"></a></p> -<p>One way to find anomalous behavior in a network is by inspecting user login behavior. In particular, if a user is logging in via vastly differing geographic locations in a short period of time, this may be evidence of malicious behavior.</p> -<p>More formally, we can encode this potentially malicious event in terms of how far from the geographic centroid of the user’s historic logins as compared to all users. For instance, if we track all users and the median distance from the central geographic location of all of their logins for the last 2 hours is 3 km and the standard deviation is 1 km, if we see a user logging in 1700 km from the central geographic location of their logins for the last 2 hours, then they MAY be exhibiting a deviation that we want to monitor since it would be hard to travel that distance in 4 hours. On the other hand, the user may have just used a VPN or proxy. Ultimately, this sort of analytic must be considered only one piece of evidence in addition to many others before we want to indicate an alert.</p> +<p>One way to find anomalous behavior in a network is by inspecting user login behavior. In particular, if a user is logging in via vastly differing geographic locations in a short period of time, this may be evidence of malicious behavior.</p> +<p>More formally, we can encode this potentially malicious event in terms of how far from the geographic centroid of the user’s historic logins as compared to all users. For instance, if we track all users and the median distance from the central geographic location of all of their logins for the last 2 hours is 3 km and the standard deviation is 1 km, if we see a user logging in 1700 km from the central geographic location of their logins for the last 2 hours, then they MAY be exhibiting a deviation that we want to monitor since it would be hard to travel that distance in 4 hours. On the other hand, the user may have just used a VPN or proxy. Ultimately, this sort of analytic must be considered only one piece of evidence in addition to many others before we want to indicate an alert.</p> <p><a name="Demonstration_Design"></a></p> <h1>Demonstration Design</h1> -<p>For the purposes of demonstration, we will construct synthetic data whereby 2 users are logging into a system rather quickly (once per second) from various hosts. Each user’s locations share the same first 2 octets, but will choose the last 2 randomly. We will then inject a data point indicating <tt>user1</tt> is logging in via a russian IP address.</p> +<p>For the purposes of demonstration, we will construct synthetic data whereby 2 users are logging into a system rather quickly (once per second) from various hosts. Each user’s locations share the same first 2 octets, but will choose the last 2 randomly. We will then inject a data point indicating <tt>user1</tt> is logging in via a russian IP address.</p> <div class="section"> <h2><a name="Preliminaries"></a>Preliminaries</h2> <p>We assume that the following environment variables are set:</p> - <ul> - + <li><tt>METRON_HOME</tt> - the home directory for metron</li> - <li><tt>ZOOKEEPER</tt> - The zookeeper quorum (comma separated with port specified: e.g. <tt>node1:2181</tt> for full-dev)</li> - <li><tt>BROKERLIST</tt> - The Kafka broker list (comma separated with port specified: e.g. <tt>node1:6667</tt> for full-dev)</li> - <li><tt>ES_HOST</tt> - The elasticsearch master (and port) e.g. <tt>node1:9200</tt> for full-dev.</li> </ul> -<p>Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.</p> +<p>Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.</p> <p>Before editing configurations, be sure to pull the configs from zookeeper locally via</p> -<div class="source"> -<div class="source"> -<pre>$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f -</pre></div></div></div> +<div> +<div> +<pre class="source">$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f +</pre></div></div> +</div> <div class="section"> <h2><a name="Configure_the_Profiler"></a>Configure the Profiler</h2> <p>First, we’ll configure the profiler to emit a profiler every 1 minute:</p> - <ul> - + <li>In Ambari, set the profiler period duration to <tt>1</tt> minute via the Profiler config section.</li> - <li>Adjust <tt>$METRON_HOME/config/zookeeper/global.json</tt> to adjust the capture duration:</li> </ul> -<div class="source"> -<div class="source"> -<pre> "profiler.client.period.duration" : "1", +<div> +<div> +<pre class="source"> "profiler.client.period.duration" : "1", "profiler.client.period.duration.units" : "MINUTES" -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h2><a name="Create_the_Data_Generator"></a>Create the Data Generator</h2> -<p>We want to create a new sensor for our synthetic data called <tt>auth</tt>. To feed it, we need a synthetic data generator. In particular, we want a process which will feed authentication events per second for a set of users where the IPs are randomly chosen, but each user’s login ip addresses share the same first 2 octets.</p> +<p>We want to create a new sensor for our synthetic data called <tt>auth</tt>. To feed it, we need a synthetic data generator. In particular, we want a process which will feed authentication events per second for a set of users where the IPs are randomly chosen, but each user’s login ip addresses share the same first 2 octets.</p> <p>Edit <tt>~/gen_data.py</tt> and paste the following into it:</p> -<div class="source"> -<div class="source"> -<pre>#!/usr/bin/python +<div> +<div> +<pre class="source">#!/usr/bin/python import random import sys @@ -276,29 +165,26 @@ def main(): if __name__ == '__main__': main() -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h2><a name="Create_the_auth_Parser"></a>Create the <tt>auth</tt> Parser</h2> <p>The message format for our simple synthetic data is a CSV with:</p> - <ul> - + <li>username</li> - <li>login ip address</li> - <li>timestamp</li> </ul> <p>We will need to parse this via our <tt>CSVParser</tt> and add the geohash of the login ip address.</p> - <ul> - + <li>To create this parser, edit <tt>$METRON_HOME/config/zookeeper/parsers/auth.json</tt> and paste the following:</li> </ul> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "parserClassName" : "org.apache.metron.parsers.csv.CSVParser" ,"sensorTopic" : "auth" ,"parserConfig" : { @@ -321,29 +207,28 @@ if __name__ == '__main__': </pre></div></div> <ul> - + <li>Create the kafka topic via:</li> </ul> -<div class="source"> -<div class="source"> -<pre>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic auth --partitions 1 --replication-factor 1 -</pre></div></div></div> +<div> +<div> +<pre class="source">/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic auth --partitions 1 --replication-factor 1 +</pre></div></div> +</div> <div class="section"> <h2><a name="Create_the_Profiles_for_Enrichment"></a>Create the Profiles for Enrichment</h2> <p>We will need to track 2 profiles to accomplish this task:</p> - <ul> - -<li><tt>locations_by_user</tt> - The geohashes of the locations the user has logged in from. This is a multiset of geohashes per user. Note that the multiset in this case is effectively a map of geohashes to occurrance counts.</li> - + +<li><tt>locations_by_user</tt> - The geohashes of the locations the user has logged in from. This is a multiset of geohashes per user. Note that the multiset in this case is effectively a map of geohashes to occurrance counts.</li> <li><tt>geo_distribution_from_centroid</tt> - The statistical distribution of the distance between a login location and the geographic centroid of the user’s previous logins from the last 2 minutes. Note, in a real installation this would be a larger temporal lookback.</li> </ul> <p>We can represent these in the <tt>$METRON_HOME/config/zookeeper/profiler.json</tt> via the following:</p> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "profiles": [ { "profile": "geo_distribution_from_centroid", @@ -371,37 +256,32 @@ if __name__ == '__main__': } ] } -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h2><a name="Enrich_authentication_Events"></a>Enrich authentication Events</h2> <p>We will need to enrich the authentication records in a couple of ways to use in the threat triage section as well as the profiles:</p> - <ul> - + <li><tt>geo_distance</tt>: representing the distance between the current geohash and the geographic centroid for the last 2 minutes.</li> - <li><tt>geo_centroid</tt>: representing the geographic centroid for the last 2 minutes</li> </ul> <p>Beyond that, we will need to determine if the authentication event is a geographic outlier by computing the following fields:</p> - <ul> - + <li><tt>dist_median</tt> : representing the median distance between a user’s login location and the geographic centroid for the last 2 minutes (essentially the median of the <tt>geo_distance</tt> values across all users).</li> - <li><tt>dist_sd</tt> : representing the standard deviation of the distance between a user’s login location and the geographic centroid for the last 2 minutes (essentially the standard deviation of the <tt>geo_distance</tt> values across all users).</li> - <li><tt>geo_outlier</tt> : whether <tt>geo_distance</tt> is more than 5 standard deviations from the median across all users.</li> </ul> -<p>We also want to set up a triage rule associating a score and setting an alert if <tt>geo_outlier</tt> is true. In reality, this would be more complex as this metric is at best circumstantial and would need supporting evidence, but for simplicity we’ll deal with the false positives.</p> - +<p>We also want to set up a triage rule associating a score and setting an alert if <tt>geo_outlier</tt> is true. In reality, this would be more complex as this metric is at best circumstantial and would need supporting evidence, but for simplicity we’ll deal with the false positives.</p> <ul> - + <li>Edit <tt>$METRON_HOME/config/zookeeper/enrichments/auth.json</tt> and paste the following:</li> </ul> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "enrichment": { "fieldMap": { "stellar" : { @@ -445,73 +325,73 @@ if __name__ == '__main__': } } } -</pre></div></div></div> +</pre></div></div> +</div> <div class="section"> <h2><a name="Execute_Demonstration"></a>Execute Demonstration</h2> <p>From here, we’ve set up our configuration and can push the configs:</p> - <ul> - + <li>Push the configs to zookeeper via</li> </ul> -<div class="source"> -<div class="source"> -<pre>$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z $ZOOKEEPER -i $METRON_HOME/config/zookeeper/ +<div> +<div> +<pre class="source">$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z $ZOOKEEPER -i $METRON_HOME/config/zookeeper/ </pre></div></div> <ul> - + <li>Start the parser via:</li> </ul> -<div class="source"> -<div class="source"> -<pre>$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s auth +<div> +<div> +<pre class="source">$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s auth </pre></div></div> <ul> - + <li>Push synthetic data into the <tt>auth</tt> topic via</li> </ul> -<div class="source"> -<div class="source"> -<pre>python ~/gen_data.py | +<div> +<div> +<pre class="source">python ~/gen_data.py | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic auth </pre></div></div> <ul> - + <li>Wait for about <tt>5</tt> minutes and kill the previous command</li> - <li>Push a synthetic record indicating <tt>user1</tt> has logged in from a russian IP (<tt>109.252.227.173</tt>):</li> </ul> -<div class="source"> -<div class="source"> -<pre>echo -e "import time\nprint 'user1,109.252.227.173,'+str(int(time.time()))" | python | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic auth +<div> +<div> +<pre class="source">echo -e "import time\nprint 'user1,109.252.227.173,'+str(int(time.time()))" | python | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic auth </pre></div></div> <ul> - + <li>Execute the following to search elasticsearch for our geographic login outliers:</li> </ul> -<div class="source"> -<div class="source"> -<pre>curl -XPOST "http://$ES_HOST/auth*/_search?pretty" -d ' +<div> +<div> +<pre class="source">curl -XPOST "http://$ES_HOST/auth*/_search?pretty" -d ' { "_source" : [ "is_alert", "threat:triage:rules:0:reason", "user", "ip", "geo_distance" ], "query": { "exists" : { "field" : "threat:triage:rules:0:reason" } } } ' </pre></div></div> + <p>You should see, among a few other false positive results, something like the following:</p> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "_index" : "auth_index_2017.09.07.20", "_type" : "auth_doc", "_id" : "f5bdbf76-9d78-48cc-b21d-bc434c96e62e", @@ -525,23 +405,17 @@ if __name__ == '__main__': } } </pre></div></div></div> - </div> - </div> - </div> - + </div> + </div> + </div> <hr/> - <footer> - <div class="container-fluid"> - <div class="row span12">Copyright © 2018 - <a href="https://www.apache.org";>The Apache Software Foundation</a>. - All Rights Reserved. - + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> </div> - - - - </div> </footer> </body> </html>
