http://git-wip-us.apache.org/repos/asf/metron/blob/f7a94f2e/site/current-book/metron-deployment/Kerberos-ambari-setup.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/Kerberos-ambari-setup.html b/site/current-book/metron-deployment/Kerberos-ambari-setup.html new file mode 100644 index 0000000..cd22919 --- /dev/null +++ b/site/current-book/metron-deployment/Kerberos-ambari-setup.html @@ -0,0 +1,275 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2017-06-27 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Setting Up Kerberos in Vagrant Full Dev</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/"; id="bannerLeft"> + <img src="../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Setting Up Kerberos in Vagrant Full Dev</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li> + + <a href="../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-down"></i> + Deployment</a> + <ul class="nav nav-list"> + + <li class="active"> + + <a href="#"><i class="none"></i>Kerberos-ambari-setup</a> + </li> + + <li> + + <a href="../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"> + <i class="none"></i> + Kerberos-manual-setup</a> + </li> + + <li> + + <a href="../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"> + <i class="none"></i> + Amazon-ec2</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker"> + <i class="none"></i> + Ansible-docker</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker"> + <i class="none"></i> + Rpm-docker</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/packer-build/index.html" title="Packer-build"> + <i class="none"></i> + Packer-build</a> + </li> + + <li> + + <a href="../metron-deployment/roles/index.html" title="Roles"> + <i class="icon-chevron-right"></i> + Roles</a> + </li> + + <li> + + <a href="../metron-deployment/vagrant/index.html" title="Vagrant"> + <i class="icon-chevron-right"></i> + Vagrant</a> + </li> + </ul> + </li> + + <li> + + <a href="../metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + + <li> + + <a href="../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-right"></i> + Platform</a> + </li> + + <li> + + <a href="../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Setting Up Kerberos in Vagrant Full Dev</h1> +<p><a name="Setting_Up_Kerberos_in_Vagrant_Full_Dev"></a></p> +<p><b>Note:</b> These are instructions for Kerberizing Metron Storm topologies from Kafka to Kafka. This does not cover the sensor connections or MAAS. General Kerberization notes can be found in the metron-deployment <a href="../index.html">README.md</a></p> +<div class="section"> +<h2><a name="Setup_a_KDC"></a>Setup a KDC</h2> +<p>See <a href="Kerberos-manual-setup.html#Setup_a_KDC">Setup a KDC</a> and <a href="Kerberos-manual-setup.html#Verify_KDC">Verify KDC</a></p></div> +<div class="section"> +<h2><a name="Ambari_Setup"></a>Ambari Setup</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Kerberize the cluster via Ambari. More detailed documentation can be found <a class="externalLink" href="http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.5.3/bk_security/content/_enabling_kerberos_security_in_ambari.html";>here</a>.</p> +<p>a. For this exercise, choose existing MIT KDC (this is what we setup and installed in the previous steps.)</p> +<p><img src="../images/enable-kerberos.png" alt="enable keberos" /></p> +<p><img src="../images/enable-kerberos-started.png" alt="enable keberos get started" /></p> +<p>b. Setup Kerberos configuration. Realm is EXAMPLE.COM. The admin principal will end up as admin/ad...@example.com when testing the KDC. Use the password you entered during the step for adding the admin principal.</p> +<p><img src="../images/enable-kerberos-configure-kerberos.png" alt="enable keberos configure" /></p> +<p>c. Click through to “Start and Test Services.” Let the cluster spin up.</p></li> +</ol></div> +<div class="section"> +<h2><a name="Push_Data"></a>Push Data</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Kinit with the metron user</p> + +<div class="source"> +<div class="source"> +<pre>kinit -kt /etc/security/keytabs/metron.headless.keytab met...@example.com +</pre></div></div></li> +</ol> +<p>See <a href="Kerberos-manual-setup.html#Push_Data">Push Data</a></p> +<div class="section"> +<h3><a name="More_Information"></a>More Information</h3> +<p>See <a href="Kerberos-manual-setup.html#More_Information">More Information</a></p></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2017 + <a href="https://www.apache.org";>The Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/metron/blob/f7a94f2e/site/current-book/metron-deployment/Kerberos-manual-setup.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/Kerberos-manual-setup.html b/site/current-book/metron-deployment/Kerberos-manual-setup.html new file mode 100644 index 0000000..49d003b --- /dev/null +++ b/site/current-book/metron-deployment/Kerberos-manual-setup.html @@ -0,0 +1,803 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2017-06-27 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20170627" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Kerberos Setup</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/"; id="bannerLeft"> + <img src="../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Kerberos Setup</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li> + + <a href="../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-down"></i> + Deployment</a> + <ul class="nav nav-list"> + + <li> + + <a href="../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup"> + <i class="none"></i> + Kerberos-ambari-setup</a> + </li> + + <li class="active"> + + <a href="#"><i class="none"></i>Kerberos-manual-setup</a> + </li> + + <li> + + <a href="../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"> + <i class="none"></i> + Amazon-ec2</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker"> + <i class="none"></i> + Ansible-docker</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker"> + <i class="none"></i> + Rpm-docker</a> + </li> + + <li> + + <a href="../metron-deployment/packaging/packer-build/index.html" title="Packer-build"> + <i class="none"></i> + Packer-build</a> + </li> + + <li> + + <a href="../metron-deployment/roles/index.html" title="Roles"> + <i class="icon-chevron-right"></i> + Roles</a> + </li> + + <li> + + <a href="../metron-deployment/vagrant/index.html" title="Vagrant"> + <i class="icon-chevron-right"></i> + Vagrant</a> + </li> + </ul> + </li> + + <li> + + <a href="../metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + + <li> + + <a href="../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-right"></i> + Platform</a> + </li> + + <li> + + <a href="../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Kerberos Setup</h1> +<p>This document provides instructions for kerberizing Metron’s Vagrant-based development environments; “Quick Dev” and “Full Dev”. These instructions do not cover the Ambari MPack or sensors. General Kerberization notes can be found in the metron-deployment <a href="../index.html">README.md</a>.</p> + +<ul> + +<li><a href="#Setup">Setup</a></li> + +<li><a href="#Setup_a_KDC">Setup a KDC</a></li> + +<li><a href="#Verify_KDC">Verify KDC</a></li> + +<li><a href="#Enable_Kerberos">Enable Kerberos</a></li> + +<li><a href="#Kafka_Authorization">Kafka Authorization</a></li> + +<li><a href="#HBase_Authorization">HBase Authorization</a></li> + +<li><a href="#Storm_Authorization">Storm Authorization</a></li> + +<li><a href="#Start_Metron">Start Metron</a></li> + +<li><a href="#Push_Data">Push Data</a></li> + +<li><a href="#More_Information">More Information</a></li> +</ul> +<div class="section"> +<h2><a name="Setup"></a>Setup</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Deploy a Vagrant development environment; either <a href="vagrant/full-dev-platform/index.html">Full Dev</a> or <a href="vagrant/quick-dev-platform/index.html">Quick Dev</a>.</p></li> + +<li> +<p>Export the following environment variables. These need to be set for the remainder of the instructions. Replace <tt>node1</tt> with the appropriate hosts, if you are running Metron anywhere other than Vagrant.</p> + +<div class="source"> +<div class="source"> +<pre># execute as root +sudo su - +export KAFKA_HOME="/usr/hdp/current/kafka-broker" +export ZOOKEEPER=node1:2181 +export ELASTICSEARCH=node1:9200 +export BROKERLIST=node1:6667 +export HDP_HOME="/usr/hdp/current" +export KAFKA_HOME="${HDP_HOME}/kafka-broker" +export METRON_VERSION="0.4.0" +export METRON_HOME="/usr/metron/${METRON_VERSION}" +</pre></div></div></li> + +<li> +<p>Execute the following commands as root.</p> + +<div class="source"> +<div class="source"> +<pre>sudo su - +</pre></div></div></li> + +<li> +<p>Stop all Metron topologies. They will be restarted again once Kerberos has been enabled.</p> + +<div class="source"> +<div class="source"> +<pre>for topology in bro snort enrichment indexing; do + storm kill $topology; +done +</pre></div></div></li> + +<li> +<p>Create the <tt>metron</tt> user’s home directory in HDFS.</p> + +<div class="source"> +<div class="source"> +<pre>sudo -u hdfs hdfs dfs -mkdir /user/metron +sudo -u hdfs hdfs dfs -chown metron:hdfs /user/metron +sudo -u hdfs hdfs dfs -chmod 770 /user/metron +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="Setup_a_KDC"></a>Setup a KDC</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Install dependencies.</p> + +<div class="source"> +<div class="source"> +<pre>yum -y install krb5-server krb5-libs krb5-workstation +</pre></div></div></li> + +<li> +<p>Define the host, <tt>node1</tt>, as the KDC.</p> + +<div class="source"> +<div class="source"> +<pre>sed -i 's/kerberos.example.com/node1/g' /etc/krb5.conf +cp -f /etc/krb5.conf /var/lib/ambari-server/resources/scripts +</pre></div></div></li> + +<li> +<p>Ensure the KDC can issue renewable tickets. This can be necessary on a real cluster, but should not be on full-dev. In /var/kerberos/krb5kdc/kdc.conf ensure the following is in the realm section</p> + +<div class="source"> +<div class="source"> +<pre>max_renewable_life = 7d +</pre></div></div></li> + +<li> +<p>Do not copy/paste this full set of commands as the <tt>kdb5_util</tt> command will not run as expected. Run the commands individually to ensure they all execute. This step takes a moment. It creates the kerberos database.</p> + +<div class="source"> +<div class="source"> +<pre>kdb5_util create -s +/etc/rc.d/init.d/krb5kdc start +chkconfig krb5kdc on +/etc/rc.d/init.d/kadmin start +chkconfig kadmin on +</pre></div></div></li> + +<li> +<p>Setup the <tt>admin</tt> and <tt>metron</tt> principals. You’ll <tt>kinit</tt> as the <tt>metron</tt> principal when running topologies. Make sure to remember the passwords.</p> + +<div class="source"> +<div class="source"> +<pre>kadmin.local -q "addprinc admin/admin" +kadmin.local -q "addprinc metron" +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="Verify_KDC"></a>Verify KDC</h2> +<p>Ticket renewal is by default disallowed in many linux distributions. If the KDC cannot issue renewable tickets, an error will be thrown when starting Metron’s Storm topologies:</p> + +<div class="source"> +<div class="source"> +<pre>Exception in thread "main" java.lang.RuntimeException: java.lang.RuntimeException: The TGT found is not renewable +</pre></div></div> +<p>Ensure the Metron keytab is renewable. Look for the ‘R’ flag from the following command</p> + +<div class="source"> +<div class="source"> +<pre>klist -f +</pre></div></div> +<p>If the ‘R’ flags are present, you may skip to next section.</p> +<p>If the ‘R’ flags are absent, you will need to follow the below steps: If the KDC is already setup, then editing max_life and max_renewable_life in <tt>/var/kerberos/krb5kdc/kdc.conf</tt>, and restarting kadmin and krb5kdc services will not change the policies for existing users. </p> +<p>You need to set the renew lifetime for existing users and krbtgt realm. Modify the appropriate principals to allow renewable tickets using the following commands. Adjust the parameters to match your desired KDC parameters:</p> + +<div class="source"> +<div class="source"> +<pre>kadmin.local -q "modprinc -maxlife 1days -maxrenewlife 7days +allow_renewable krbtgt/example....@example.com" +kadmin.local -q "modprinc -maxlife 1days -maxrenewlife 7days +allow_renewable met...@example.com" +</pre></div></div></div> +<div class="section"> +<h2><a name="Enable_Kerberos"></a>Enable Kerberos</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>In <a class="externalLink" href="http://node1:8080";>Ambari</a>, setup Storm to use Kerberos and run worker jobs as the submitting user.</p> +<p>a. Add the following properties to the custom storm-site:</p> + +<div class="source"> +<div class="source"> +<pre>topology.auto-credentials=['org.apache.storm.security.auth.kerberos.AutoTGT'] +nimbus.credential.renewers.classes=['org.apache.storm.security.auth.kerberos.AutoTGT'] +supervisor.run.worker.as.user=true +</pre></div></div> +<p>b. In the Storm config section in Ambari, choose “Add Property” under custom storm-site:</p> +<p><img src="../images/ambari-storm-site.png" alt="custom storm-site" /></p> +<p>c. In the dialog window, choose the “bulk property add mode” toggle button and add the below values:</p> +<p><img src="../images/ambari-storm-site-properties.png" alt="custom storm-site properties" /></p></li> + +<li> +<p>Kerberize the cluster via Ambari. More detailed documentation can be found <a class="externalLink" href="http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.5.3/bk_security/content/_enabling_kerberos_security_in_ambari.html";>here</a>.</p> +<p>a. For this exercise, choose existing MIT KDC (this is what we setup and installed in the previous steps.)</p> +<p><img src="../images/enable-kerberos.png" alt="enable keberos" /></p> +<p><img src="../images/enable-kerberos-started.png" alt="enable keberos get started" /></p> +<p>b. Setup Kerberos configuration. Realm is EXAMPLE.COM. The admin principal will end up as admin/ad...@example.com when testing the KDC. Use the password you entered during the step for adding the admin principal.</p> +<p><img src="../images/enable-kerberos-configure-kerberos.png" alt="enable keberos configure" /></p> +<p>c. Click through to “Start and Test Services.” Let the cluster spin up, but don’t worry about starting up Metron via Ambari - we’re going to run the parsers manually against the rest of the Hadoop cluster Kerberized. The wizard will fail at starting Metron, but this is OK. Click “continue.” When you’re finished, the custom storm-site should look similar to the following:</p> +<p><img src="../images/custom-storm-site-final.png" alt="enable keberos configure" /></p></li> + +<li> +<p>Create a Metron keytab</p> + +<div class="source"> +<div class="source"> +<pre>kadmin.local -q "ktadd -k metron.headless.keytab met...@example.com" +cp metron.headless.keytab /etc/security/keytabs +chown metron:hadoop /etc/security/keytabs/metron.headless.keytab +chmod 440 /etc/security/keytabs/metron.headless.keytab +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="Kafka_Authorization"></a>Kafka Authorization</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Acquire a Kerberos ticket using the <tt>metron</tt> principal.</p> + +<div class="source"> +<div class="source"> +<pre>kinit -kt /etc/security/keytabs/metron.headless.keytab met...@example.com +</pre></div></div></li> + +<li> +<p>Create any additional Kafka topics that you will need. We need to create the topics before adding the required ACLs. The current full dev installation will deploy bro, snort, enrichments, and indexing only. For example, you may want to add a topic for ‘yaf’ telemetry.</p> + +<div class="source"> +<div class="source"> +<pre>${KAFKA_HOME}/bin/kafka-topics.sh \ + --zookeeper ${ZOOKEEPER} \ + --create \ + --topic yaf \ + --partitions 1 \ + --replication-factor 1 +</pre></div></div></li> + +<li> +<p>Setup Kafka ACLs for the <tt>bro</tt>, <tt>snort</tt>, <tt>enrichments</tt>, and <tt>indexing</tt> topics. Run the same command against any additional topics that you might be using; for example <tt>yaf</tt>.</p> + +<div class="source"> +<div class="source"> +<pre>export KERB_USER=metron +for topic in bro snort enrichments indexing; do + ${KAFKA_HOME}/bin/kafka-acls.sh \ + --authorizer kafka.security.auth.SimpleAclAuthorizer \ + --authorizer-properties zookeeper.connect=${ZOOKEEPER} \ + --add \ + --allow-principal User:${KERB_USER} \ + --topic ${topic} +done +</pre></div></div></li> + +<li> +<p>Setup Kafka ACLs for the consumer groups. This command sets the ACLs for Bro, Snort, YAF, Enrichments, Indexing, and the Profiler. Execute the same command for any additional Parsers that you may be running.</p> + +<div class="source"> +<div class="source"> +<pre>export KERB_USER=metron +for group in bro_parser snort_parser yaf_parser enrichments indexing profiler; do + ${KAFKA_HOME}/bin/kafka-acls.sh \ + --authorizer kafka.security.auth.SimpleAclAuthorizer \ + --authorizer-properties zookeeper.connect=${ZOOKEEPER} \ + --add \ + --allow-principal User:${KERB_USER} \ + --group ${group} +done +</pre></div></div></li> + +<li> +<p>Add the <tt>metron</tt> principal to the <tt>kafka-cluster</tt> ACL.</p> + +<div class="source"> +<div class="source"> +<pre>${KAFKA_HOME}/bin/kafka-acls.sh \ + --authorizer kafka.security.auth.SimpleAclAuthorizer \ + --authorizer-properties zookeeper.connect=${ZOOKEEPER} \ + --add \ + --allow-principal User:${KERB_USER} \ + --cluster kafka-cluster +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="HBase_Authorization"></a>HBase Authorization</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Acquire a Kerberos ticket using the <tt>hbase</tt> principal</p> + +<div class="source"> +<div class="source"> +<pre>kinit -kt /etc/security/keytabs/hbase.headless.keytab hbase-metron_clus...@example.com +</pre></div></div></li> + +<li> +<p>Grant permissions for the HBase tables used in Metron.</p> + +<div class="source"> +<div class="source"> +<pre>echo "grant 'metron', 'RW', 'threatintel'" | hbase shell +echo "grant 'metron', 'RW', 'enrichment'" | hbase shell +</pre></div></div></li> + +<li> +<p>If you are using the Profiler, do the same for its HBase table.</p> + +<div class="source"> +<div class="source"> +<pre>echo "create 'profiler', 'P'" | hbase shell +echo "grant 'metron', 'RW', 'profiler', 'P'" | hbase shell +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="Storm_Authorization"></a>Storm Authorization</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Switch to the <tt>metron</tt> user and acquire a Kerberos ticket for the <tt>metron</tt> principal.</p> + +<div class="source"> +<div class="source"> +<pre>su metron +kinit -kt /etc/security/keytabs/metron.headless.keytab met...@example.com +</pre></div></div></li> + +<li> +<p>Create the directory <tt>/home/metron/.storm</tt> and switch to that directory.</p> + +<div class="source"> +<div class="source"> +<pre>mkdir /home/metron/.storm +cd /home/metron/.storm +</pre></div></div></li> + +<li> +<p>Ensure the Metron keytab is renewable. See <a href="#Verify_KDC">Verify KDC</a> above.</p></li> + +<li> +<p>Create a client JAAS file at <tt>/home/metron/.storm/client_jaas.conf</tt>. This should look identical to the Storm client JAAS file located at <tt>/etc/storm/conf/client_jaas.conf</tt> except for the addition of a <tt>Client</tt> stanza. The <tt>Client</tt> stanza is used for Zookeeper. All quotes and semicolons are necessary.</p> + +<div class="source"> +<div class="source"> +<pre>cat << EOF > client_jaas.conf +StormClient { + com.sun.security.auth.module.Krb5LoginModule required + useTicketCache=true + renewTicket=true + serviceName="nimbus"; +}; +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="/etc/security/keytabs/metron.headless.keytab" + storeKey=true + useTicketCache=false + serviceName="zookeeper" + principal="met...@example.com"; +}; +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="/etc/security/keytabs/metron.headless.keytab" + storeKey=true + useTicketCache=false + serviceName="kafka" + principal="met...@example.com"; +}; +EOF +</pre></div></div></li> + +<li> +<p>Create a YAML file at <tt>/home/metron/.storm/storm.yaml</tt>. This should point to the client JAAS file. Set the array of nimbus hosts accordingly.</p> + +<div class="source"> +<div class="source"> +<pre>cat << EOF > /home/metron/.storm/storm.yaml +nimbus.seeds : ['node1'] +java.security.auth.login.config : '/home/metron/.storm/client_jaas.conf' +storm.thrift.transport : 'org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin' +EOF +</pre></div></div></li> + +<li> +<p>Create an auxiliary storm configuration file at <tt>/home/metron/storm-config.json</tt>. Note the login config option in the file points to the client JAAS file.</p> + +<div class="source"> +<div class="source"> +<pre>cat << EOF > /home/metron/storm-config.json +{ + "topology.worker.childopts" : "-Djava.security.auth.login.config=/home/metron/.storm/client_jaas.conf" +} +EOF +</pre></div></div></li> + +<li> +<p>Configure the Enrichment, Indexing and Profiler topologies to use the client JAAS file. To do this, the following key-value pairs:</p> + +<ul> + +<li><tt>kafka.security.protocol=PLAINTEXTSASL</tt></li> + +<li><tt>topology.worker.childopts=-Djava.security.auth.login.config=/home/metron/.storm/client_jaas.conf</tt></li> + </ul> +<p>must be added to each of the topology properties files:</p> + +<ul> + +<li><tt>${METRON_HOME}/config/enrichment.properties</tt></li> + +<li><tt>${METRON_HOME}/config/elasticsearch.properties</tt></li> + +<li><tt>${METRON_HOME}/config/profiler.properties</tt></li> + </ul> +<p>You may use the following command to automate this step:</p> + +<div class="source"> +<div class="source"> +<pre>for file in enrichment.properties elasticsearch.properties profiler.properties; do + echo ${file} + sed -i "s/^kafka.security.protocol=.*/kafka.security.protocol=PLAINTEXTSASL/" "${METRON_HOME}/config/${file}" + sed -i "s/^topology.worker.childopts=.*/topology.worker.childopts=-Djava.security.auth.login.config=\/home\/metron\/.storm\/client_jaas.conf/" "${METRON_HOME}/config/${file}" +done +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="Start_Metron"></a>Start Metron</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Switch to the <tt>metron</tt> user and acquire a Kerberos ticket for the <tt>metron</tt> principal.</p> + +<div class="source"> +<div class="source"> +<pre>su metron +kinit -kt /etc/security/keytabs/metron.headless.keytab met...@example.com +</pre></div></div></li> + +<li> +<p>Restart the parser topologies. Be sure to pass in the new parameter, <tt>-ksp</tt> or <tt>--kafka_security_protocol</tt>. The following command will start only the Bro and Snort topologies. Execute the same command for any other Parsers that you may need, for example <tt>yaf</tt>.</p> + +<div class="source"> +<div class="source"> +<pre>for parser in bro snort; do + ${METRON_HOME}/bin/start_parser_topology.sh \ + -z ${ZOOKEEPER} \ + -s ${parser} \ + -ksp SASL_PLAINTEXT \ + -e /home/metron/storm-config.json; +done +</pre></div></div></li> + +<li> +<p>Restart the Enrichment and Indexing topologies.</p> + +<div class="source"> +<div class="source"> +<pre>${METRON_HOME}/bin/start_enrichment_topology.sh +${METRON_HOME}/bin/start_elasticsearch_topology.sh +</pre></div></div></li> +</ol> +<p>Metron should be ready to receive data.</p></div> +<div class="section"> +<h2><a name="Push_Data"></a>Push Data</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Push some sample data to one of the parser topics. E.g for Bro we took raw data from <a href="../metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput/index.html">metron/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput</a></p> + +<div class="source"> +<div class="source"> +<pre>cat sample-bro.txt | ${KAFKA_HOME}/kafka-broker/bin/kafka-console-producer.sh \ + --broker-list ${BROKERLIST} \ + --security-protocol SASL_PLAINTEXT \ + --topic bro +</pre></div></div></li> + +<li> +<p>Wait a few moments for data to flow through the system and then check for data in the Elasticsearch indices. Replace yaf with whichever parser type you’ve chosen.</p> + +<div class="source"> +<div class="source"> +<pre>curl -XGET "${ELASTICSEARCH}/bro*/_search" +curl -XGET "${ELASTICSEARCH}/bro*/_count" +</pre></div></div></li> + +<li> +<p>You should have data flowing from the parsers all the way through to the indexes. This completes the Kerberization instructions</p></li> +</ol></div> +<div class="section"> +<h2><a name="More_Information"></a>More Information</h2> +<div class="section"> +<h3><a name="Kerberos"></a>Kerberos</h3> +<p>Unsure of your Kerberos principal associated with a keytab? There are a couple ways to get this. One is via the list of principals that Ambari provides via downloadable csv. If you didn’t download this list, you can also check the principal manually by running the following against the keytab.</p> + +<div class="source"> +<div class="source"> +<pre>klist -kt /etc/security/keytabs/<keytab-file-name> +</pre></div></div> +<p>E.g.</p> + +<div class="source"> +<div class="source"> +<pre>klist -kt /etc/security/keytabs/hbase.headless.keytab +Keytab name: FILE:/etc/security/keytabs/hbase.headless.keytab +KVNO Timestamp Principal +---- ----------------- -------------------------------------------------------- + 1 03/28/17 19:29:36 hbase-metron_clus...@example.com + 1 03/28/17 19:29:36 hbase-metron_clus...@example.com + 1 03/28/17 19:29:36 hbase-metron_clus...@example.com + 1 03/28/17 19:29:36 hbase-metron_clus...@example.com + 1 03/28/17 19:29:36 hbase-metron_clus...@example.com +</pre></div></div></div> +<div class="section"> +<h3><a name="Kafka_with_Kerberos_enabled"></a>Kafka with Kerberos enabled</h3> +<div class="section"> +<h4><a name="Running_Sensors"></a>Running Sensors</h4> +<p>A couple steps are required to produce data to a Kerberized Kafka topic. On the host you’ll be setting up your sensor(s), switch to the metron user and create a client_jaas.conf file in the metron home directory if one doesn’t already exist. It should be owned by metron:metron and contain at least the following stanza that tells the Kafka client how to interact with Kerberos:</p> + +<div class="source"> +<div class="source"> +<pre>su - metron +cat ${METRON_HOME}/client_jaas.conf +... +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="/etc/security/keytabs/metron.headless.keytab" + storeKey=true + useTicketCache=false + serviceName="kafka" + principal="met...@example.com"; +}; +</pre></div></div> +<p>You’ll also need to set KAFKA_OPTS to tell the Kafka client how to interact with Kerberos.</p> + +<div class="source"> +<div class="source"> +<pre>export KAFKA_OPTS="-Djava.security.auth.login.config=${METRON_HOME}/client_jaas.conf" +</pre></div></div> +<p>For sensors that leverage the Kafka console producer to pipe data into Metron, e.g. Snort and Yaf, you will need to modify the corresponding sensor shell scripts or config to append the SASL security protocol property. <tt>--security-protocol SASL_PLAINTEXT</tt>. Be sure to kinit with the metron user’s keytab before executing the script that starts the sensor.</p> +<p>More notes can be found in <a href="../metron-sensors/index.html">metron/metron-sensors/README.md</a></p></div> +<div class="section"> +<h4><a name="Write_data_to_a_topic_with_SASL"></a>Write data to a topic with SASL</h4> + +<div class="source"> +<div class="source"> +<pre>cat sample-yaf.txt | ${KAFKA_HOME}/bin/kafka-console-producer.sh \ + --broker-list ${BROKERLIST} \ + --security-protocol PLAINTEXTSASL \ + --topic yaf +</pre></div></div></div> +<div class="section"> +<h4><a name="View_topic_data_from_latest_offset_with_SASL"></a>View topic data from latest offset with SASL</h4> + +<div class="source"> +<div class="source"> +<pre>${KAFKA_HOME}/bin/kafka-console-consumer.sh \ + --zookeeper ${ZOOKEEPER} \ + --security-protocol PLAINTEXTSASL \ + --topic yaf +</pre></div></div></div> +<div class="section"> +<h4><a name="Modify_the_sensor-stubs_to_send_logs_via_SASL"></a>Modify the sensor-stubs to send logs via SASL</h4> + +<div class="source"> +<div class="source"> +<pre>sed -i 's/node1:6667 --topic/node1:6667 --security-protocol PLAINTEXTSASL --topic/' /opt/sensor-stubs/bin/start-*-stub +for sensorstub in bro snort; do + service sensor-stubs stop ${sensorstub}; + service sensor-stubs start ${sensorstub}; +done +</pre></div></div></div></div> +<div class="section"> +<h3><a name="References"></a>References</h3> + +<ul> + +<li><a class="externalLink" href="https://github.com/apache/storm/blob/master/SECURITY.md";>https://github.com/apache/storm/blob/master/SECURITY.md</a></li> +</ul></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2017 + <a href="https://www.apache.org";>The Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/metron/blob/f7a94f2e/site/current-book/metron-deployment/amazon-ec2/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/amazon-ec2/index.html b/site/current-book/metron-deployment/amazon-ec2/index.html index 8eab79a..0b155b0 100644 --- a/site/current-book/metron-deployment/amazon-ec2/index.html +++ b/site/current-book/metron-deployment/amazon-ec2/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-02-23 + | Generated by Apache Maven Doxia at 2017-06-27 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170223" /> + <meta name="Date-Revision-yyyymmdd" content="20170627" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Apache Metron on Amazon EC2</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,14 +30,11 @@ <div class="container-fluid"> <div id="banner"> <div class="pull-left"> - <a href="http://metron.incubator.apache.org/"; id="bannerLeft"> - <img src="../../images/metron-logo.png" alt="Apache Metron - Incubating" width="148px" height="48px"/> + <a href="http://metron.apache.org/"; id="bannerLeft"> + <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> </a> </div> - <div class="pull-right"> <a href="http://incubator.apache.org/"; id="bannerRight"> - <img src="../../images/ApacheIncubating_Logo.png" alt="Apache Incubating" width="192px" height="48px"/> - </a> - </div> + <div class="pull-right"> </div> <div class="clear"><hr/></div> </div> @@ -51,8 +48,8 @@ </li> <li class="divider ">/</li> <li class=""> - <a href="http://metron.incubator.apache.org/"; class="externalLink" title="Metron-Incubating"> - Metron-Incubating</a> + <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> + Metron</a> </li> <li class="divider ">/</li> <li class=""> @@ -64,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-02-23</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.3.1</li> + <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.0</li> </ul> </div> @@ -78,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../../index.html" title="Metron"> @@ -99,7 +96,7 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + <li> <a href="../../metron-deployment/index.html" title="Deployment"> @@ -107,6 +104,20 @@ Deployment</a> <ul class="nav nav-list"> + <li> + + <a href="../../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup"> + <i class="none"></i> + Kerberos-ambari-setup</a> + </li> + + <li> + + <a href="../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"> + <i class="none"></i> + Kerberos-manual-setup</a> + </li> + <li class="active"> <a href="#"><i class="none"></i>Amazon-ec2</a> @@ -128,11 +139,11 @@ <li> - <a href="../../metron-deployment/packer-build/index.html" title="Packer-build"> + <a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"> <i class="none"></i> Packer-build</a> </li> - + <li> <a href="../../metron-deployment/roles/index.html" title="Roles"> @@ -155,14 +166,28 @@ <i class="none"></i> Docker</a> </li> - + + <li> + + <a href="../../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + <li> <a href="../../metron-platform/index.html" title="Platform"> <i class="icon-chevron-right"></i> Platform</a> </li> - + <li> <a href="../../metron-sensors/index.html" title="Sensors"> @@ -202,7 +227,7 @@ <ul> -<li>Ansible 2.0.0.2</li> +<li>Ansible 2.0.0.2 or 2.2.2.0</li> <li>Python 2.7.11</li> @@ -227,12 +252,13 @@ <div class="source"> <pre> brew cask install java brew install maven git - brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/ee1273bf919a5e4e50838513a9e55ea423e1d7ce/Formula/ansible.rb - brew switch ansible 2.0.0.2 </pre></div></div></li> <li> -<p>Ensure that a public SSH key is located at <tt>~/.ssh/id_rsa.pub</tt>. </p> +<p>Install Ansible by following the instructions <a class="externalLink" href="http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip";>here</a>.</p></li> + +<li> +<p>Ensure that a public SSH key is located at <tt>~/.ssh/id_rsa.pub</tt>.</p> <div class="source"> <div class="source"> @@ -315,7 +341,7 @@ " Metron @ http://ec2-52-37-255-142.us-west-2.compute.amazonaws.com:5000", " Ambari @ http://ec2-52-37-225-202.us-west-2.compute.amazonaws.com:8080", " Sensors @ ec2-52-37-225-202.us-west-2.compute.amazonaws.com on tap0", - "For additional information, see https://metron.incubator.apache.org/'" + "For additional information, see https://metron.apache.org/'" ] } </pre></div></div></li> @@ -372,36 +398,49 @@ Enter same passphrase again: <div class="section"> <h4><a name="Solution"></a>Solution</h4> <p>If you see this error, please report your findings by creating a JIRA or dropping an email to the Metron Users mailing list. Follow these steps to work around the problem.</p> -<p>(1) Define which Elasticsearch host to interact with. Any Elasticsearch host should work.</p> +<ol style="list-style-type: decimal"> + +<li> +<p>Define which Elasticsearch host to interact with. Any Elasticsearch host should work.</p> + <div class="source"> <div class="source"> <pre>export ES_HOST="http://ec2-52-25-237-20.us-west-2.compute.amazonaws.com:9200" -</pre></div></div> -<p>(2) Confirm the index templates are in fact missing. </p> - +</pre></div></div></li> + +<li> +<p>Confirm the index templates are in fact missing. </p> + <div class="source"> <div class="source"> <pre>curl -s -XGET $ES_HOST/_template -</pre></div></div> -<p>(3) Manually load the index templates.</p> - +</pre></div></div></li> + +<li> +<p>Manually load the index templates.</p> + <div class="source"> <div class="source"> <pre>cd metron-deployment curl -s -XPOST $ES_HOST/_template/bro_index -d @roles/metron_elasticsearch_templates/files/es_templates/bro_index.template curl -s -XPOST $ES_HOST/_template/snort_index -d @roles/metron_elasticsearch_templates/files/es_templates/snort_index.template curl -s -XPOST $ES_HOST/_template/yaf_index -d @roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template -</pre></div></div> -<p>(4) Delete the existing indexes. Only a new index will use the templates defined in the previous step.</p> - +</pre></div></div></li> + +<li> +<p>Delete the existing indexes. Only a new index will use the templates defined in the previous step.</p> + <div class="source"> <div class="source"> <pre>curl -s -XDELETE "$ES_HOST/yaf_index*" curl -s -XDELETE "$ES_HOST/bro_index*" curl -s -XDELETE "$ES_HOST/snort_index*" -</pre></div></div> -<p>(5) Open up Kibana and wait for the new indexes to be created. The dashboard should now work.</p></div></div> +</pre></div></div></li> + +<li> +<p>Open up Kibana and wait for the new indexes to be created. The dashboard should now work.</p></li> +</ol></div></div> <div class="section"> <h3><a name="Error:_No_handler_was_ready_to_authenticateCheck_your_credentials"></a>Error: ‘No handler was ready to authenticate…Check your credentials’</h3> @@ -508,8 +547,9 @@ fatal: [ec2-52-26-113-221.us-west-2.compute.amazonaws.com]: UNREACHABLE! => { <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017. - All Rights Reserved. + <div class="row span12">Copyright © 2017 + <a href="https://www.apache.org";>The Apache Software Foundation</a>. + All Rights Reserved. </div> http://git-wip-us.apache.org/repos/asf/metron/blob/f7a94f2e/site/current-book/metron-deployment/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-deployment/index.html b/site/current-book/metron-deployment/index.html index 43e6678..7865a52 100644 --- a/site/current-book/metron-deployment/index.html +++ b/site/current-book/metron-deployment/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2017-02-23 + | Generated by Apache Maven Doxia at 2017-06-27 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20170223" /> + <meta name="Date-Revision-yyyymmdd" content="20170627" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Overview</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,14 +30,11 @@ <div class="container-fluid"> <div id="banner"> <div class="pull-left"> - <a href="http://metron.incubator.apache.org/"; id="bannerLeft"> - <img src="../images/metron-logo.png" alt="Apache Metron - Incubating" width="148px" height="48px"/> + <a href="http://metron.apache.org/"; id="bannerLeft"> + <img src="../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> </a> </div> - <div class="pull-right"> <a href="http://incubator.apache.org/"; id="bannerRight"> - <img src="../images/ApacheIncubating_Logo.png" alt="Apache Incubating" width="192px" height="48px"/> - </a> - </div> + <div class="pull-right"> </div> <div class="clear"><hr/></div> </div> @@ -51,8 +48,8 @@ </li> <li class="divider ">/</li> <li class=""> - <a href="http://metron.incubator.apache.org/"; class="externalLink" title="Metron-Incubating"> - Metron-Incubating</a> + <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> + Metron</a> </li> <li class="divider ">/</li> <li class=""> @@ -64,8 +61,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2017-02-23</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.3.1</li> + <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.0</li> </ul> </div> @@ -78,7 +75,7 @@ <ul class="nav nav-list"> <li class="nav-header">User Documentation</li> - + <li> <a href="../index.html" title="Metron"> @@ -99,7 +96,7 @@ <i class="icon-chevron-right"></i> Analytics</a> </li> - + <li class="active"> <a href="#"><i class="icon-chevron-down"></i>Deployment</a> @@ -107,6 +104,20 @@ <li> + <a href="../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup"> + <i class="none"></i> + Kerberos-ambari-setup</a> + </li> + + <li> + + <a href="../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"> + <i class="none"></i> + Kerberos-manual-setup</a> + </li> + + <li> + <a href="../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"> <i class="none"></i> Amazon-ec2</a> @@ -128,11 +139,11 @@ <li> - <a href="../metron-deployment/packer-build/index.html" title="Packer-build"> + <a href="../metron-deployment/packaging/packer-build/index.html" title="Packer-build"> <i class="none"></i> Packer-build</a> </li> - + <li> <a href="../metron-deployment/roles/index.html" title="Roles"> @@ -155,14 +166,28 @@ <i class="none"></i> Docker</a> </li> - + + <li> + + <a href="../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + <li> <a href="../metron-platform/index.html" title="Platform"> <i class="icon-chevron-right"></i> Platform</a> </li> - + <li> <a href="../metron-sensors/index.html" title="Sensors"> @@ -193,8 +218,30 @@ <h1>Overview</h1> <p><a name="Overview"></a></p> -<p>This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Metron services, or both using ansible playbooks. These playbooks currently only target RHEL/CentOS 6.x operating systems.</p> -<p>In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed in this README.</p> +<p>This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster containing Metron services using Ansible. These playbooks target RHEL/CentOS 6.x operating systems.</p> +<p>Installation consists of -</p> + +<ul> + +<li>Building Metron tarballs, RPMs and the Ambari MPack</li> + +<li>Deploying Ambari</li> + +<li>Leveraging Ambari to install: + +<ul> + +<li>The required Hadoop Components</li> + +<li>Core Metron (Parsing, Enrichment, Indexing)</li> + +<li>Elasticsearch</li> + +<li>Kibana</li> + </ul></li> + +<li>Starting All Services</li> +</ul> <div class="section"> <h2><a name="Prerequisites"></a>Prerequisites</h2> <p>The following tools are required to run these scripts:</p> @@ -205,16 +252,10 @@ <li><a class="externalLink" href="https://git-scm.com/";>Git</a></li> -<li><a class="externalLink" href="http://www.ansible.com/";>Ansible</a> (version 2.0 or greater)</li> -</ul> -<p>Currently Metron must be built from source. Before running these scripts perform the following steps:</p> - -<ol style="list-style-type: decimal"> +<li><a class="externalLink" href="http://www.ansible.com/";>Ansible</a> (2.0.0.2 or 2.2.2.0)</li> -<li>Clone the Metron git repository with <tt>git clone g...@github.com:apache/incubator-metron.git</tt></li> - -<li>Navigate to <tt>incubator-metron</tt> and run <tt>mvn clean package</tt></li> -</ol> +<li><a class="externalLink" href="https://www.docker.com/";>Docker</a> (Docker for Mac on OSX)</li> +</ul> <p>These scripts depend on two files for configuration:</p> <ul> @@ -223,75 +264,29 @@ <li>group_vars/all - various configuration settings needed to install Metron</li> </ul> -<p>Examples can be found in the <tt>incubator-metron/metron-deployment/inventory/metron_example</tt> directory and are a good starting point. Copy this directory into <tt>incubator-metron/metron-deployment/inventory/</tt> and rename it to your <tt>project_name</tt>. More information about Ansible files and directory structure can be found at <a class="externalLink" href="http://docs.ansible.com/ansible/playbooks_best_practices.html";>http://docs.ansible.com/ansible/playbooks_best_practices.html</a>.</p></div> +<p>For production use, it is recommended that Metron be installed on an existing cluster managed by Ambari as described in the Installing Management Pack section below.</p></div> <div class="section"> <h2><a name="Ambari"></a>Ambari</h2> -<p>The Ambari playbook will install a Hadoop cluster with all the services and configuration required by Metron. This section can be skipped if installing Metron on a pre-existing cluster.</p> -<p>Currently, this playbook supports building a local development cluster running on one node but options for other types of clusters will be added in the future.</p> +<p>The Ambari playbook will install a Hadoop cluster including the Metron Services (Parsing, Enrichment, Indexing). Ambari will also install Elasticsearch and Kibana.</p> +<p>Currently, the playbooks supports building a local development cluster running on one node or deploying to a 10 node cluster on AWS EC2.</p></div> <div class="section"> -<h3><a name="Setting_up_your_inventory"></a>Setting up your inventory</h3> -<p>Make sure to update the hosts file in <tt>incubator-metron/metron-deployment/inventory/project_name/hosts</tt> or provide an alternate inventory file when you launch the playbooks, including the ssh user(s) and ssh keyfile location(s). These playbooks expect two host groups:</p> - -<ul> - -<li>ambari_master</li> - -<li>ambari_slaves</li> -</ul></div> -<div class="section"> -<h3><a name="Running_the_playbook"></a>Running the playbook</h3> -<p>This playbook will install the Ambari server on the ambari_master, install the ambari agents on the ambari_slaves, and create a cluster in Ambari with a blueprint for the required Metron components.</p> -<p>Navigate to <tt>incubator-metron/metron-deployment/playbooks</tt> and run: <tt>ansible-playbook -i ../inventory/project_name ambari_install.yml</tt></p></div></div> -<div class="section"> -<h2><a name="Metron"></a>Metron</h2> -<p>The Metron playbook will gather the necessary cluster settings from Ambari and install the Metron services.</p> +<h2><a name="Vagrant"></a>Vagrant</h2> +<p>There are current two Vagrant modes, full-dev and quick-dev. Full-dev installs the entire Ambari/Metron stack. This is useful in testing out changes to the installation procedure. Quick-dev re-installs the core Metron Services (Parsing, Enrichment, and Indexing)on a pre-built instance. Use quick-dev for testing out changes to core Metron services.</p> <div class="section"> -<h3><a name="Setting_up_your_inventory"></a>Setting up your inventory</h3> -<p>Edit the hosts file at <tt>incubator-metron/metron-deployment/inventory/project_name/hosts</tt>. Declare where which hosts the Metron services will be installed on by updating these groups:</p> - -<ul> - -<li>enrichment - submits the topology code to Storm and requires a storm client</li> - -<li>search - host where Elasticsearch will be run</li> - -<li>web - host where the Metron UI and underlying services will run</li> - -<li>sensors - host where network data will be collected and published to Kafka</li> -</ul> -<p>The Metron topologies depend on Kafka topics and HBase tables being created beforehand. Declare a host that has Kafka and HBase clients installed by updating these groups:</p> +<h3><a name="Prerequsities"></a>Prerequsities</h3> <ul> -<li>metron_kafka_topics</li> +<li>Install <a class="externalLink" href="https://www.vagrantup.com/";>Vagrant</a> (5.0.16+)</li> -<li>metron_hbase_tables</li> -</ul> -<p>If only installing Metron, these groups can be ignored:</p> - -<ul> - -<li>ambari_master</li> - -<li>ambari_slaves</li> +<li>Install the Hostmanager plugin for vagrant - Run <tt>vagrant plugin install vagrant-hostmanager</tt> on the machine where Vagrant is installed</li> </ul></div> <div class="section"> -<h3><a name="Configuring_group_variables"></a>Configuring group variables</h3> -<p>The Metron Ansible scripts depend on a set of variables. These variables can be found in the file at <tt>incubator-metron/metron-deployment/inventory/project_name/group_vars/all</tt>. Edit the ambari* variables to match your Ambari instance and update the java_home variable to match the java path on your hosts.</p></div> -<div class="section"> -<h3><a name="Running_the_playbook"></a>Running the playbook</h3> -<p>Navigate to <tt>incubator-metron/metron-deployment/playbooks</tt> and run: <tt>ansible-playbook -i ../inventory/project_name metron_install.yml</tt></p></div></div> +<h3><a name="Full-Dev"></a>Full-Dev</h3> +<p>Navigate to <tt>metron/metron-deployment/vagrant/full-dev-platform</tt> and run <tt>vagrant up</tt>.</p></div> <div class="section"> -<h2><a name="Vagrant"></a>Vagrant</h2> -<p>A VagrantFile is included and will install a working version of the entire Metron stack. The following is required to run this:</p> - -<ul> - -<li><a class="externalLink" href="https://www.vagrantup.com/";>Vagrant</a></li> - -<li>Hostmanager plugin for vagrant - Run <tt>vagrant plugin install vagrant-hostmanager</tt> on the machine where Vagrant is installed</li> -</ul> -<p>Navigate to <tt>incubator-metron/metron-deployment/vagrant/full-dev-platform</tt> and run <tt>vagrant up</tt>. This also provides a good example of how to run a full end-to-end Metron install.</p></div> +<h3><a name="Quick-Dev"></a>Quick-Dev</h3> +<p>Navigate to <tt>metron/metron-deployment/vagrant/quick-dev-platform</tt> and run <tt>vagrant up</tt>.</p></div></div> <div class="section"> <h2><a name="Ambari_Management_Pack"></a>Ambari Management Pack</h2> <p>An Ambari Management Pack can be built in order to make the Metron service available on top of an existing stack, rather than needing a direct stack update.</p> @@ -316,9 +311,11 @@ <ul> -<li>A cluster managed by Ambari 2.4</li> +<li>A cluster managed by Ambari 2.4.2+</li> <li>Metron RPMs available on the cluster in the /localrepo directory. See <a href="#RPM">RPM</a> for further information.</li> + +<li><a class="externalLink" href="https://nodejs.org/en/download/package-manager/";>Node.js</a> repository installed on the Management UI host</li> </ul></div> <div class="section"> <h3><a name="Building_Management_Pack"></a>Building Management Pack</h3> @@ -391,6 +388,30 @@ <li>Docker. The image detailed in: <tt>metron-deployment/packaging/docker/rpm-docker/README.md</tt> will automatically be built (or rebuilt if necessary).</li> <li>Artifacts for metron-platform have been produced. E.g. <tt>mvn clean package -DskipTests</tt> in <tt>metron-platform</tt></li> +</ul> +<p>The artifacts are required because there is a dependency on modules not expressed via Maven (we grab the resulting assemblies, but don’t need the jars). These are</p> + +<ul> + +<li>metron-common</li> + +<li>metron-data-management</li> + +<li>metron-elasticsearch</li> + +<li>metron-enrichment</li> + +<li>metron-indexing</li> + +<li>metron-parsers</li> + +<li>metron-pcap-backend</li> + +<li>metron-solr</li> + +<li>metron-profiler</li> + +<li>metron-config</li> </ul></div> <div class="section"> <h3><a name="Building_RPMs"></a>Building RPMs</h3> @@ -407,6 +428,49 @@ <pre>rpm -i <package> </pre></div></div></div></div> <div class="section"> +<h2><a name="Kibana_Dashboards"></a>Kibana Dashboards</h2> +<p>The dashboards installed by the Kibana custom action are managed by the dashboard.p file. This file is created by exporting existing dashboards from a running Kibana instance.</p> +<p>To create a new version of the file, make any necessary changes to Kibana (e.g. on quick-dev), and export with the appropriate script.</p> + +<div class="source"> +<div class="source"> +<pre>python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboardindex.py \ +$ES_HOST 9200 \ +packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p -s +</pre></div></div> +<p>Build the Ambari Mpack to get the dashboard updated appropriately.</p> +<p>Once the MPack is installed, run the Kibana service’s action “Load Template” to install dashboards. This will completely overwrite the .kibana in Elasticsearch, so use with caution.</p></div> +<div class="section"> +<h2><a name="Kerberos"></a>Kerberos</h2> +<p>The MPack can allow Metron to be installed and then Kerberized, or installed on top of an already Kerberized cluster. This is done through Ambari’s standard Kerberization setup.</p> +<div class="section"> +<h3><a name="Caveats"></a>Caveats</h3> + +<ul> + +<li>For nodes using a Metron client and a local repo, the repo must exist on all nodes (e.g via createrepo). This repo can be empty; only the main Metron services need the RPMs.</li> + +<li>A Metron client must be installed on each supervisor node in a secured cluster. This is to ensure that the Metron keytab and client_jaas.conf get distributed in order to allow reading and writing from Kafka. + +<ul> + +<li>When Metron is already installed on the cluster, this should be done before Kerberizing.</li> + +<li>When addding Metron to an already Kerberized cluster, ensure that all supervisor nodes receive a Metron client.</li> + </ul></li> + +<li>Storm (and Metron) must be restarted after Metron is installed on an already Kerberized cluster. Several Storm configs get updated, and Metron will be unable to write to Kafka without a restart. + +<ul> + +<li>Kerberizing a cluster with an existing Metron already has restarts of all services during Kerberization, so it’s unneeded.</li> + </ul></li> +</ul> +<p>Instructions for setup on Full Dev can be found at <a href="Kerberos-ambari-setup.html">Kerberos-ambari-setup.md</a>. These instructions reference the manual install instructions.</p></div> +<div class="section"> +<h3><a name="Kerberos_Without_an_MPack"></a>Kerberos Without an MPack</h3> +<p>Using the MPack is preferred, but instructions for Kerberizing manually can be found at <a href="Kerberos-manual-setup.html">Kerberos-manual-setup.md</a>. These instructions are reference by the Ambari Kerberos install instructions and include commands for setting up a KDC.</p></div></div> +<div class="section"> <h2><a name="TODO"></a>TODO</h2> <ul> @@ -421,8 +485,9 @@ <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2017. - All Rights Reserved. + <div class="row span12">Copyright © 2017 + <a href="https://www.apache.org";>The Apache Software Foundation</a>. + All Rights Reserved. </div>
