Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/quick-dev-platform/index.html ============================================================================== --- dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/quick-dev-platform/index.html (added) +++ dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/quick-dev-platform/index.html Fri Apr 28 03:53:51 2017 @@ -0,0 +1,363 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2017-04-27 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20170427" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Quick Development Platform</title> + <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../../css/site.css" /> + <link rel="stylesheet" href="../../../css/print.css" media="print" /> + + + <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/" id="bannerLeft"> + <img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org" class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/" class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../../../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Quick Development Platform</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../../../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../../../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li> + + <a href="../../../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-down"></i> + Deployment</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup"> + <i class="none"></i> + Kerberos-manual-setup</a> + </li> + + <li> + + <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2"> + <i class="none"></i> + Amazon-ec2</a> + </li> + + <li> + + <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker"> + <i class="none"></i> + Ansible-docker</a> + </li> + + <li> + + <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker"> + <i class="none"></i> + Rpm-docker</a> + </li> + + <li> + + <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"> + <i class="none"></i> + Packer-build</a> + </li> + + <li> + + <a href="../../../metron-deployment/roles/index.html" title="Roles"> + <i class="icon-chevron-right"></i> + Roles</a> + </li> + + <li> + + <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant"> + <i class="icon-chevron-down"></i> + Vagrant</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup"> + <i class="none"></i> + Kerberos-setup</a> + </li> + + <li> + + <a href="../../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform"> + <i class="none"></i> + Codelab-platform</a> + </li> + + <li> + + <a href="../../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform"> + <i class="none"></i> + Fastcapa-test-platform</a> + </li> + + <li> + + <a href="../../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform"> + <i class="none"></i> + Full-dev-platform</a> + </li> + + <li class="active"> + + <a href="#"><i class="none"></i>Quick-dev-platform</a> + </li> + </ul> + </li> + </ul> + </li> + + <li> + + <a href="../../../metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + + <li> + + <a href="../../../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../../../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../../../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-right"></i> + Platform</a> + </li> + + <li> + + <a href="../../../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Quick Development Platform</h1> +<p>This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running on Virtualbox. </p> +<p>This image is designed for quick deployment of a single node Metron cluster running on Virtualbox. This platform is ideal for use by Metron developers. It uses a base image that has been pre-loaded with Ambari and HDP.</p> +<p>Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. To that end the vagrant file configuration defaults to disabling solr and yaf.</p> +<div class="section"> +<h2><a name="Getting_Started"></a>Getting Started</h2> +<div class="section"> +<h3><a name="Prerequisites"></a>Prerequisites</h3> +<p>As with the Full Development Platform (<tt>metron-deployment/vagrant/full-dev-platform</tt>), the computer used to deploy Apache Metron will need the following components installed.</p> + +<ul> + +<li><a class="externalLink" href="https://github.com/ansible/ansible">Ansible</a> (2.0.0.2 or 2.2.2.0)</li> + +<li><a class="externalLink" href="https://www.docker.com/community-edition">Docker</a></li> + +<li><a class="externalLink" href="https://www.vagrantup.com">Vagrant</a> 1.8.1</li> + +<li><a class="externalLink" href="https://github.com/devopsgroup-io/vagrant-hostmanager">Vagrant Hostmanager Plugin</a> <tt>vagrant plugin install vagrant-hostmanager</tt></li> + +<li><a class="externalLink" href="https://virtualbox.org">Virtualbox</a> 5.0.16</li> + +<li>Python 2.7.11</li> + +<li>Maven 3.3.9</li> +</ul> +<div class="section"> +<h4><a name="macOS"></a>macOS</h4> +<p>Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent <a class="externalLink" href="http://brew.sh/">Homebrew</a> project.</p> + +<ol style="list-style-type: decimal"> + +<li> +<p>Install Homebrew by following the instructions at <a class="externalLink" href="http://brew.sh/">Homebrew</a>.</p></li> + +<li> +<p>Run the following command in a terminal to install all of the required tools.</p> + +<div class="source"> +<div class="source"> +<pre> brew cask install vagrant virtualbox java docker + brew install maven git +</pre></div></div></li> + +<li> +<p>Install Ansible by following the instructions <a class="externalLink" href="http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip">here</a>.</p></li> +</ol></div></div> +<div class="section"> +<h3><a name="Deploy_Metron"></a>Deploy Metron</h3> + +<ol style="list-style-type: decimal"> + +<li> +<p>Ensure that the Docker service is running.</p></li> + +<li> +<p>Deploy Metron</p> + +<div class="source"> +<div class="source"> +<pre>cd metron-deployment/vagrant/quick-dev-platform +vagrant up +</pre></div></div> +<p>Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.</p> + +<div class="source"> +<div class="source"> +<pre>vagrant provision +</pre></div></div></li> +</ol></div> +<div class="section"> +<h3><a name="Explore_Metron"></a>Explore Metron</h3> +<p>Navigate to the following resources to explore your newly minted Apache Metron environment.</p> + +<ul> + +<li><a class="externalLink" href="http://node1:8080">Ambari</a></li> + +<li><a class="externalLink" href="http://node1:5000">Metron</a></li> + +<li><a class="externalLink" href="http://node1:2812">Services</a></li> +</ul></div> +<div class="section"> +<h3><a name="Working_with_Metron"></a>Working with Metron</h3> +<p>As you build out new capabilities for Metron, you will need to re-deploy the Storm topologies. To do so, first HALT the running Storm topologies and then run the following command.</p> + +<div class="source"> +<div class="source"> +<pre>./run_enrichment_role.sh +</pre></div></div> +<p>Connecting to the host through SSH is as simple as running the following command.</p> + +<div class="source"> +<div class="source"> +<pre>vagrant ssh +</pre></div></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2017. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-docker/index.html ============================================================================== --- dev/incubator/metron/0.4.0-RC2/book-site/metron-docker/index.html (added) +++ dev/incubator/metron/0.4.0-RC2/book-site/metron-docker/index.html Fri Apr 28 03:53:51 2017 @@ -0,0 +1,485 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2017-04-27 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20170427" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Metron Docker</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/" id="bannerLeft"> + <img src="../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org" class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/" class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Metron Docker</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li> + + <a href="../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-right"></i> + Deployment</a> + </li> + + <li class="active"> + + <a href="#"><i class="none"></i>Docker</a> + </li> + + <li> + + <a href="../metron-interface/metron-config/index.html" title="Config"> + <i class="none"></i> + Config</a> + </li> + + <li> + + <a href="../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-right"></i> + Platform</a> + </li> + + <li> + + <a href="../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Metron Docker</h1> +<p><a name="Metron_Docker"></a></p> +<p>Metron Docker is a <a class="externalLink" href="https://docs.docker.com/compose/overview/">Docker Compose</a> application that is intended for development and integration testing of Metron. Use this instead of Vagrant when:</p> + +<ul> + +<li>You want an environment that can be built and spun up quickly</li> + +<li>You need to frequently rebuild and restart services</li> + +<li>You only need to test, troubleshoot or develop against a subset of services</li> +</ul> +<p>Metron Docker includes these images that have been customized for Metron:</p> + +<ul> + +<li>Kafka (with Zookeeper)</li> + +<li>HBase</li> + +<li>Storm (with all topologies deployed)</li> + +<li>Elasticsearch</li> + +<li>Kibana</li> + +<li>HDFS</li> +</ul> +<div class="section"> +<h2><a name="Setup"></a>Setup</h2> +<p>Install <a class="externalLink" href="https://docs.docker.com/docker-for-mac/">Docker for Mac</a> or <a class="externalLink" href="https://docs.docker.com/docker-for-windows/">Docker for Windows</a>. The following versions have been tested:</p> + +<ul> + +<li>Docker version 1.12.0</li> + +<li>docker-machine version 0.8.0</li> + +<li>docker-compose version 1.8.0</li> +</ul> +<p>Build Metron from the top level directory with:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_HOME +$ mvn clean install -DskipTests +</pre></div></div> +<p>You are welcome to use an existing Docker host but we prefer one with more resources. You can create one of those with this script:</p> + +<div class="source"> +<div class="source"> +<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker +$ cd $METRON_DOCKER_HOME && ./scripts/create-docker-machine.sh +</pre></div></div> +<p>This will create a host called “metron-machine”. Anytime you want to run Docker commands against this host, make sure you run this first to set the Docker environment variables:</p> + +<div class="source"> +<div class="source"> +<pre>$ eval "$(docker-machine env metron-machine)" +</pre></div></div> +<p>If you wish to use a local docker-engine install, please set an environment variable BROKER_IP_ADDR to the IP address of your host machine. This cannot be the loopback address.</p></div> +<div class="section"> +<h2><a name="Usage"></a>Usage</h2> +<p>Navigate to the compose application root:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose/ +</pre></div></div> +<p>The Metron Docker environment lifecycle is controlled by the <a class="externalLink" href="https://docs.docker.com/compose/reference/overview/">docker-compose</a> command. The service names can be found in the docker-compose.yml file. For example, to build and start the environment run this command:</p> + +<div class="source"> +<div class="source"> +<pre>$ eval "$(docker-machine env metron-machine)" +$ docker-compose up -d +</pre></div></div> +<p>After all services have started list the containers and ensure their status is ‘Up’:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker ps --format 'table {{.Names}}\t{{.Status}}' +NAMES STATUS +metron_storm_1 Up 5 minutes +metron_hbase_1 Up 5 minutes +metron_kibana_1 Up 5 minutes +metron_kafkazk_1 Up 5 minutes +metron_elasticsearch_1 Up 5 minutes +</pre></div></div> +<p>Various services are exposed through http on the Docker host. Get the host ip from the URL property:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-machine ls +NAME ACTIVE DRIVER STATE URL SWARM DOCKER ERRORS +metron-machine * virtualbox Running tcp://192.168.99.100:2376 v1.12.5 +</pre></div></div> +<p>Then, assuming a host ip of <tt>192.168.99.100</tt>, the UIs and APIs are available at:</p> + +<ul> + +<li>Storm - <a class="externalLink" href="http://192.168.99.100:8080/">http://192.168.99.100:8080/</a></li> + +<li>HBase - <a class="externalLink" href="http://192.168.99.100:16010/">http://192.168.99.100:16010/</a></li> + +<li>Elasticsearch - <a class="externalLink" href="http://192.168.99.100:9200/_plugin/head/">http://192.168.99.100:9200/_plugin/head/</a></li> + +<li>Kibana - <a class="externalLink" href="http://192.168.99.100:5601/">http://192.168.99.100:5601/</a></li> + +<li>HDFS (Namenode) - <a class="externalLink" href="http://192.168.99.100:50070/">http://192.168.99.100:50070/</a></li> +</ul> +<p>The Storm logs can be useful when troubleshooting topologies. They can be found on the Storm container in <tt>/usr/share/apache-storm/logs</tt>.</p> +<p>When done using the machine, shut it down with:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose down +</pre></div></div></div> +<div class="section"> +<h2><a name="Examples"></a>Examples</h2> + +<ul> + +<li><a href="#Deploy_a_new_parser_class">Deploy a new parser class</a></li> + +<li><a href="#Connect_to_a_container">Connect to a container</a></li> + +<li><a href="create-a-sensor-from-sample-data/index.html">Create a sensor from sample data</a></li> + +<li><a href="upload-configs-to-zookeeper/index.html">Upload configs to Zookeeper</a></li> + +<li><a href="manage-a-topology/index.html">Manage a topology</a></li> + +<li><a href="run-sensor-data-end-to-end/index.html">Run sensor data end to end</a></li> +</ul> +<div class="section"> +<h3><a name="Deploy_a_new_parser_class"></a>Deploy a new parser class</h3> +<p>After adding a new parser to metron-parsers, build Metron from the top level directory:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_HOME +$ mvn clean install -DskipTests +</pre></div></div> +<p>Then run these commands to redeploy the parsers to the Storm image:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ docker-compose down +$ docker-compose build storm +$ docker-compose up -d +</pre></div></div></div> +<div class="section"> +<h3><a name="Connect_to_a_container"></a>Connect to a container</h3> +<p>Suppose there is a problem with Kafka and the logs are needed for further investigation. Run this command to connect and explore the running Kafka container:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ docker-compose exec kafkazk bash +</pre></div></div></div> +<div class="section"> +<h3><a name="Create_a_sensor_from_sample_data"></a>Create a sensor from sample data</h3> +<p>A tool for producing test data in Kafka is included with the Kafka/Zookeeper image. It loops through lines in a test data file and outputs them to Kafka at the desired frequency. Create a test data file in <tt>./kafkazk/data/</tt> and rebuild the Kafka/Zookeeper image:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ printf 'first test data\nsecond test data\nthird test data\n' > ./kafkazk/data/TestData.txt +$ docker-compose down +$ docker-compose build kafkazk +$ docker-compose up -d +</pre></div></div> +<p>This will deploy the test data file to the Kafka/Zookeeper container. Now that data can be streamed to a Kafka topic:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk ./bin/produce-data.sh +Usage: produce-data.sh data_path topic [message_delay_in_seconds] + +# Stream data in TestData.txt to the 'test' Kafka topic at a frequency of 5 seconds (default is 1 second) +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5 +</pre></div></div> +<p>The Kafka/Zookeeper image comes with sample Bro and Squid data:</p> + +<div class="source"> +<div class="source"> +<pre># Stream Bro test data every 1 second +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro + +# Stream Squid test data every 0.1 seconds +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/SquidExampleOutput.txt squid 0.1 +</pre></div></div></div> +<div class="section"> +<h3><a name="Upload_configs_to_Zookeeper"></a>Upload configs to Zookeeper</h3> +<p>Parser configs and a global config configured for this Docker environment are included with the Kafka/Zookeeper image. Load them with:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk bash +# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m PUSH -i $METRON_HOME/config/zookeeper +# exit +</pre></div></div> +<p>Dump out the configs with:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk bash +# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m DUMP +# exit +</pre></div></div></div> +<div class="section"> +<h3><a name="Manage_a_topology"></a>Manage a topology</h3> +<p>The Storm image comes with a script to easily start parser topologies:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_docker_parser_topology.sh sensor_name +</pre></div></div> +<p>The enrichment topology can be started with:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh +</pre></div></div> +<p>The indexing topology can be started with:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh +</pre></div></div> +<p>Topologies can be stopped using the Storm CLI. For example, stop the enrichment topology with:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm storm kill enrichments -w 0 +</pre></div></div></div> +<div class="section"> +<h3><a name="Run_sensor_data_end_to_end"></a>Run sensor data end to end</h3> +<p>First ensure configs were uploaded as described in the previous example. Then start a sensor and leave it running:</p> + +<div class="source"> +<div class="source"> +<pre>$ cd $METRON_DOCKER_HOME/compose +$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro +</pre></div></div> +<p>Open a separate console session and verify the sensor is running by consuming a message from Kafka:</p> + +<div class="source"> +<div class="source"> +<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker +$ cd $METRON_DOCKER_HOME/compose +$ docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic bro +</pre></div></div> +<p>A new message should be printed every second. Now kill the consumer and start the Bro parser topology:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec storm ./bin/start_docker_parser_topology.sh bro +</pre></div></div> +<p>Bro data should be flowing through the bro parser topology and into the Kafka enrichments topic. The enrichments topic should be created automatically:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-compose exec kafkazk ./bin/kafka-topics.sh --zookeeper localhost:2181 --list +bro +enrichments +indexing +</pre></div></div> +<p>Verify parsed Bro data is in the Kafka enrichments topic:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic enrichments +</pre></div></div> +<p>Now start the enrichment topology:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh +</pre></div></div> +<p>Parsed Bro data should be flowing through the enrichment topology and into the Kafka indexing topic. Verify enriched Bro data is in the Kafka indexing topic:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic indexing +</pre></div></div> +<p>Now start the indexing topology:</p> + +<div class="source"> +<div class="source"> +<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh +</pre></div></div> +<p>Enriched Bro data should now be present in the Elasticsearch container:</p> + +<div class="source"> +<div class="source"> +<pre>$ docker-machine ls +NAME ACTIVE DRIVER STATE URL SWARM DOCKER ERRORS +metron-machine * virtualbox Running tcp://192.168.99.100:2376 v1.12.5 + +$ curl -XGET http://192.168.99.100:9200/_cat/indices?v +health status index pri rep docs.count docs.deleted store.size pri.store.size +yellow open .kibana 1 1 1 0 3.1kb 3.1kb +yellow open bro_index_2016.12.19.18 5 1 180 0 475kb 475kb +</pre></div></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2017. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html> Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-interface/metron-config/index.html ============================================================================== --- dev/incubator/metron/0.4.0-RC2/book-site/metron-interface/metron-config/index.html (added) +++ dev/incubator/metron/0.4.0-RC2/book-site/metron-interface/metron-config/index.html Fri Apr 28 03:53:51 2017 @@ -0,0 +1,290 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2017-04-27 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20170427" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Metron Management UI</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + + + <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="http://metron.apache.org/" id="bannerLeft"> + <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> + </a> + </div> + <div class="pull-right"> </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org" class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="http://metron.apache.org/" class="externalLink" title="Metron"> + Metron</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="../../index.html" title="Documentation"> + Documentation</a> + </li> + <li class="divider ">/</li> + <li class="">Metron Management UI</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.4.0</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + + <li> + + <a href="../../index.html" title="Metron"> + <i class="icon-chevron-down"></i> + Metron</a> + <ul class="nav nav-list"> + + <li> + + <a href="../../Upgrading.html" title="Upgrading"> + <i class="none"></i> + Upgrading</a> + </li> + + <li> + + <a href="../../metron-analytics/index.html" title="Analytics"> + <i class="icon-chevron-right"></i> + Analytics</a> + </li> + + <li> + + <a href="../../metron-deployment/index.html" title="Deployment"> + <i class="icon-chevron-right"></i> + Deployment</a> + </li> + + <li> + + <a href="../../metron-docker/index.html" title="Docker"> + <i class="none"></i> + Docker</a> + </li> + + <li class="active"> + + <a href="#"><i class="none"></i>Config</a> + </li> + + <li> + + <a href="../../metron-interface/metron-rest/index.html" title="Rest"> + <i class="none"></i> + Rest</a> + </li> + + <li> + + <a href="../../metron-platform/index.html" title="Platform"> + <i class="icon-chevron-right"></i> + Platform</a> + </li> + + <li> + + <a href="../../metron-sensors/index.html" title="Sensors"> + <i class="icon-chevron-right"></i> + Sensors</a> + </li> + </ul> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <h1>Metron Management UI</h1> +<p><a name="Metron_Management_UI"></a></p> +<p>This module provides a user interface for management functions in Metron.</p> +<div class="section"> +<h2><a name="Prerequisites"></a>Prerequisites</h2> + +<ul> + +<li>A network accessible Metron REST application</li> + +<li>nodejs v6.9+ (nodejs can be installed on quick dev with <tt>curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - && yum install -y nodejs</tt>)</li> +</ul></div> +<div class="section"> +<h2><a name="Installation"></a>Installation</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Build Metron:</p> + +<div class="source"> +<div class="source"> +<pre>mvn clean package -DskipTests +</pre></div></div></li> + +<li> +<p>Copy <tt>incubator-metron/metron-interface/metron-config/target/metron-config-METRON_VERSION-archive.tar.gz</tt> to the desired host.</p></li> + +<li> +<p>Untar the archive in the target directory. The directory structure will look like:</p> + +<div class="source"> +<div class="source"> +<pre>bin + start_management_ui.sh +web + expressjs + package.json + server.js + management-ui + web assets (html, css, js, ...) +</pre></div></div></li> + +<li> +<p>For production use, the contents of the <tt>./web/management-ui</tt> directory should be deployed to a web server with paths <tt>/api/v1</tt> and <tt>/logout</tt> mapped to the REST application url. </p></li> + +<li> +<p>As an example, a convenience script is included that will install a simple <a class="externalLink" href="https://github.com/expressjs/express">expressjs</a> webserver.</p></li> + +<li> +<p>Then start the application with the script:</p> + +<div class="source"> +<div class="source"> +<pre>./bin/start_management_ui.sh +Usage: server.js -p [port] -r [restUrl] +Options: + -p Port to run metron management ui [required] + -r, --resturl Url where metron rest application is available [required] +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="Usage"></a>Usage</h2> +<p>The application will be available at <a class="externalLink" href="http://host:4200">http://host:4200</a> with credentials <tt>user/password</tt>, assuming the default port is configured and the <tt>dev</tt> profile is included when starting the REST application. See the <a href="../metron-rest/index.html#security">REST application</a> documentation for more information about security configuration for production.</p></div> +<div class="section"> +<h2><a name="Development"></a>Development</h2> +<p>The Management UI can also be started in development mode. This allows changes to web assets to be seen interactively.</p> + +<ol style="list-style-type: decimal"> + +<li> +<p>Install the application with dev dependencies:</p> + +<div class="source"> +<div class="source"> +<pre>npm install +</pre></div></div></li> + +<li> +<p>Start the application:</p> + +<div class="source"> +<div class="source"> +<pre>./scripts/start_dev.sh +</pre></div></div></li> +</ol> +<p>The application will be available at <a class="externalLink" href="http://localhost:4200/">http://localhost:4200/</a>. The REST application url defaults to <tt>http://localhost:8080</tt> but can be changed in the <tt>proxy.conf.json</tt> file.</p></div> +<div class="section"> +<h2><a name="Testing"></a>Testing</h2> + +<ol style="list-style-type: decimal"> + +<li> +<p>Install the application with dev dependencies:</p> + +<div class="source"> +<div class="source"> +<pre>npm install +</pre></div></div></li> + +<li> +<p>Unit tests can be run with:</p> + +<div class="source"> +<div class="source"> +<pre>npm test +</pre></div></div></li> +</ol></div> +<div class="section"> +<h2><a name="License"></a>License</h2> +<p>This projects bundles Font Awesome which is available under the SIL Open Font License. See <a class="externalLink" href="http://fontawesome.io/license/">http://fontawesome.io/license/</a> for more details.</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2017. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>