http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-api/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-api/index.html b/site/current-book/metron-platform/metron-api/index.html index 690ab94..ddcaec5 100644 --- a/site/current-book/metron-platform/metron-api/index.html +++ b/site/current-book/metron-platform/metron-api/index.html @@ -1,346 +1,161 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2018-01-03 - | Rendered using Apache Maven Fluido Skin 1.3.0 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-api/index.md at 2018-06-07 + | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180103" /> + <meta name="Date-Revision-yyyymmdd" content="20180607" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Metron PCAP Service</title> - <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> <link rel="stylesheet" href="../../css/site.css" /> <link rel="stylesheet" href="../../css/print.css" media="print" /> - - - <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> - - - -<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> - - </head> - <body class="topBarDisabled"> - - - - - <div class="container-fluid"> - <div id="banner"> - <div class="pull-left"> - <a href="http://metron.apache.org/"; id="bannerLeft"> - <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> - </a> - </div> - <div class="pull-right"> </div> + <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> <div class="clear"><hr/></div> </div> <div id="breadcrumbs"> <ul class="breadcrumb"> - - - <li class=""> - <a href="http://www.apache.org"; class="externalLink" title="Apache"> - Apache</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> - Metron</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="../../index.html" title="Documentation"> - Documentation</a> - </li> - <li class="divider ">/</li> - <li class="">Metron PCAP Service</li> - - - - <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.2</li> - - </ul> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active ">Metron PCAP Service</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> + <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + </ul> </div> - - <div class="row-fluid"> - <div id="leftColumn" class="span3"> + <div id="leftColumn" class="span2"> <div class="well sidebar-nav"> - - - <ul class="nav nav-list"> - <li class="nav-header">User Documentation</li> - - <li> - - <a href="../../index.html" title="Metron"> - <i class="icon-chevron-down"></i> - Metron</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../Upgrading.html" title="Upgrading"> - <i class="none"></i> - Upgrading</a> - </li> - - <li> - - <a href="../../metron-analytics/index.html" title="Analytics"> - <i class="icon-chevron-right"></i> - Analytics</a> - </li> - - <li> - - <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> - <i class="none"></i> - Docker</a> - </li> - - <li> - - <a href="../../metron-deployment/index.html" title="Deployment"> - <i class="icon-chevron-right"></i> - Deployment</a> - </li> - - <li> - - <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> - <i class="none"></i> - Alerts</a> - </li> - - <li> - - <a href="../../metron-interface/metron-config/index.html" title="Config"> - <i class="none"></i> - Config</a> - </li> - - <li> - - <a href="../../metron-interface/metron-rest/index.html" title="Rest"> - <i class="none"></i> - Rest</a> - </li> - - <li> - - <a href="../../metron-platform/index.html" title="Platform"> - <i class="icon-chevron-down"></i> - Platform</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"> - <i class="none"></i> - Performance-tuning-guide</a> - </li> - - <li class="active"> - - <a href="#"><i class="none"></i>Api</a> - </li> - - <li> - - <a href="../../metron-platform/metron-common/index.html" title="Common"> - <i class="none"></i> - Common</a> - </li> - - <li> - - <a href="../../metron-platform/metron-data-management/index.html" title="Data-management"> - <i class="none"></i> - Data-management</a> - </li> - - <li> - - <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> - <i class="none"></i> - Elasticsearch</a> - </li> - - <li> - - <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> - <i class="none"></i> - Enrichment</a> - </li> - - <li> - - <a href="../../metron-platform/metron-indexing/index.html" title="Indexing"> - <i class="none"></i> - Indexing</a> - </li> - - <li> - - <a href="../../metron-platform/metron-management/index.html" title="Management"> - <i class="none"></i> - Management</a> - </li> - - <li> - - <a href="../../metron-platform/metron-parsers/index.html" title="Parsers"> - <i class="icon-chevron-right"></i> - Parsers</a> - </li> - - <li> - - <a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"> - <i class="none"></i> - Pcap-backend</a> - </li> - - <li> - - <a href="../../metron-platform/metron-writer/index.html" title="Writer"> - <i class="none"></i> - Writer</a> - </li> - </ul> - </li> - - <li> - - <a href="../../metron-sensors/index.html" title="Sensors"> - <i class="icon-chevron-right"></i> - Sensors</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> - <i class="none"></i> - Stellar-3rd-party-example</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> - <i class="icon-chevron-right"></i> - Stellar-common</a> - </li> - - <li> - - <a href="../../use-cases/index.html" title="Use-cases"> - <i class="icon-chevron-right"></i> - Use-cases</a> - </li> - </ul> - </li> - </ul> - - - - <hr class="divider" /> - - <div id="poweredBy"> - <div class="clear"></div> - <div class="clear"></div> - <div class="clear"></div> - <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> - <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> - </a> - </div> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> + <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> + <ul class="nav nav-list"> + <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> + <li class="active"><a href="#"><span class="none"></span>Api</a></li> + <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> + <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> + <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> + <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> + <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> + <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li> + <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> + </ul> +</li> + <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> + </div> </div> </div> - - - <div id="bodyColumn" class="span9" > - - <h1>Metron PCAP Service</h1> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<h1>Metron PCAP Service</h1> <p><a name="Metron_PCAP_Service"></a></p> -<p>The purpose of the Metron PCAP service is to provide a middle tier to negotiate retrieving packet capture data which flows into Metron. This packet data is of a form which <tt>libpcap</tt> based tools can read.</p> +<p>The purpose of the Metron PCAP service is to provide a middle tier to negotiate retrieving packet capture data which flows into Metron. This packet data is of a form which <tt>libpcap</tt> based tools can read.</p> <div class="section"> <h2><a name="Starting_the_Service"></a>Starting the Service</h2> <p>You can start the service either via the init.d script installed, <tt>/etc/init.d/pcapservice</tt> or directly via the <tt>yarn jar</tt> command: <tt>yarn jar $METRON_HOME/lib/metron-api-$METRON_VERSION.jar org.apache.metron.pcapservice.rest.PcapService -port $SERVICE_PORT -query_hdfs_path $QUERY_PATH -pcap_hdfs_path $PCAP_PATH</tt></p> <p>where</p> - <ul> - + <li><tt>METRON_HOME</tt> is the location of the metron installation</li> - <li><tt>METRON_VERSION</tt> is the version of the metron installation</li> - <li><tt>SERVICE_PORT</tt> is the port to bind the REST service to.</li> - -<li><tt>QUERY_PATH</tt> is the temporary location to store query results. They are deleted after the service reads them.</li> - +<li><tt>QUERY_PATH</tt> is the temporary location to store query results. They are deleted after the service reads them.</li> <li><tt>PCAP_PATH</tt> is the path to the packet data on HDFS</li> </ul></div> <div class="section"> -<h2><a name="The_pcapGettergetPcapsByIdentifiers_endpoint"></a>The <tt>/pcapGetter/getPcapsByIdentifiers</tt> endpoint</h2> +<h2><a name="The_.2FpcapGetter.2FgetPcapsByIdentifiers_endpoint"></a>The <tt>/pcapGetter/getPcapsByIdentifiers</tt> endpoint</h2> <p>This endpoint takes the following query parameters and returns the subset of packets matching this query:</p> - <ul> - + <li><tt>srcIp</tt> : The source IP to match on</li> - <li><tt>srcPort</tt> : The source port to match on</li> - <li><tt>dstIp</tt> : The destination IP to match on</li> - <li><tt>dstPort</tt> : The destination port to match on</li> - <li><tt>startTime</tt> : The start time in milliseconds</li> - <li><tt>endTime</tt> : The end time in milliseconds</li> - <li><tt>numReducers</tt> : Specify the number of reducers to use when executing the mapreduce job</li> - <li><tt>includeReverseTraffic</tt> : Indicates if filter should check swapped src/dest addresses and IPs</li> </ul></div> <div class="section"> -<h2><a name="The_pcapGettergetPcapsByQuery_endpoint"></a>The <tt>/pcapGetter/getPcapsByQuery</tt> endpoint</h2> +<h2><a name="The_.2FpcapGetter.2FgetPcapsByQuery_endpoint"></a>The <tt>/pcapGetter/getPcapsByQuery</tt> endpoint</h2> <p>This endpoint takes the following query parameters and returns the subset of packets matching this query. This endpoint exposes Stellar querying capabilities:</p> - <ul> - + <li><tt>query</tt> : The Stellar query to execute</li> - <li><tt>startTime</tt> : The start time in milliseconds</li> - <li><tt>endTime</tt> : The end time in milliseconds</li> - <li><tt>numReducers</tt> : Specify the number of reducers to use when executing the mapreduce job</li> </ul> <p>Example: <tt>curl -XGET "http://node1:8081/pcapGetter/getPcapsByQuery?query=ip_src_addr+==+'192.168.66.121'+and+ip_src_port+==+'60500'&startTime=1476936000000"</tt></p> -<p>All of these parameters are optional. In the case of a missing parameter, it is treated as a wildcard.</p> +<p>All of these parameters are optional. In the case of a missing parameter, it is treated as a wildcard.</p> <p>Unlike the CLI tool, there is no paging mechanism. The REST API will stream back data as a single file.</p></div> - </div> - </div> - </div> - + </div> + </div> + </div> <hr/> - <footer> - <div class="container-fluid"> - <div class="row span12">Copyright © 2018 - <a href="https://www.apache.org";>The Apache Software Foundation</a>. - All Rights Reserved. - + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> </div> - - - - </div> </footer> </body> </html>
http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-platform/metron-common/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-common/index.html b/site/current-book/metron-platform/metron-common/index.html index 7111111..8d0dc0e 100644 --- a/site/current-book/metron-platform/metron-common/index.html +++ b/site/current-book/metron-platform/metron-common/index.html @@ -1,330 +1,152 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2018-01-03 - | Rendered using Apache Maven Fluido Skin 1.3.0 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-common/index.md at 2018-06-07 + | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180103" /> + <meta name="Date-Revision-yyyymmdd" content="20180607" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Contents</title> - <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> <link rel="stylesheet" href="../../css/site.css" /> <link rel="stylesheet" href="../../css/print.css" media="print" /> - - - <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script> - - - -<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> - - </head> - <body class="topBarDisabled"> - - - - - <div class="container-fluid"> - <div id="banner"> - <div class="pull-left"> - <a href="http://metron.apache.org/"; id="bannerLeft"> - <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/> - </a> - </div> - <div class="pull-right"> </div> + <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> <div class="clear"><hr/></div> </div> <div id="breadcrumbs"> <ul class="breadcrumb"> - - - <li class=""> - <a href="http://www.apache.org"; class="externalLink" title="Apache"> - Apache</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="http://metron.apache.org/"; class="externalLink" title="Metron"> - Metron</a> - </li> - <li class="divider ">/</li> - <li class=""> - <a href="../../index.html" title="Documentation"> - Documentation</a> - </li> - <li class="divider ">/</li> - <li class="">Contents</li> - - - - <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.4.2</li> - - </ul> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active ">Contents</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> + <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + </ul> </div> - - <div class="row-fluid"> - <div id="leftColumn" class="span3"> + <div id="leftColumn" class="span2"> <div class="well sidebar-nav"> - - - <ul class="nav nav-list"> - <li class="nav-header">User Documentation</li> - - <li> - - <a href="../../index.html" title="Metron"> - <i class="icon-chevron-down"></i> - Metron</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../Upgrading.html" title="Upgrading"> - <i class="none"></i> - Upgrading</a> - </li> - - <li> - - <a href="../../metron-analytics/index.html" title="Analytics"> - <i class="icon-chevron-right"></i> - Analytics</a> - </li> - - <li> - - <a href="../../metron-contrib/metron-docker/index.html" title="Docker"> - <i class="none"></i> - Docker</a> - </li> - - <li> - - <a href="../../metron-deployment/index.html" title="Deployment"> - <i class="icon-chevron-right"></i> - Deployment</a> - </li> - - <li> - - <a href="../../metron-interface/metron-alerts/index.html" title="Alerts"> - <i class="none"></i> - Alerts</a> - </li> - - <li> - - <a href="../../metron-interface/metron-config/index.html" title="Config"> - <i class="none"></i> - Config</a> - </li> - - <li> - - <a href="../../metron-interface/metron-rest/index.html" title="Rest"> - <i class="none"></i> - Rest</a> - </li> - - <li> - - <a href="../../metron-platform/index.html" title="Platform"> - <i class="icon-chevron-down"></i> - Platform</a> - <ul class="nav nav-list"> - - <li> - - <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"> - <i class="none"></i> - Performance-tuning-guide</a> - </li> - - <li> - - <a href="../../metron-platform/metron-api/index.html" title="Api"> - <i class="none"></i> - Api</a> - </li> - - <li class="active"> - - <a href="#"><i class="none"></i>Common</a> - </li> - - <li> - - <a href="../../metron-platform/metron-data-management/index.html" title="Data-management"> - <i class="none"></i> - Data-management</a> - </li> - - <li> - - <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"> - <i class="none"></i> - Elasticsearch</a> - </li> - - <li> - - <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"> - <i class="none"></i> - Enrichment</a> - </li> - - <li> - - <a href="../../metron-platform/metron-indexing/index.html" title="Indexing"> - <i class="none"></i> - Indexing</a> - </li> - - <li> - - <a href="../../metron-platform/metron-management/index.html" title="Management"> - <i class="none"></i> - Management</a> - </li> - - <li> - - <a href="../../metron-platform/metron-parsers/index.html" title="Parsers"> - <i class="icon-chevron-right"></i> - Parsers</a> - </li> - - <li> - - <a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"> - <i class="none"></i> - Pcap-backend</a> - </li> - - <li> - - <a href="../../metron-platform/metron-writer/index.html" title="Writer"> - <i class="none"></i> - Writer</a> - </li> - </ul> - </li> - - <li> - - <a href="../../metron-sensors/index.html" title="Sensors"> - <i class="icon-chevron-right"></i> - Sensors</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"> - <i class="none"></i> - Stellar-3rd-party-example</a> - </li> - - <li> - - <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"> - <i class="icon-chevron-right"></i> - Stellar-common</a> - </li> - - <li> - - <a href="../../use-cases/index.html" title="Use-cases"> - <i class="icon-chevron-right"></i> - Use-cases</a> - </li> - </ul> - </li> - </ul> - - - - <hr class="divider" /> - - <div id="poweredBy"> - <div class="clear"></div> - <div class="clear"></div> - <div class="clear"></div> - <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> - <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /> - </a> - </div> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> + <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> + <ul class="nav nav-list"> + <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> + <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> + <li class="active"><a href="#"><span class="none"></span>Common</a></li> + <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> + <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> + <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> + <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> + <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li> + <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> + </ul> +</li> + <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> + </div> </div> </div> - - - <div id="bodyColumn" class="span9" > - - <h1>Contents</h1> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<h1>Contents</h1> <p><a name="Contents"></a></p> - <ul> - + <li><a href="#Stellar_Language">Stellar Language</a></li> - <li><a href="#Global_Configuration">Global Configuration</a></li> - <li><a href="#Validation_Framework">Validation Framework</a></li> - <li><a href="#Management_Utility">Management Utility</a></li> - <li><a href="topology-errors/index.html">Topology Errors</a></li> - <li><a href="#Performance_Logging">Performance Logging</a></li> +<li><a href="#Metron_Debugging">Metron Debugging</a></li> </ul> <p><a name="Stellar_Language"></a></p> <h1>Stellar Language</h1> -<p>For a variety of components (threat intelligence triage and field transformations) we have the need to do simple computation and transformation using the data from messages as variables.<br />For those purposes, there exists a simple, scaled down DSL created to do simple computation and transformation.</p> +<p>For a variety of components (threat intelligence triage and field transformations) we have the need to do simple computation and transformation using the data from messages as variables. For those purposes, there exists a simple, scaled down DSL created to do simple computation and transformation.</p> <p>The query language supports the following:</p> - <ul> - + <li>Referencing fields in the enriched JSON</li> - <li>String literals are quoted with either <tt>'</tt> or <tt>"</tt>, and support escaping for <tt>'</tt>, <tt>"</tt>, <tt>\t</tt>, <tt>\r</tt>, <tt>\n</tt>, and backslash</li> - <li>Simple boolean operations: <tt>and</tt>, <tt>not</tt>, <tt>or</tt> - <ul> - + <li>Boolean expressions are short-circuited (e.g. <tt>true or FUNC()</tt> would never execute <tt>FUNC</tt>)</li> - </ul></li> - +</ul> +</li> <li>Simple arithmetic operations: <tt>*</tt>, <tt>/</tt>, <tt>+</tt>, <tt>-</tt> on real numbers or integers</li> - <li>Simple comparison operations <tt><</tt>, <tt>></tt>, <tt><=</tt>, <tt>>=</tt></li> - <li>Simple equality comparison operations <tt>==</tt>, <tt>!=</tt></li> - <li>if/then/else comparisons (i.e. <tt>if var1 < 10 then 'less than 10' else '10 or more'</tt>)</li> - <li>Determining whether a field exists (via <tt>exists</tt>)</li> - <li>An <tt>in</tt> operator that works like the <tt>in</tt> in Python</li> - <li>The ability to have parenthesis to make order of operations explicit</li> - <li>User defined functions, including Lambda expressions</li> </ul> <p>For documentation of Stellar, please see the <a href="../../metron-stellar/stellar-common/index.html">Stellar README</a>.</p> <p><a name="Global_Configuration"></a></p> <h1>Global Configuration</h1> -<p>The format of the global enrichment is a JSON String to Object map. This is intended for configuration which is non sensor specific configuration.</p> +<p>The format of the global enrichment is a JSON String to Object map. This is intended for configuration which is non sensor specific configuration.</p> <p>This configuration is stored in zookeeper, but looks something like</p> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "es.clustername": "metron", "es.ip": "node1", "es.port": "9300", @@ -337,237 +159,140 @@ "config" : { "type" : "IPV4" } - } + } ] } </pre></div></div> -<p>Various parts of our stack uses the global config are documented throughout the Metron documentation, but a convenient index is provided here:</p> +<p>Various parts of our stack uses the global config are documented throughout the Metron documentation, but a convenient index is provided here:</p> <table border="0" class="table table-striped"> - <thead> - +<thead> + <tr class="a"> - -<th>Property Name </th> - -<th>Subsystem </th> - -<th>Type </th> - -<th>Ambari Property </th> - </tr> - </thead> - <tbody> - +<th> Property Name </th> +<th> Subsystem </th> +<th> Type </th> +<th> Ambari Property </th></tr> +</thead><tbody> + <tr class="b"> - -<td><a href="../metron-elasticsearch/index.html#es.clustername"><tt>es.clustername</tt></a> </td> - -<td>Indexing </td> - -<td>String </td> - -<td><tt>es_cluster_name</tt> </td> - </tr> - +<td> <a href="../metron-elasticsearch/index.html#es.clustername"><tt>es.clustername</tt></a> </td> +<td> Indexing </td> +<td> String </td> +<td> <tt>es_cluster_name</tt> </td></tr> <tr class="a"> - -<td><a href="../metron-elasticsearch/index.html#es.ip"><tt>es.ip</tt></a> </td> - -<td>Indexing </td> - -<td>String </td> - -<td><tt>es_hosts</tt> </td> - </tr> - +<td> <a href="../metron-elasticsearch/index.html#es.ip"><tt>es.ip</tt></a> </td> +<td> Indexing </td> +<td> String </td> +<td> <tt>es_hosts</tt> </td></tr> <tr class="b"> - -<td><a href="../metron-elasticsearch/index.html#es.port"><tt>es.port</tt></a> </td> - -<td>Indexing </td> - -<td>String </td> - -<td><tt>es_port</tt> </td> - </tr> - +<td> <a href="../metron-elasticsearch/index.html#es.port"><tt>es.port</tt></a> </td> +<td> Indexing </td> +<td> String </td> +<td> <tt>es_port</tt> </td></tr> <tr class="a"> - -<td><a href="../metron-elasticsearch/index.html#es.date.format"><tt>es.date.format</tt></a> </td> - -<td>Indexing </td> - -<td>String </td> - -<td><tt>es_date_format</tt> </td> - </tr> - +<td> <a href="../metron-elasticsearch/index.html#es.date.format"><tt>es.date.format</tt></a> </td> +<td> Indexing </td> +<td> String </td> +<td> <tt>es_date_format</tt> </td></tr> <tr class="b"> - -<td><a href="#validation-framework"><tt>fieldValidations</tt></a> </td> - -<td>Parsing </td> - -<td>Object </td> - -<td>N/A </td> - </tr> - +<td> <a href="#validation-framework"><tt>fieldValidations</tt></a> </td> +<td> Parsing </td> +<td> Object </td> +<td> N/A </td></tr> <tr class="a"> - -<td><a href="../metron-parsers/index.html#parser.error.topic"><tt>parser.error.topic</tt></a> </td> - -<td>Parsing </td> - -<td>String </td> - -<td>N/A </td> - </tr> - +<td> <a href="../metron-parsers/index.html#parser.error.topic"><tt>parser.error.topic</tt></a> </td> +<td> Parsing </td> +<td> String </td> +<td> N/A </td></tr> <tr class="b"> - -<td><a href="../../metron-stellar/stellar-common/index.html#stellar.function.paths"><tt>stellar.function.paths</tt></a> </td> - -<td>Stellar </td> - -<td>CSV String </td> - -<td>N/A </td> - </tr> - +<td> <a href="../../metron-stellar/stellar-common/index.html#stellar.function.paths"><tt>stellar.function.paths</tt></a> </td> +<td> Stellar </td> +<td> CSV String </td> +<td> N/A </td></tr> <tr class="a"> - -<td><a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.includes</tt></a> </td> - -<td>Stellar </td> - -<td>CSV String </td> - -<td>N/A </td> - </tr> - +<td> <a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.includes</tt></a> </td> +<td> Stellar </td> +<td> CSV String </td> +<td> N/A </td></tr> <tr class="b"> - -<td><a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.excludes</tt></a> </td> - -<td>Stellar </td> - -<td>CSV String </td> - -<td>N/A </td> - </tr> - +<td> <a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.excludes</tt></a> </td> +<td> Stellar </td> +<td> CSV String </td> +<td> N/A </td></tr> <tr class="a"> - -<td><a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration"><tt>profiler.period.duration</tt></a> </td> - -<td>Profiler </td> - -<td>Integer </td> - -<td><tt>profiler_period_duration</tt> </td> - </tr> - +<td> <a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration"><tt>profiler.period.duration</tt></a> </td> +<td> Profiler </td> +<td> Integer </td> +<td> <tt>profiler_period_duration</tt> </td></tr> <tr class="b"> - -<td><a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration.units"><tt>profiler.period.duration.units</tt></a> </td> - -<td>Profiler </td> - -<td>String </td> - -<td><tt>profiler_period_units</tt> </td> - </tr> - +<td> <a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration.units"><tt>profiler.period.duration.units</tt></a> </td> +<td> Profiler </td> +<td> String </td> +<td> <tt>profiler_period_units</tt> </td></tr> <tr class="a"> - -<td><a href="../metron-indexing/index.html#update.hbase.table"><tt>update.hbase.table</tt></a> </td> - -<td>REST/Indexing </td> - -<td>String </td> - -<td><tt>update_hbase_table</tt> </td> - </tr> - +<td> <a href="../metron-indexing/index.html#update.hbase.table"><tt>update.hbase.table</tt></a> </td> +<td> REST/Indexing </td> +<td> String </td> +<td> <tt>update_hbase_table</tt> </td></tr> <tr class="b"> - -<td><a href="../metron-indexing/index.html#update.hbase.cf"><tt>update.hbase.cf</tt></a> </td> - -<td>REST/Indexing </td> - -<td>String </td> - -<td><tt>update_hbase_cf</tt> </td> - </tr> - +<td> <a href="../metron-indexing/index.html#update.hbase.cf"><tt>update.hbase.cf</tt></a> </td> +<td> REST/Indexing </td> +<td> String </td> +<td> <tt>update_hbase_cf</tt> </td></tr> <tr class="a"> - -<td><a href="../metron-enrichment/index.html#geo.hdfs.file"><tt>geo.hdfs.file</tt></a> </td> - -<td>Enrichment </td> - -<td>String </td> - -<td><tt>geo_hdfs_file</tt> </td> - </tr> - </tbody> +<td> <a href="../metron-enrichment/index.html#geo.hdfs.file"><tt>geo.hdfs.file</tt></a> </td> +<td> Enrichment </td> +<td> String </td> +<td> <tt>geo_hdfs_file</tt> </td></tr> +<tr class="b"> +<td> <a href="../../metron-interface/metron-alerts/index.html#source.type.field"><tt>source.type.field</tt></a> </td> +<td> UI </td> +<td> String </td> +<td> N/A </td></tr> +</tbody> </table> <div class="section"> <h2><a name="Note_Configs_in_Ambari"></a>Note Configs in Ambari</h2> -<p>If a field is managed via ambari, you should change the field via ambari. Otherwise, upon service restarts, you may find your update overwritten.</p> +<p>If a field is managed via ambari, you should change the field via ambari. Otherwise, upon service restarts, you may find your update overwritten.</p> <p><a name="Validation_Framework"></a></p> <h1>Validation Framework</h1> -<p>Inside of the global configuration, there is a validation framework in place that enables the validation that messages coming from all parsers are valid. This is done in the form of validation plugins where assertions about fields or whole messages can be made. </p> -<p>The format for this is a <tt>fieldValidations</tt> field inside of global config. This is associated with an array of field validation objects structured like so:</p> - +<p>Inside of the global configuration, there is a validation framework in place that enables the validation that messages coming from all parsers are valid. This is done in the form of validation plugins where assertions about fields or whole messages can be made.</p> +<p>The format for this is a <tt>fieldValidations</tt> field inside of global config. This is associated with an array of field validation objects structured like so:</p> <ul> - -<li><tt>input</tt> : An array of input fields or a single field. If this is omitted, then the whole messages is passed to the validator.</li> - -<li><tt>config</tt> : A String to Object map for validation configuration. This is optional if the validation function requires no configuration.</li> - -<li><tt>validation</tt> : The validation function to be used. This is one of - + +<li><tt>input</tt> : An array of input fields or a single field. If this is omitted, then the whole messages is passed to the validator.</li> +<li><tt>config</tt> : A String to Object map for validation configuration. This is optional if the validation function requires no configuration.</li> +<li><tt>validation</tt> : The validation function to be used. This is one of <ul> - -<li><tt>STELLAR</tt> : Execute a Stellar Language statement. Expects the query string in the <tt>condition</tt> field of the config.</li> - -<li><tt>IP</tt> : Validates that the input fields are an IP address. By default, if no configuration is set, it assumes <tt>IPV4</tt>, but you can specify the type by passing in the config by passing in <tt>type</tt> with either <tt>IPV6</tt> or <tt>IPV4</tt> or by passing in a list [<tt>IPV4</tt>,<tt>IPV6</tt>] in which case the input(s) will be validated against both.</li> - + +<li><tt>STELLAR</tt> : Execute a Stellar Language statement. Expects the query string in the <tt>condition</tt> field of the config.</li> +<li><tt>IP</tt> : Validates that the input fields are an IP address. By default, if no configuration is set, it assumes <tt>IPV4</tt>, but you can specify the type by passing in the config by passing in <tt>type</tt> with either <tt>IPV6</tt> or <tt>IPV4</tt> or by passing in a list [<tt>IPV4</tt>,<tt>IPV6</tt>] in which case the input(s) will be validated against both.</li> <li><tt>DOMAIN</tt> : Validates that the fields are all domains.</li> - <li><tt>EMAIL</tt> : Validates that the fields are all email addresses</li> - <li><tt>URL</tt> : Validates that the fields are all URLs</li> - -<li><tt>DATE</tt> : Validates that the fields are a date. Expects <tt>format</tt> in the config.</li> - -<li><tt>INTEGER</tt> : Validates that the fields are an integer. String representation of an integer is allowed.</li> - -<li><tt>REGEX_MATCH</tt> : Validates that the fields match a regex. Expects <tt>pattern</tt> in the config.</li> - +<li><tt>DATE</tt> : Validates that the fields are a date. Expects <tt>format</tt> in the config.</li> +<li><tt>INTEGER</tt> : Validates that the fields are an integer. String representation of an integer is allowed.</li> +<li><tt>REGEX_MATCH</tt> : Validates that the fields match a regex. Expects <tt>pattern</tt> in the config.</li> <li><tt>NOT_EMPTY</tt> : Validates that the fields exist and are not empty (after trimming.)</li> - </ul></li> +</ul> +</li> </ul> <p><a name="Management_Utility"></a></p> <h1>Management Utility</h1> <p>Configurations should be stored on disk in the following structure starting at <tt>$BASE_DIR</tt>:</p> - <ul> - + <li>global.json : The global config</li> - <li><tt>sensors</tt> : The subdirectory containing sensor enrichment configuration JSON (e.g. <tt>snort.json</tt>, <tt>bro.json</tt>)</li> </ul> <p>By default, this directory as deployed by the ansible infrastructure is at <tt>$METRON_HOME/config/zookeeper</tt></p> -<p>While the configs are stored on disk, they must be loaded into Zookeeper to be used. To this end, there is a utility program to assist in this called <tt>$METRON_HOME/bin/zk_load_config.sh</tt></p> +<p>While the configs are stored on disk, they must be loaded into Zookeeper to be used. To this end, there is a utility program to assist in this called <tt>$METRON_HOME/bin/zk_load_config.sh</tt></p> <p>This has the following options:</p> -<div class="source"> -<div class="source"> -<pre> -c,--config_type <CONFIG_TYPE> The configuration type: GLOBAL, +<div> +<div> +<pre class="source"> -c,--config_type <CONFIG_TYPE> The configuration type: GLOBAL, PARSER, ENRICHMENT, INDEXING, PROFILER -f,--force Force operation @@ -592,22 +317,16 @@ -z,--zk_quorum <host:port,[host:port]*> Zookeeper Quorum URL (zk1:port,zk2:port,...) </pre></div></div> -<p>Usage examples:</p> +<p>Usage examples:</p> <ul> - + <li>To dump the existing configs from zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP</tt></li> - <li>To dump the existing GLOBAL configs from zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP -c GLOBAL</tt></li> - <li>To push the configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper</tt></li> - <li>To push only the GLOBAL configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c GLOBAL</tt></li> - <li>To push only the PARSER configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c PARSER</tt></li> - <li>To push only the PARSER ‘bro’ configs into zookeeper on the singlenode vagrant machine: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c PARSER -n bro</tt></li> - <li>To pull all configs from zookeeper to the singlenode vagrant machine disk: <tt>$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PULL -o $METRON_HOME/config/zookeeper -f</tt></li> </ul></div> <div class="section"> @@ -617,19 +336,13 @@ <p>or key/value pair:</p> <p><tt>$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pm ADD -pk foo -pv \"\"bar\"\"</tt></p> <p>The options exposed via patch file are the full range of options from RFC-6902:</p> - <ul> - + <li>ADD</li> - <li>REMOVE</li> - <li>REPLACE</li> - <li>MOVE</li> - <li>COPY</li> - <li>TEST</li> </ul> <p>whereas with key/value patching, we only current expose ADD and REMOVE. Note that ADD will function as a REPLACE when the key already exists.</p> @@ -637,19 +350,20 @@ <h3><a name="Patch_File"></a>Patch File</h3> <p>Let’s say we want to add a complex JSON object to our configuration with a patch file. e.g.</p> -<div class="source"> -<div class="source"> -<pre>"foo" : { +<div> +<div> +<pre class="source">"foo" : { "bar" : { "baz" : [ "bazval1", "bazval2" ] } } </pre></div></div> + <p>We would write a patch file “/tmp/mypatch.txt” with contents:</p> -<div class="source"> -<div class="source"> -<pre>[ +<div> +<div> +<pre class="source">[ { "op": "add", "path": "/foo", @@ -657,37 +371,41 @@ } ] </pre></div></div> + <p>And submit via zk_load_configs as follows:</p> -<div class="source"> -<div class="source"> -<pre> $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt -</pre></div></div></div> +<div> +<div> +<pre class="source"> $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt +</pre></div></div> +</div> <div class="section"> -<h3><a name="Patch_KeyValue"></a>Patch Key/Value</h3> +<h3><a name="Patch_Key.2FValue"></a>Patch Key/Value</h3> <p>Now let’s try the same without using a patch file, instead using the patch_key and patch_value options right from the command line utility. This would like like the following.</p> -<div class="source"> -<div class="source"> -<pre>$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pm ADD -pk "/foo" -pv "{ \"bar\" : { \"baz\" : [ \"bazval1\", \"bazval2\" ] } }" -</pre></div></div></div> +<div> +<div> +<pre class="source">$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pm ADD -pk "/foo" -pv "{ \"bar\" : { \"baz\" : [ \"bazval1\", \"bazval2\" ] } }" +</pre></div></div> +</div> <div class="section"> <h3><a name="Applying_Multiple_Patches"></a>Applying Multiple Patches</h3> <p>Applying multiple patches is also pretty straightforward. You can achieve this in a single command using patch files, or simply execute multiple commands in sequence using the patch_key/value approach.</p> <p>Let’s say we wanted to add the following to our global config:</p> -<div class="source"> -<div class="source"> -<pre>"apache" : "metron", +<div> +<div> +<pre class="source">"apache" : "metron", "is" : "the best", "streaming" : "analytics platform" </pre></div></div> + <p>and remove the /foo key from the previous example.</p> <p>Create a patch file /tmp/mypatch.txt with four separate patch operations.</p> -<div class="source"> -<div class="source"> -<pre>[ +<div> +<div> +<pre class="source">[ { "op": "remove", "path": "/foo" @@ -709,27 +427,30 @@ } ] </pre></div></div> + <p>Now submit again and you should see a Global config with the “foo” key removed and three new keys added.</p> -<div class="source"> -<div class="source"> -<pre> $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt -</pre></div></div></div> +<div> +<div> +<pre class="source"> $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt +</pre></div></div> +</div> <div class="section"> <h3><a name="Notes_On_Patching"></a>Notes On Patching</h3> <p>For any given patch key, the last/leaf node in the key’s parent <i>must</i> exist, otherwise an exception will be thrown. For example, if you want to add the following:</p> -<div class="source"> -<div class="source"> -<pre>"foo": { +<div> +<div> +<pre class="source">"foo": { "bar": "baz" } </pre></div></div> + <p>It is not sufficient to use /foo/bar as a key if foo does not already exist. You would either need to incrementally build the JSON and make this a two step process</p> -<div class="source"> -<div class="source"> -<pre>[ +<div> +<div> +<pre class="source">[ { "op": "add", "path": "/foo", @@ -742,11 +463,12 @@ } ] </pre></div></div> + <p>Or provide the value as a complete JSON object.</p> -<div class="source"> -<div class="source"> -<pre>[ +<div> +<div> +<pre class="source">[ { "op": "add", "path": "/foo", @@ -754,14 +476,15 @@ } ] </pre></div></div> + <p>The REMOVE operation is idempotent. Running the remove command on the same key multiple times will not fail once the key has been removed.</p> <p><a name="Topology_Errors"></a></p> <h1>Topology Errors</h1> <p>Errors generated in Metron topologies are transformed into JSON format and follow this structure:</p> -<div class="source"> -<div class="source"> -<pre>{ +<div> +<div> +<pre class="source">{ "exception": "java.lang.IllegalStateException: Unable to parse Message: ...", "failed_sensor_type": "bro", "stack": "java.lang.IllegalStateException: Unable to parse Message: ...", @@ -774,11 +497,12 @@ "timestamp": 1488809630698 } </pre></div></div> -<p>Each topology can be configured to send error messages to a specific Kafka topic. The parser topologies retrieve this setting from the the <tt>parser.error.topic</tt> setting in the global config:</p> -<div class="source"> -<div class="source"> -<pre>{ +<p>Each topology can be configured to send error messages to a specific Kafka topic. The parser topologies retrieve this setting from the the <tt>parser.error.topic</tt> setting in the global config:</p> + +<div> +<div> +<pre class="source">{ "es.clustername": "metron", "es.ip": "node1", "es.port": "9300", @@ -786,16 +510,17 @@ "parser.error.topic": "indexing" } </pre></div></div> -<p>Error topics for enrichment and threat intel errors are passed into the enrichment topology as flux properties named <tt>enrichment.error.topic</tt> and <tt>threat.intel.error.topic</tt>. These properties can be found in <tt>$METRON_HOME/config/enrichment.properties</tt>.</p> -<p>The error topic for indexing errors is passed into the indexing topology as a flux property named <tt>index.error.topic</tt>. This property can be found in either <tt>$METRON_HOME/config/elasticsearch.properties</tt> or <tt>$METRON_HOME/config/solr.properties</tt> depending on the search engine selected.</p> -<p>By default all error messages are sent to the <tt>indexing</tt> topic so that they are indexed and archived, just like other messages. The indexing config for error messages can be found at <tt>$METRON_HOME/config/zookeeper/indexing/error.json</tt>.</p> + +<p>Error topics for enrichment and threat intel errors are passed into the enrichment topology as flux properties named <tt>enrichment.error.topic</tt> and <tt>threat.intel.error.topic</tt>. These properties can be found in <tt>$METRON_HOME/config/enrichment.properties</tt>.</p> +<p>The error topic for indexing errors is passed into the indexing topology as a flux property named <tt>index.error.topic</tt>. This property can be found in either <tt>$METRON_HOME/config/elasticsearch.properties</tt> or <tt>$METRON_HOME/config/solr.properties</tt> depending on the search engine selected.</p> +<p>By default all error messages are sent to the <tt>indexing</tt> topic so that they are indexed and archived, just like other messages. The indexing config for error messages can be found at <tt>$METRON_HOME/config/zookeeper/indexing/error.json</tt>.</p> <p><a name="Performance_Logging"></a></p> <h1>Performance Logging</h1> <p>The PerformanceLogger class provides functionality that enables developers to debug performance issues. Basic usage looks like the following:</p> -<div class="source"> -<div class="source"> -<pre>// create a simple inner performance class to use for logger instantiation +<div> +<div> +<pre class="source">// create a simple inner performance class to use for logger instantiation public static class Perf {} // instantiation PerformnanceLogger perfLog = new PerformanceLogger(() -> getConfigurations().getGlobalConfig(), Perf.class.getName()); @@ -807,45 +532,37 @@ perfLog.log("mark1", "My high performance stuff is very performan // log no additional message, just the basics perfLog.log("mark1"); </pre></div></div> + <p>The logger maintains a Map<String, Long> of named markers that correspond to start times. Calling mark() performs a put on the underlying timing store. Output includes the mark name, elapsed time in nanoseconds, as well as any custom messaging you provide. A sample log would look like the following:</p> -<div class="source"> -<div class="source"> -<pre>[DEBUG] markName=execute,time(ns)=121411,message=key=7a8dbe44-4cb9-4db2-9d04-7632f543b56c, elapsed time to run execute +<div> +<div> +<pre class="source">[DEBUG] markName=execute,time(ns)=121411,message=key=7a8dbe44-4cb9-4db2-9d04-7632f543b56c, elapsed time to run execute </pre></div></div> + <p><b>Configuration</b></p> <p>The first argument to the logger is a java.util.function.Supplier<Map<String, Object>>. The offers flexibility in being able to provide multiple configuration “suppliers” depending on your individual usage requirements. The example above, taken from org.apache.metron.enrichment.bolt.GenericEnrichmentBolt, leverages the global config to dymanically provide configuration from Zookeeper. Any updates to the global config via Zookeeper are reflected live at runtime. Currently, the PerformanceLogger supports the following options:</p> - <table border="0" class="table table-striped"> - <thead> - +<thead> + <tr class="a"> - -<th>Property Name </th> - -<th>Type </th> - -<th>Valid Values </th> - </tr> - </thead> - <tbody> - +<th>Property Name </th> +<th>Type </th> +<th>Valid Values </th></tr> +</thead><tbody> + <tr class="b"> - -<td>performance.logging.percent.records </td> - -<td>Integer </td> - -<td>0-100 </td> - </tr> - </tbody> +<td>performance.logging.percent.records </td> +<td>Integer </td> +<td>0-100 </td></tr> +</tbody> </table> <p><b>Other Usage Details</b></p> <p>You can also provide your own format String and provide arguments that will be used when formatting that String. This code avoids expensive String concatenation by only formatting when debugging is enabled. For more complex arguments, e.g. JSON serialization, we expose an isDebugEnabled() method.</p> -<div class="source"> -<div class="source"> -<pre>// log with format String and single argument +<div> +<div> +<pre class="source">// log with format String and single argument perfLog.log("join-message", "key={}, elapsed time to join messages", key); // check if debugging is enabled for the performance logger to avoid more expensive operations @@ -853,25 +570,51 @@ if (perfLog.isDebugEnabled()) { perfLog.log("join-message", "key={}, elapsed time to join messages, message={}", key, rawMessage.toJSONString()); } </pre></div></div> -<p><b>Side Effects</b></p> -<p>Calling the mark() method multiple times simply resets the start time to the current nano time. Calling log() with a non-existent mark name will log 0 ns elapsed time with a warning indicating that log has been invoked for a mark name that does not exist. The class is not thread-safe and makes no attempt at keeping multiple threads from modifying the same markers.</p></div></div> - </div> - </div> - </div> +<p><b>Side Effects</b></p> +<p>Calling the mark() method multiple times simply resets the start time to the current nano time. Calling log() with a non-existent mark name will log 0 ns elapsed time with a warning indicating that log has been invoked for a mark name that does not exist. The class is not thread-safe and makes no attempt at keeping multiple threads from modifying the same markers.</p> +<p><a name="Metron_Debugging"></a></p> +<h1>Metron Debugging</h1> +<p>A Python script is provided for gathering information useful in debugging your Metron cluster. Run from the node that has Metron installed on it. All options listed below are required.</p> +<p><i>Note:</i> Be aware that no anonymization/scrubbing is performed on the captured configuration details.</p> + +<div> +<div> +<pre class="source"># $METRON_HOME/bin/cluster_info.py -h +Usage: cluster_info.py [options] + +Options: + -h, --help show this help message and exit + -a HOST:PORT, --ambari-host=HOST:PORT + Connect to Ambari via the supplied host:port + -c NAME, --cluster-name=NAME + Name of cluster in Ambari to retrieve info for + -o DIRECTORY, --out-dir=DIRECTORY + Write debugging data to specified root directory + -s HOST:PORT, --storm-host=HOST:PORT + Connect to Storm via the supplied host:port + -b HOST1:PORT,HOST2:PORT, --broker_list=HOST1:PORT,HOST2:PORT + Connect to Kafka via the supplied comma-delimited + host:port list + -z HOST1:PORT,HOST2:PORT, --zookeeper_quorum=HOST1:PORT,HOST2:PORT + Connect to Zookeeper via the supplied comma-delimited + host:port quorum list + -m DIRECTORY, --metron_home=DIRECTORY + Metron home directory + -p DIRECTORY, --hdp_home=DIRECTORY + HDP home directory +</pre></div></div></div></div> + </div> + </div> + </div> <hr/> - <footer> - <div class="container-fluid"> - <div class="row span12">Copyright © 2018 - <a href="https://www.apache.org";>The Apache Software Foundation</a>. - All Rights Reserved. - + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> </div> - - - - </div> </footer> </body> </html>
