http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-enrichment/Performance.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-enrichment/Performance.html b/site/current-book/metron-platform/metron-enrichment/Performance.html index 136d939..0857ee4 100644 --- a/site/current-book/metron-platform/metron-enrichment/Performance.html +++ b/site/current-book/metron-platform/metron-enrichment/Performance.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/Performance.md at 2018-06-07 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/Performance.md at 2018-09-12 | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Enrichment Performance</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> @@ -32,8 +32,8 @@ <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> <li class="active ">Enrichment Performance</li> - <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> - <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> </ul> </div> <div class="row-fluid"> @@ -55,7 +55,6 @@ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> - <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> @@ -65,9 +64,11 @@ </ul> </li> <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li> <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> </ul> </li>
http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-enrichment/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-enrichment/index.html b/site/current-book/metron-platform/metron-enrichment/index.html index de1e4fa..e750946 100644 --- a/site/current-book/metron-platform/metron-enrichment/index.html +++ b/site/current-book/metron-platform/metron-enrichment/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/index.md at 2018-06-07 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/index.md at 2018-09-12 | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Enrichment</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> @@ -32,8 +32,8 @@ <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> <li class="active ">Enrichment</li> - <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> - <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> </ul> </div> <div class="row-fluid"> @@ -55,7 +55,6 @@ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> - <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> @@ -65,9 +64,11 @@ </ul> </li> <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li> <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> </ul> </li> @@ -162,14 +163,22 @@ limitations under the License. <p>There are two types of configurations at the moment, <tt>global</tt> and <tt>sensor</tt> specific.</p></div> <div class="section"> <h2><a name="Global_Configuration"></a>Global Configuration</h2> -<p>There are a few enrichments which have independent configurations, such as from the global config.</p> +<p>There are a few enrichments which have independent configurations, such as from the global config. You can also configure the enrichment topology’s writer batching settings.</p> <p>Also, see the “<a href="../metron-common/index.html">Global Configuration</a>” section for more discussion of the global config.</p> <div class="section"> <h3><a name="GeoIP"></a>GeoIP</h3> <p>Metron supports enrichment of IP information using <a class="externalLink" href="https://dev.maxmind.com/geoip/geoip2/geolite2/";>GeoLite2</a>. The location of the file is managed in the global config.</p> <div class="section"> <h4><a name="geo.hdfs.file"></a><tt>geo.hdfs.file</tt></h4> -<p>The location on HDFS of the GeoLite2 database file to use for GeoIP lookups. This file will be localized on the storm supervisors running the topology and used from there. This is lazy, so if this property changes in a running topology, the file will be localized from HDFS upon first time the file is used via the geo enrichment.</p></div></div></div> +<p>The location on HDFS of the GeoLite2 database file to use for GeoIP lookups. This file will be localized on the storm supervisors running the topology and used from there. This is lazy, so if this property changes in a running topology, the file will be localized from HDFS upon first time the file is used via the geo enrichment.</p></div></div> +<div class="section"> +<h3><a name="Writer_Batching"></a>Writer Batching</h3> +<div class="section"> +<h4><a name="enrichment.writer.batchSize"></a><tt>enrichment.writer.batchSize</tt></h4> +<p>The size of the batch that is written to Kafka at once. Defaults to <tt>15</tt> (size of 1 disables batching).</p></div> +<div class="section"> +<h4><a name="enrichment.writer.batchTimeout"></a><tt>enrichment.writer.batchTimeout</tt></h4> +<p>The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to <tt>0</tt>, it defaults to a system-determined duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>. Ignored if batchSize is <tt>1</tt>, since this disables batching.</p></div></div></div> <div class="section"> <h2><a name="Sensor_Enrichment_Configuration"></a>Sensor Enrichment Configuration</h2> <p>The sensor specific configuration is intended to configure the individual enrichments and threat intelligence enrichments for a given sensor type (e.g. <tt>snort</tt>).</p> http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-indexing/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-indexing/index.html b/site/current-book/metron-platform/metron-indexing/index.html index 6d4b8d6..65c0bad 100644 --- a/site/current-book/metron-platform/metron-indexing/index.html +++ b/site/current-book/metron-platform/metron-indexing/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-indexing/index.md at 2018-06-07 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-indexing/index.md at 2018-09-12 | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Indexing</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> @@ -32,8 +32,8 @@ <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> <li class="active ">Indexing</li> - <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> - <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> </ul> </div> <div class="row-fluid"> @@ -55,15 +55,16 @@ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> - <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> <li class="active"><a href="#"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li> <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> </ul> </li> @@ -139,15 +140,38 @@ limitations under the License. <li><tt>hdfs</tt></li> <li><tt>solr</tt></li> </ul> -<p>Depending on how you start the indexing topology, it will have either elasticsearch or solr and hdfs writers running.</p> -<p>The configuration for an individual writer-specific configuration is a JSON map with the following fields:</p> -<ul> +<p>Depending on how you start the indexing topology, it will have either Elasticsearch or Solr and HDFS writers running.</p> +<table border="0" class="table table-striped"> +<thead> -<li><tt>index</tt> : The name of the index to write to (defaulted to the name of the sensor).</li> -<li><tt>batchSize</tt> : The size of the batch that is written to the indices at once. Defaults to <tt>1</tt> (no batching).</li> -<li><tt>batchTimeout</tt> : The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to <tt>0</tt>, it defaults to a system-determined duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>. Ignored if batchSize is <tt>1</tt>, since this disables batching.</li> -<li><tt>enabled</tt> : Whether the writer is enabled (default <tt>true</tt>).</li> -</ul> +<tr class="a"> +<th> Property </th> +<th> Description </th> +<th> Default Value </th></tr> +</thead><tbody> + +<tr class="b"> +<td> <tt>index</tt> </td> +<td> The name of the index to write to. </td> +<td> Defaults to the name of the sensor. </td></tr> +<tr class="a"> +<td> <tt>batchSize</tt> </td> +<td> The size of the batch that is written to the indices at once. </td> +<td> Defaults to <tt>1</tt>; no batching. </td></tr> +<tr class="b"> +<td> <tt>batchTimeout</tt> </td> +<td> The timeout after which a batch will be flushed even if <tt>batchSize</tt> has not been met. </td> +<td> Defaults to a duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>, if left undefined or set to 0. Ignored if batchSize is <tt>1</tt>, since this disables batching.</td></tr> +<tr class="a"> +<td> <tt>enabled</tt> </td> +<td> A boolean indicating whether the writer is enabled. </td> +<td> Defaults to <tt>true</tt> </td></tr> +<tr class="b"> +<td> <tt>fieldNameConverter</tt> </td> +<td> Defines how field names are transformed before being written to the index. Only applicable to <tt>elasticsearch</tt>. </td> +<td> Defaults to <tt>DEDOT</tt>. Acceptable values are <tt>DEDOT</tt> that replaces all ‘.’ with ‘:’ or <tt>NOOP</tt> that does not change the field names . </td></tr> +</tbody> +</table> <div class="section"> <h3><a name="Meta_Alerts"></a>Meta Alerts</h3> <p>Alerts can be grouped, after appropriate searching, into a set of alerts called a meta alert. A meta alert is useful for maintaining the context of searching and grouping during further investigations. Standard searches can return meta alerts, but grouping and other aggregation or sorting requests will not, because there’s not a clear way to aggregate in many cases if there are multiple alerts contained in the meta alert. All meta alerts will have the source type of metaalert, regardless of the contained alert’s origins.</p></div> @@ -155,6 +179,19 @@ limitations under the License. <h3><a name="Elasticsearch"></a>Elasticsearch</h3> <p>Metron comes with built-in templates for the default sensors for Elasticsearch. When adding a new sensor, it will be necessary to add a new template defining the output fields appropriately. In addition, there is a requirement for a field <tt>alert</tt> of type <tt>nested</tt> for Elasticsearch 2.x installs. This is detailed at <a href="../metron-elasticsearch/index.html#Using_Metron_with_Elasticsearch_2.x">Using Metron with Elasticsearch 2.x</a></p></div> <div class="section"> +<h3><a name="Solr"></a>Solr</h3> +<p>Metron comes with built-in schemas for the default sensors for Solr. When adding a new sensor, it will be necessary to add a new schema defining the output fields appropriately. In addition, these fields are used internally by Metron and also required:</p> +<ul> + +<li><tt><field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /></tt></li> +<li><tt><field name="source.type" type="string" indexed="true" stored="true" /></tt></li> +<li><tt><field name="timestamp" type="timestamp" indexed="true" stored="true" /></tt></li> +<li><tt><field name="comments" type="string" indexed="true" stored="true" multiValued="true"/></tt></li> +<li><tt><field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/></tt></li> +</ul> +<p>The unique key should be set to <tt>guid</tt> by including <tt><uniqueKey>guid</uniqueKey></tt> in the schema.</p> +<p>It is strongly suggested the <tt>fieldTypes</tt> match those in the built-in schemas.</p></div> +<div class="section"> <h3><a name="Indexing_Configuration_Examples"></a>Indexing Configuration Examples</h3> <p>For a given sensor, the following scenarios would be indicated by the following cases:</p> <div class="section"> @@ -294,7 +331,7 @@ limitations under the License. <p>The HBase column family to use for message updates.</p></div></div> <div class="section"> <h3><a name="The_MetaAlertDao"></a>The <tt>MetaAlertDao</tt></h3> -<p>The goal of meta alerts is to be able to group together a set of alerts while being able to transparently perform actions like searches, as if meta alerts were normal alerts. <tt>org.apache.metron.indexing.dao.MetaAlertDao</tt> extends <tt>IndexDao</tt> and enables several features:</p> +<p>The goal of meta alerts is to be able to group together a set of alerts while being able to transparently perform actions like searches, as if meta alerts were normal alerts. <tt>org.apache.metron.indexing.dao.metaalert.MetaAlertDao</tt> extends <tt>IndexDao</tt> and enables several features:</p> <ul> <li>the ability to get all meta alerts associated with an alert</li> http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-job/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-job/index.html b/site/current-book/metron-platform/metron-job/index.html new file mode 100644 index 0000000..388ea80 --- /dev/null +++ b/site/current-book/metron-platform/metron-job/index.html @@ -0,0 +1,126 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-job/index.md at 2018-09-12 + | Rendered using Apache Maven Fluido Skin 1.7 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Metron Job</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active ">Metron Job</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> + </ul> + </div> + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> + <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> + <ul class="nav nav-list"> + <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> + <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> + <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> + <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> + <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> + <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li class="active"><a href="#"><span class="none"></span>Job</a></li> + <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> + <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li> + <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> + <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> + </ul> +</li> + <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> + </div> + </div> + </div> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<h1>Metron Job</h1> +<p><a name="Metron_Job"></a></p> +<p>This module holds abstractions for creating jobs. The main actors are a JobManager interface and subsequent implementation, InMemoryJobManger, that handles maintaining a cache of running and completed Statusable jobs. Each Statusable can provide a Finalizer implementation that should be executed on completion of the underlying job. Successful jobs should return a Pageable object that allow consumers to request results on a per-page basis.</p> +<div class="section"> +<h2><a name="Job_State_Statechart"></a>Job State Statechart</h2> +<p><img src="../../images/metron-job_state_statechart_diagram.svg" alt="Job State Statechart" /></p></div> + </div> + </div> + </div> + <hr/> + <footer> + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-management/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-management/index.html b/site/current-book/metron-platform/metron-management/index.html index f9ea1ce..0aa9eda 100644 --- a/site/current-book/metron-platform/metron-management/index.html +++ b/site/current-book/metron-platform/metron-management/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-management/index.md at 2018-06-07 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-management/index.md at 2018-09-12 | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Stellar REPL Management Utilities</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> @@ -32,8 +32,8 @@ <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> <li class="active ">Stellar REPL Management Utilities</li> - <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> - <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> </ul> </div> <div class="row-fluid"> @@ -55,15 +55,16 @@ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> - <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> <li class="active"><a href="#"><span class="none"></span>Management</a></li> <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li> <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> </ul> </li> http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html b/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html index 988580b..8d75d41 100644 --- a/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html +++ b/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/3rdPartyParser.md at 2018-06-07 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/3rdPartyParser.md at 2018-09-12 | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Custom Metron Parsers</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> @@ -32,8 +32,8 @@ <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> <li class="active ">Custom Metron Parsers</li> - <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> - <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> </ul> </div> <div class="row-fluid"> @@ -55,20 +55,22 @@ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> - <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-down"></span>Parsers</a> <ul class="nav nav-list"> <li class="active"><a href="#"><span class="none"></span>3rdPartyParser</a></li> + <li><a href="../../metron-platform/metron-parsers/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li> <li><a href="../../metron-platform/metron-parsers/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li> </ul> </li> <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> </ul> </li> @@ -145,18 +147,18 @@ limitations under the License. <p>For this demonstration, let’s create a maven project to compile our project. We’ll call it <tt>extra_parsers</tt>, so in your workspace, let’s set up the maven project:</p> <ul> -<li>Create the maven infrastructure for <tt>extra_parsers</tt> via</li> -</ul> +<li> + +<p>Create the maven infrastructure for <tt>extra_parsers</tt> via</p> <div> <div> <pre class="source">mkdir -p extra_parsers/src/{main,test}/java </pre></div></div> +</li> +<li> -<ul> - -<li>Create a pom file indicating how we should build our parsers by editing <tt>extra_parsers/pom.xml</tt> with the following content:</li> -</ul> +<p>Create a pom file indicating how we should build our parsers by editing <tt>extra_parsers/pom.xml</tt> with the following content:</p> <div> <div> @@ -207,7 +209,7 @@ limitations under the License. <!-- We will set up the shade plugin to create a single jar at the end of the build lifecycle. We will exclude some things and relocate others to simulate a real situation. - + One thing to note is that it's a good practice to shade and relocate common libraries that may be dependencies in Metron. Your jar will be merged with the parsers jar, so the metron @@ -289,11 +291,10 @@ limitations under the License. </build> </project> </pre></div></div> +</li> +<li> -<ul> - -<li>Now let’s create our parser <tt>com.thirdparty.SimpleParser</tt> by creating the file <tt>extra-parsers/src/main/java/com/thirdparty/SimpleParser.java</tt> with the following content:</li> -</ul> +<p>Now let’s create our parser <tt>com.thirdparty.SimpleParser</tt> by creating the file <tt>extra-parsers/src/main/java/com/thirdparty/SimpleParser.java</tt> with the following content:</p> <div> <div> @@ -332,12 +333,13 @@ public class SimpleParser extends BasicParser { } } </pre></div></div> - -<ul> - +</li> <li>Compile the parser via <tt>mvn clean package</tt> in <tt>extra_parsers</tt></li> -</ul> +<li> + <p>This will create a jar containing your parser and its dependencies (sans Metron dependencies) in <tt>extra-parsers/target/extra-parsers-1.0-SNAPSHOT-uber.jar</tt></p> +</li> +</ul> <p><a name="Deploying_Your_Custom_Parser"></a></p> <h1>Deploying Your Custom Parser</h1> <p>In order to deploy your newly built custom parser, you would place the jar file above in the <tt>$METRON_HOME/parser_contrib</tt> directory on the Metron host (i.e. any host you would start parsers from or, alternatively, where the Metron REST is hosted).</p></div> @@ -362,11 +364,15 @@ public class SimpleParser extends BasicParser { <h3><a name="Restart_the_REST_service_in_Ambari"></a>Restart the REST service in Ambari</h3> <p>In order for new parsers to be picked up, the REST service must be restarted. You can do that from within Ambari by restarting the <tt>Metron REST</tt> service.</p></div> <div class="section"> -<h3><a name="Push_the_Zookeeper_Configs"></a>Push the Zookeeper Configs</h3> -<p>Now push the config to Zookeeper with the following command: <tt>$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER</tt></p></div> -<div class="section"> <h3><a name="Create_a_Kafka_Topic"></a>Create a Kafka Topic</h3> -<p>Create a kafka topic, let’s call it <tt>test</tt> via: <tt>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic test --partitions 1 --replication-factor 1</tt></p> +<p>Create a kafka topic, let’s call it <tt>test</tt>.</p> + +<div> +<div> +<pre class="source">KAFKA_HOME=/usr/hdp/current/kafka-broker +$KAFKA_HOME/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic test --partitions 1 --replication-factor 1 +</pre></div></div> + <p>Note, in a real deployment, that topic would be named something more descriptive and would have replication factor and partitions set to something less trivial.</p></div> <div class="section"> <h3><a name="Configure_Test_Parser"></a>Configure Test Parser</h3> @@ -381,32 +387,40 @@ public class SimpleParser extends BasicParser { </pre></div></div> </div> <div class="section"> +<h3><a name="Push_the_Zookeeper_Configs"></a>Push the Zookeeper Configs</h3> +<p>Now push the config to Zookeeper with the following command.</p> + +<div> +<div> +<pre class="source">$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER +</pre></div></div> +</div> +<div class="section"> <h3><a name="Start_Parser"></a>Start Parser</h3> <p>Now we can start the parser and send some data through:</p> <ul> -<li>Start the parser</li> -</ul> +<li> + +<p>Start the parser</p> <div> <div> <pre class="source">$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s test </pre></div></div> +</li> +<li> -<ul> - -<li>Send example data through:</li> -</ul> +<p>Send example data through:</p> <div> <div> <pre class="source">echo "apache,metron" | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic test </pre></div></div> +</li> +<li> -<ul> - -<li>Validate data was written in ES:</li> -</ul> +<p>Validate data was written in ES:</p> <div> <div> @@ -416,6 +430,8 @@ public class SimpleParser extends BasicParser { } ' </pre></div></div> +</li> +<li> <p>This should yield something like:</p> @@ -447,7 +463,8 @@ public class SimpleParser extends BasicParser { } } </pre></div></div> -</div> +</li> +</ul></div> <div class="section"> <h3><a name="Via_the_Management_UI"></a>Via the Management UI</h3> <p>As long as the REST service is restarted after new parsers are added to <tt>$METRON_HOME/parser_contrib</tt>, they are available in the UI to creating and deploying parsers.</p></div></div> http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-parsers/ParserChaining.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-parsers/ParserChaining.html b/site/current-book/metron-platform/metron-parsers/ParserChaining.html new file mode 100644 index 0000000..a4c9360 --- /dev/null +++ b/site/current-book/metron-platform/metron-parsers/ParserChaining.html @@ -0,0 +1,259 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/ParserChaining.md at 2018-09-12 + | Rendered using Apache Maven Fluido Skin 1.7 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Metron – Parser Chaining</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> +<script type="text/javascript"> + $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); + </script> + </head> + <body class="topBarDisabled"> + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"><a href="http://metron.apache.org/"; id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> + <div class="pull-right"></div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> + <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> + <li class="active ">Parser Chaining</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> + </ul> + </div> + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + <ul class="nav nav-list"> + <li class="nav-header">User Documentation</li> + <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> + <ul class="nav nav-list"> + <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> + <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> + <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> + <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> + <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> + <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> + <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li> + <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li> + <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li> + <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> + <ul class="nav nav-list"> + <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> + <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> + <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> + <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> + <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> + <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> + <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> + <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-down"></span>Parsers</a> + <ul class="nav nav-list"> + <li><a href="../../metron-platform/metron-parsers/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li> + <li class="active"><a href="#"><span class="none"></span>ParserChaining</a></li> + <li><a href="../../metron-platform/metron-parsers/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li> + </ul> +</li> + <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> + <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> + </ul> +</li> + <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> + <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> + <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> + <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> + <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> + </ul> +</li> +</ul> + <hr /> + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> +<a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> + </div> + </div> + </div> + <div id="bodyColumn" class="span10" > +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<h1>Parser Chaining</h1> +<p><a name="Parser_Chaining"></a></p> +<p>Aggregating many different types sensors into a single data source (e.g. syslog) and ingesting that aggregate sensor into Metron is a common pattern. It is not obvious precisely how to manage these types of aggregate sensors as they require two-pass parsing. This document will walk through an example of supporting this kind of multi-pass ingest.</p> +<p>Multi-pass parser involves the following requirements:</p> +<ul> + +<li>The enveloping parser (e.g. the aggregation format such as syslog or plain CSV) may contain metadata which should be ingested along with the data.</li> +<li>The enveloping sensor contains many different sensor types</li> +</ul> +<p><a name="High_Level_Solution"></a></p> +<h1>High Level Solution</h1> +<p><img src="../../use-cases/parser_chaining/message_routing_high_level.svg" alt="High Level Approach" /></p> +<p>At a high level, we continue to maintain the architectural invariant of a 1-1 relationship between logical sensors and storm topologies. Eventually this relationship may become more complex, but at the moment the approach is to construct a routing parser which will have two responsibilities:</p> +<ul> + +<li>Parse the envelope (e.g. syslog data) and extract any metadata fields from the envelope to pass along</li> +<li>Route the unfolded data to the appropriate kafka topic associated with the enveloped sensor data</li> +</ul> +<p>Because the data emitted from the routing parser is just like any data emitted from any other parser, in that it is a JSON blob like any data emitted from any parser, we will need to adjust the downstream parsers to extract the enveloped data from the JSON blob and treat it as the data to parse.</p> +<p><a name="Architecting_a_Parser_Chaining_Solution_in_Metron"></a></p> +<h1>Architecting a Parser Chaining Solution in Metron</h1> +<p>Currently the approach to fulfill this requirement involves a couple knobs in the Parser infrastructure for Metron.</p> +<p>Consider the case, for instance, where we have many different TYPES of messages wrapped inside of syslog. As an architectural abstraction, we would want to have the following properties:</p> +<ul> + +<li>separate the concerns of parsing the individual types of messages from each other</li> +<li>separate the concerns of parsing the individual types of messages from parsing the envelope</li> +</ul> +<div class="section"> +<h2><a name="Data_Dependent_Parser_Writing"></a>Data Dependent Parser Writing</h2> +<p>Parsers allow users to configure the topic which the kafka producer uses in a couple of ways (from the parser config in an individual parser):</p> +<ul> + +<li><tt>kafka.topic</tt> - Specify the topic in the config. This can be updated by updating the config, but it is data independent (e.g. not dependent on the data in a message).</li> +<li><tt>kafka.topicField</tt> - Specify the topic as the value of a particular field. If unpopulated, then the message is dropped. This is inherrently data dependent.</li> +</ul> +<p>The <tt>kafka.topicField</tt> parameter allows for data dependent topic selection and this inherrently enables the routing capabilities necessary for handling enveloped data.</p></div> +<div class="section"> +<h2><a name="Flexibly_Interpreting_Data"></a>Flexibly Interpreting Data</h2> +<div class="section"> +<h3><a name="Aside:_The_Role_of_Metadata_in_Metron"></a>Aside: The Role of Metadata in Metron</h3> +<p>Before we continue, let’s briefly talk about metadata. We have exposed the ability to pass along metadata and interact with metadata in a decoupled way from the actual parser logic (i.e. the GrokParser should not have to consider how to interpret metadata).</p> +<p>There are three choices about manipulating metadata in Metron:</p> +<ul> + +<li>Should you merge metadata into the downstream message?</li> +<li>If you do, should you use a key prefix to set it off from the message by default?</li> +</ul> +<p>This enables users to specify metadata independent of the data that is persisted downstream and can inform the operations of enrichment and the profiler.</p></div> +<div class="section"> +<h3><a name="Interpretation"></a>Interpretation</h3> +<p>Now that we have an approach which enables the routing of the data, the remaining question is how to decouple <i>parsing</i> data from <i>interpreting</i> data and metadata. By default, Metron operates like so:</p> +<ul> + +<li>The kafka record key (as a JSON Map) is considered metadata</li> +<li>The kafka record value is considered data</li> +</ul> +<p>Beyond that, we presume defaults for this default strategy around handling metadata. In particular, by default we do not merge metadata and use a <tt>metron.metadata</tt> prefix for all metadata.</p> +<p>In order to enable chained parser WITH metadata, we allow the following to be specified via strategy in the parser config:</p> +<ul> + +<li>How to extract the data from the kafka record</li> +<li>How to extract the metadata from the kafka record</li> +<li>The default operations for merging</li> +<li>The prefix for the metadata key</li> +</ul> +<p>The available strategies, specified by the <tt>rawMessageStrategy</tt> configuration is either<tt>ENVELOPE</tt> or <tt>DEFAULT</tt>.</p> +<p>Specifically, to enable parsing enveloped data (i.e. data in a field of a JSON blob with the other fields being metadata), one can specify the strategy and configuration of that strategy in the parser config. One must specify the <tt>rawMessageStrategy</tt> as <tt>ENVELOPE</tt> in the parser and the <tt>rawMessageStrategyConfig</tt> to indicate the field which contains the data.</p> +<p>Together with routing, we have the complete solution to chain parsers which can:</p> +<ul> + +<li>parse the envelope</li> +<li>route the parsed data to specific parsers</li> +<li>have the specific parsers interpret the data via the <tt>rawMessageStrategy</tt> whereby they pull the data out from JSON Map that they receive</li> +</ul> +<p>Together this enables a directed acyclic graph of parsers to handle single or multi-layer parsing.</p></div> +<div class="section"> +<h3><a name="Example"></a>Example</h3> +<p>For a complete example, look at the <a href="../../use-cases/parser_chaining/index.html">parser chaining use-case</a>, however for a simple example the following should suffice.</p> +<p>If I want to configure a CSV parser to parse data which has 3 columns <tt>f1</tt>, <tt>f2</tt> and <tt>f3</tt> and is held in a field called <tt>payload</tt> inside of a JSON Map, I can do so like this:</p> + +<div> +<div> +<pre class="source">{ + "parserClassName" : "org.apache.metron.parsers.csv.CSVParser" + ,"sensorTopic" : "my_topic" + ,"rawMessageStrategy" : "ENVELOPE" + ,"rawMessageStrategyConfig" : { + "messageField" : "payload", + "metadataPrefix" : "" + } + , "parserConfig": { + "columns" : { "f1": 0, + , "f2": 1, + , "f3": 2 + } + } +} +</pre></div></div> + +<p>This would parse the following message:</p> + +<div> +<div> +<pre class="source">{ + "meta_f1" : "val1", + "payload" : "foo,bar,grok", + "original_string" : "2019 Jul, 01: val1 foo,bar,grok", + "timestamp" : 10000 +} +</pre></div></div> + +<p>into</p> + +<div> +<div> +<pre class="source">{ + "meta_f1" : "val1", + "f1" : "foo", + "f2" : "bar", + "f3" : "grok", + "original_string" : "2019 Jul, 01: val1 foo,bar,grok", + "timestamp" : 10002 +} +</pre></div></div> + +<p>Note a couple of things here:</p> +<ul> + +<li>The metadata field <tt>meta_f1</tt> is not prefixed here because we configured the strategy with <tt>metadataPrefix</tt> as empty string.</li> +<li>The <tt>timestamp</tt> is not inherited from the metadata</li> +<li>The <tt>original_string</tt> is inherited from the metadata</li> +</ul></div></div> + </div> + </div> + </div> + <hr/> + <footer> + <div class="container-fluid"> + <div class="row-fluid"> +é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, + and the Apache Metron project logo are trademarks of The Apache Software Foundation. + </div> + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-parsers/index.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-parsers/index.html b/site/current-book/metron-platform/metron-parsers/index.html index 807a24e..0d1b7e4 100644 --- a/site/current-book/metron-platform/metron-parsers/index.html +++ b/site/current-book/metron-platform/metron-parsers/index.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/index.md at 2018-06-07 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/index.md at 2018-09-12 | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Parsers</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> @@ -32,8 +32,8 @@ <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> <li class="active ">Parsers</li> - <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> - <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> </ul> </div> <div class="row-fluid"> @@ -55,20 +55,22 @@ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> - <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> <li class="active"><a href="#"><span class="icon-chevron-down"></span>Parsers</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/metron-parsers/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li> + <li><a href="../../metron-platform/metron-parsers/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li> <li><a href="../../metron-platform/metron-parsers/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li> </ul> </li> <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> </ul> </li> @@ -152,6 +154,8 @@ limitations under the License. </ul> </li> <li><tt>jsonpQuery</tt> : A <a href="#json_path">JSON Path</a> query string. If present, the result of the JSON Path query should be a list of messages. This is useful if you have a JSON document which contains a list or array of messages embedded in it, and you do not have another means of splitting the message.</li> +<li><tt>wrapInEntityArray</tt> : <tt>"true" or "false"</tt>. If <tt>jsonQuery</tt> is present and this flag is present and set to <tt>"true"</tt>, the incoming message will be wrapped in a JSON entity and array. for example: <tt>{"name":"value"},{"name2","value2}</tt> will be wrapped as <tt>{"message" : [{"name":"value"},{"name2","value2}]}</tt>. This is using the default value for <tt>wrapEntityName</tt> if that property is not set.</li> +<li><tt>wrapEntityName</tt> : Sets the name to use when wrapping JSON using <tt>wrapInEntityArray</tt>. The <tt>jsonpQuery</tt> should reference this name.</li> <li>A field called <tt>timestamp</tt> is expected to exist and, if it does not, then current time is inserted.</li> </ul> </li> @@ -179,7 +183,9 @@ limitations under the License. <div class="section"> <h2><a name="Parser_Architecture"></a>Parser Architecture</h2> <p><img src="../../images/parser_arch.png" alt="Architecture" /></p> -<p>Data flows through the parser bolt via kafka and into the <tt>enrichments</tt> topology in kafka. Errors are collected with the context of the error (e.g. stacktrace) and original message causing the error and sent to an <tt>error</tt> queue. Invalid messages as determined by global validation functions are also treated as errors and sent to an <tt>error</tt> queue.</p></div> +<p>Data flows through the parser bolt via kafka and into the <tt>enrichments</tt> topology in kafka. Errors are collected with the context of the error (e.g. stacktrace) and original message causing the error and sent to an <tt>error</tt> queue. Invalid messages as determined by global validation functions are also treated as errors and sent to an <tt>error</tt> queue.</p> +<p>Multiple sensors can be aggregated into a single Storm topology. When this is done, there will be multiple Kafka spouts, but only a single parser bolt which will handle delegating to the correct parser as needed. There are some constraints around this, in particular regarding some configuration. Additionally, all sensors must flow to the same error topic. The Kafka topic is retrieved from the input Tuple itself.</p> +<p>A worked example of this can be found in the <a href="../../use-cases/parser_chaining/index.html#aggregated-parsers-with-parser-chaining">Parser Chaining use case</a>.</p></div> <div class="section"> <h2><a name="Message_Format"></a>Message Format</h2> <p>All Metron messages follow a specific format in order to ingest a message. If a message does not conform to this format it will be dropped and put onto an error queue for further examination. The message must be of a JSON format and must have a JSON tag message like so:</p> @@ -200,7 +206,7 @@ limitations under the License. <li>timestamp (epoch)</li> <li>original_string: A human friendly string representation of the message</li> </ul> -<p>The timestamp and original_string fields are madatory. The remaining standard fields are optional. If any of the optional fields are not applicable then the field should be left out of the JSON.</p> +<p>The timestamp and original_string fields are mandatory. The remaining standard fields are optional. If any of the optional fields are not applicable then the field should be left out of the JSON.</p> <p>So putting it all together a typical Metron message with all 5-tuple fields present would look like the following:</p> <div> @@ -234,7 +240,8 @@ limitations under the License. <li><tt>raw_message</tt> : The raw message in string form</li> <li><tt>raw_message_bytes</tt> : The raw message bytes</li> <li><tt>error_hash</tt> : A hash of the error message</li> -</ul></div></div> +</ul> +<p>When aggregating multiple sensors, all sensors must be using the same error topic.</p></div></div> <div class="section"> <h2><a name="Parser_Configuration"></a>Parser Configuration</h2> <p>The configuration for the various parser topologies is defined by JSON documents stored in zookeeper.</p> @@ -263,20 +270,29 @@ limitations under the License. <ul> <li><tt>sensorTopic</tt> : The kafka topic to send the parsed messages to. If the topic is prefixed and suffixed by <tt>/</tt> then it is assumed to be a regex and will match any topic matching the pattern (e.g. <tt>/bro.*/</tt> would match <tt>bro_cust0</tt>, <tt>bro_cust1</tt> and <tt>bro_cust2</tt>)</li> -<li><tt>readMetadata</tt> : Boolean indicating whether to read metadata or not (<tt>false</tt> by default). See below for a discussion about metadata.</li> -<li><tt>mergeMetadata</tt> : Boolean indicating whether to merge metadata with the message or not (<tt>false</tt> by default). See below for a discussion about metadata.</li> -<li><tt>parserConfig</tt> : A JSON Map representing the parser implementation specific configuration.</li> +<li><tt>readMetadata</tt> : Boolean indicating whether to read metadata or not (The default is raw message strategy dependent). See below for a discussion about metadata.</li> +<li><tt>mergeMetadata</tt> : Boolean indicating whether to merge metadata with the message or not (The default is raw message strategy dependent). See below for a discussion about metadata.</li> +<li><tt>rawMessageStrategy</tt> : The strategy to use when reading the raw data and metadata. See below for a discussion about message reading strategies.</li> +<li><tt>rawMessageStrategyConfig</tt> : The raw message strategy configuration map. See below for a discussion about message reading strategies.</li> +<li><tt>parserConfig</tt> : A JSON Map representing the parser implementation specific configuration. Also include batch sizing and timeout for writer configuration here. +<ul> + +<li><tt>batchSize</tt> : Integer indicating number of records to batch together before sending to the writer. (default to <tt>15</tt>)</li> +<li><tt>batchTimeout</tt> : The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to <tt>0</tt>, it defaults to a system-determined duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>. Ignored if batchSize is <tt>1</tt>, since this disables batching.</li> +<li>The kafka writer can be configured within the parser config as well. (This is all configured a priori, but this is convenient for overriding the settings). See <a href="../metron-writer/index.html#kafka-writer">here</a></li> +</ul> +</li> <li><tt>fieldTransformations</tt> : An array of complex objects representing the transformations to be done on the message generated from the parser before writing out to the kafka topic.</li> -<li><tt>spoutParallelism</tt> : The kafka spout parallelism (default to <tt>1</tt>). This can be overridden on the command line.</li> -<li><tt>spoutNumTasks</tt> : The number of tasks for the spout (default to <tt>1</tt>). This can be overridden on the command line.</li> -<li><tt>parserParallelism</tt> : The parser bolt parallelism (default to <tt>1</tt>). This can be overridden on the command line.</li> -<li><tt>parserNumTasks</tt> : The number of tasks for the parser bolt (default to <tt>1</tt>). This can be overridden on the command line.</li> +<li><tt>spoutParallelism</tt> : The kafka spout parallelism (default to <tt>1</tt>). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.</li> +<li><tt>spoutNumTasks</tt> : The number of tasks for the spout (default to <tt>1</tt>). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.</li> +<li><tt>parserParallelism</tt> : The parser bolt parallelism (default to <tt>1</tt>). If there are multiple sensors, the last one’s configuration will be used. This can be overridden on the command line.</li> +<li><tt>parserNumTasks</tt> : The number of tasks for the parser bolt (default to <tt>1</tt>). If there are multiple sensors, the last one’s configuration will be used. This can be overridden on the command line.</li> <li><tt>errorWriterParallelism</tt> : The error writer bolt parallelism (default to <tt>1</tt>). This can be overridden on the command line.</li> <li><tt>errorWriterNumTasks</tt> : The number of tasks for the error writer bolt (default to <tt>1</tt>). This can be overridden on the command line.</li> <li><tt>numWorkers</tt> : The number of workers to use in the topology (default is the storm default of <tt>1</tt>).</li> <li><tt>numAckers</tt> : The number of acker executors to use in the topology (default is the storm default of <tt>1</tt>).</li> -<li><tt>spoutConfig</tt> : A map representing a custom spout config (this is a map). This can be overridden on the command line.</li> -<li><tt>securityProtocol</tt> : The security protocol to use for reading from kafka (this is a string). This can be overridden on the command line and also specified in the spout config via the <tt>security.protocol</tt> key. If both are specified, then they are merged and the CLI will take precedence.</li> +<li><tt>spoutConfig</tt> : A map representing a custom spout config (this is a map). If there are multiple sensors, the configs will be merged with the last specified taking precedence. This can be overridden on the command line.</li> +<li><tt>securityProtocol</tt> : The security protocol to use for reading from kafka (this is a string). This can be overridden on the command line and also specified in the spout config via the <tt>security.protocol</tt> key. If both are specified, then they are merged and the CLI will take precedence. If multiple sensors are used, any non “PLAINTEXT” value will be used.</li> <li><tt>stormConfig</tt> : The storm config to use (this is a map). This can be overridden on the command line. If both are specified, they are merged with CLI properties taking precedence.</li> <li><tt>cacheConfig</tt> : Cache config for stellar field transformations. This configures a least frequently used cache. This is a map with the following keys. If not explicitly configured (the default), then no cache will be used. <ul> @@ -331,17 +347,52 @@ Consider the following scenarios:</p> </li> <li>Custom metadata: Custom metadata from an individual telemetry source that one might want to use within Metron.</li> </ul> -<p>Metadata is controlled by two fields in the parser:</p> +<p>Metadata is controlled by the following parser configs:</p> +<ul> + +<li><tt>rawMessageStrategy</tt> : This is a strategy which indicates how to read data and metadata. The strategies supported are: +<ul> + +<li><tt>DEFAULT</tt> : Data is read directly from the kafka record value and metadata, if any, is read from the kafka record key. This strategy defaults to not reading metadata and not merging metadata. This is the default strategy.</li> +<li><tt>ENVELOPE</tt> : Data from kafka record value is presumed to be a JSON blob. One of these fields must contain the raw data to pass to the parser. All other fields should be considered metadata. The field containing the raw data is specified in the <tt>rawMessageStrategyConfig</tt>. Data held in the kafka key as well as the non-data fields in the JSON blob passed into the kafka value are considered metadata. Note that the exception to this is that any <tt>original_string</tt> field is inherited from the envelope data so that the original string contains the envelope data. If you do not prefer this behavior, remove this field from the envelope data.</li> +</ul> +</li> +<li><tt>rawMessageStrategyConfig</tt> : The configuration (a map) for the <tt>rawMessageStrategy</tt>. Available configurations are strategy dependent: +<ul> + +<li><tt>DEFAULT</tt> +<ul> + +<li><tt>metadataPrefix</tt> defines the key prefix for metadata (default is <tt>metron.metadata</tt>).</li> +</ul> +</li> +<li><tt>ENVELOPE</tt> +<ul> + +<li><tt>metadataPrefix</tt> defines the key prefix for metadata (default is <tt>metron.metadata</tt>)</li> +<li><tt>messageField</tt> defines the field from the envelope to use as the data. All other fields are considered metadata.</li> +</ul> +</li> +</ul> +</li> +<li><tt>readMetadata</tt> : This is a boolean indicating whether metadata will be read and made available to Field transformations (i.e. Stellar field transformations). The default is dependent upon the <tt>rawMessageStrategy</tt>: +<ul> + +<li><tt>DEFAULT</tt> : default to <tt>false</tt>.</li> +<li><tt>ENVELOPE</tt> : default to <tt>true</tt>.</li> +</ul> +</li> +<li><tt>mergeMetadata</tt> : This is a boolean indicating whether metadata fields will be merged with the message automatically. That is to say, if this property is set to <tt>true</tt> then every metadata field will become part of the messages and, consequently, also available for use in field transformations. The default is dependent upon the <tt>rawMessageStrategy</tt>: <ul> -<li><tt>readMetadata</tt> : This is a boolean indicating whether metadata will be read and made available to Field transformations (i.e. Stellar field transformations). The default is <tt>false</tt>.</li> -<li><tt>mergeMetadata</tt> : This is a boolean indicating whether metadata fields will be merged with the message automatically.<br /> -That is to say, if this property is set to <tt>true</tt> then every metadata field will become part of the messages and, consequently, also available for use in field transformations.</li> +<li><tt>DEFAULT</tt> : default to <tt>false</tt>.</li> +<li><tt>ENVELOPE</tt> : default to <tt>true</tt>.</li> +</ul> +</li> </ul> <div class="section"> <h4><a name="Field_Naming"></a>Field Naming</h4> -<p>In order to avoid collisions from metadata fields, metadata fields will be prefixed with <tt>metron.metadata.</tt>.<br /> -So, for instance the kafka topic would be in the field <tt>metron.metadata.topic</tt>.</p></div> +<p>In order to avoid collisions from metadata fields, metadata fields will be prefixed (the default is <tt>metron.metadata.</tt>, but this is configurable in the <tt>rawMessageStrategyConfig</tt>). So, for instance the kafka topic would be in the field <tt>metron.metadata.topic</tt>.</p></div> <div class="section"> <h4><a name="Specifying_Custom_Metadata"></a>Specifying Custom Metadata</h4> <p>Custom metadata is specified by sending a JSON Map in the key. If no key is sent, then, obviously, no metadata will be parsed. For instance, sending a metadata field called <tt>customer_id</tt> could be done by sending</p> @@ -470,6 +521,36 @@ So, for instance the kafka topic would be in the field <tt>metron.metadata.topic ] } </pre></div></div> + +<ul> + +<li><tt>REGEX_SELECT</tt> : This transformation lets users set an output field to one of a set of possibilities based on matching regexes. This transformation is useful when the number or conditions are large enough to make a stellar language match statement unwieldy.</li> +</ul> +<p>The following config will set the field <tt>logical_source_type</tt> to one of the following, dependent upon the value of the <tt>pix_type</tt> field:</p> +<ul> + +<li><tt>cisco-6-302</tt> if <tt>pix_type</tt> starts with either <tt>6-302</tt> or <tt>06-302</tt></li> +<li><tt>cisco-5-304</tt> if <tt>pix_type</tt> starts with <tt>5-304</tt></li> +</ul> + +<div> +<div> +<pre class="source">{ +... + "fieldTransformations" : [ + { + "transformation" : "REGEX_ROUTING" + ,"input" : "pix_type" + ,"output" : "logical_source_type" + ,"config" : { + "cisco-6-302" : [ "^6-302.*", "^06-302.*"] + "cisco-5-304" : "^5-304.*" + } + } + ] +... +} +</pre></div></div> </div> <div class="section"> <h3><a name="Assignment_to_null"></a>Assignment to <tt>null</tt></h3> @@ -685,10 +766,12 @@ HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )" <p><a name="Notes_on_Performance_Tuning"></a></p> <h1>Notes on Performance Tuning</h1> <p>Default installed Metron is untuned for production deployment. There are a few knobs to tune to get the most out of your system.</p> +<p>When using aggregated parsers, it’s highly recommended to aggregate parsers with similar velocity and parser complexity together.</p> <p><a name="Notes_on_Adding_a_New_Sensor"></a></p> <h1>Notes on Adding a New Sensor</h1> <p>In order to allow for meta alerts to be queries alongside regular alerts in Elasticsearch 2.x, it is necessary to add an additional field to the templates and mapping for existing sensors.</p> -<p>Please see a description of the steps necessary to make this change in the metron-elasticsearch <a href="../../metron-platform/metron-elasticsearch/index.html#Using_Metron_with_Elasticsearch_2.x">Using Metron with Elasticsearch 2.x</a></p></div> +<p>Please see a description of the steps necessary to make this change in the metron-elasticsearch <a href="../../metron-platform/metron-elasticsearch/index.html#Using_Metron_with_Elasticsearch_2.x">Using Metron with Elasticsearch 2.x</a></p> +<p>If Solr is selected as the real-time store, it is also necessary to add additional fields. See the <a href="../metron-indexing/index.html#Solr">Solr</a> section in metron-indexing for more details.</p></div> <div class="section"> <h2><a name="Kafka_Queue"></a>Kafka Queue</h2> <p>The kafka queue associated with your parser is a collection point for all of the data sent to your parser. As such, make sure that the number of partitions in the kafka topic is sufficient to handle the throughput that you expect from your parser topology.</p></div> http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-parsers/parser-testing.html ---------------------------------------------------------------------- diff --git a/site/current-book/metron-platform/metron-parsers/parser-testing.html b/site/current-book/metron-platform/metron-parsers/parser-testing.html index ff22013..945aebc 100644 --- a/site/current-book/metron-platform/metron-parsers/parser-testing.html +++ b/site/current-book/metron-platform/metron-parsers/parser-testing.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/parser-testing.md at 2018-06-07 + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/parser-testing.md at 2018-09-12 | Rendered using Apache Maven Fluido Skin 1.7 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20180607" /> + <meta name="Date-Revision-yyyymmdd" content="20180912" /> <meta http-equiv="Content-Language" content="en" /> <title>Metron – Parser Contribution and Testing</title> <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> @@ -32,8 +32,8 @@ <li class=""><a href="http://metron.apache.org/"; class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> <li class="active ">Parser Contribution and Testing</li> - <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li> - <li id="projectVersion" class="pull-right">Version: 0.5.0</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li> + <li id="projectVersion" class="pull-right">Version: 0.6.0</li> </ul> </div> <div class="row-fluid"> @@ -55,20 +55,22 @@ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> - <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li> <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> + <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li> <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-down"></span>Parsers</a> <ul class="nav nav-list"> <li><a href="../../metron-platform/metron-parsers/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li> + <li><a href="../../metron-platform/metron-parsers/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li> <li class="active"><a href="#"><span class="none"></span>parser-testing</a></li> </ul> </li> <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> + <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> </ul> </li>
