Added: eagle/site/docs/v0.5.0/using-eagle/index.html
URL:
http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/using-eagle/index.html?rev=1789965&view=auto
==============================================================================
--- eagle/site/docs/v0.5.0/using-eagle/index.html (added)
+++ eagle/site/docs/v0.5.0/using-eagle/index.html Mon Apr 3 11:33:14 2017
@@ -0,0 +1,1178 @@
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+
+
+ <title>Using Eagle - Apache Eagle Documentation</title>
+
+
+ <link rel="shortcut icon" href="../include/images/favicon.png">
+
+
+
+ <link
href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700'
rel='stylesheet' type='text/css'>
+
+ <link rel="stylesheet" href="../css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
+ <link rel="stylesheet" href="../css/highlight.css">
+
+
+ <script>
+ // Current page data
+ var mkdocs_page_name = "Using Eagle";
+ var mkdocs_page_input_path = "using-eagle.md";
+ var mkdocs_page_url = "/using-eagle/";
+ </script>
+
+ <script src="../js/jquery-2.1.1.min.js"></script>
+ <script src="../js/modernizr-2.8.3.min.js"></script>
+ <script type="text/javascript" src="../js/highlight.pack.js"></script>
+ <script src="../js/theme.js"></script>
+
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+ <div class="wy-side-nav-search">
+ <a href=".." class="icon icon-home"> Apache Eagle Documentation</a>
+ <div role="search">
+ <form id ="rtd-search-form" class="wy-form" action="../search.html"
method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ </form>
+</div>
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation"
aria-label="main navigation">
+ <ul class="current">
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="..">Home</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../getting-started/">Getting Started</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 current">
+ <a class="current" href="./">Using Eagle</a>
+
+ <ul>
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#manage-eagle-and-services">Manage Eagle and Services</a></li>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#use-eagle-web-interface">Use Eagle Web Interface</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#home">Home</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#alert">Alert</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#integration">Integration</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#sites">Sites</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#setup-the-monitoring-application">Setup The Monitoring
Application</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#monitoring-applications">Monitoring Applications</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#managing-sites">Managing Sites</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#sites_1">Sites</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#create-site">Create Site</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#configuring-a-site">Configuring a Site</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#install-and-run-applications-in-site">Install and Run Applications in
Site</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#define-policies">Define Policies</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#policies">Policies</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#define-or-edit-policies">Define or Edit Policies</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#source-stream">Source Stream</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#policy-name">Policy Name</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#publish-alerts">Publish Alerts</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#policy-syntax">Policy Syntax</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#monitoring-dashboard">Monitoring Dashboard</a></li>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#check-the-alerts">Check The Alerts</a></li>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#how-to-stream-audit-log-into-kafka">How to stream audit log into
Kafka</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#logstash">Logstash</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#filebeat">Filebeat</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#log4j-kafka-appender">Log4j Kafka Appender</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+ </ul>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../applications/">Applications</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../developing-application/">Developing
Application</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../underlying-design/">Underlying Design</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../reference/">Reference</a>
+
+ </li>
+<li>
+
+ </ul>
+ </div>
+
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="..">Apache Eagle Documentation</a>
+ </nav>
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+ <div role="navigation" aria-label="breadcrumbs navigation">
+ <ul class="wy-breadcrumbs">
+ <li><a href="..">Docs</a> »</li>
+
+
+
+ <li>Using Eagle</li>
+ <li class="wy-breadcrumbs-aside">
+
+
+ <a href="https://github.com/apache/eagle/tree/master/docs";
class="icon icon-github"> Edit on GitHub</a>
+
+
+ </li>
+ </ul>
+ <hr/>
+</div>
+ <div role="main">
+ <div class="section">
+
+ <h1 id="manage-eagle-and-services">Manage Eagle and
Services</h1>
+<ul>
+<li>
+<p>After Apache Eagle has been deployed (please reference <a
href="../getting-started/#deployment"><strong>deployment</strong></a>), you can
enter deployment directory and use commands below to control Apache Eagle
Server.</p>
+<pre><code>./bin/eagle-server.sh start|stop|status
+</code></pre>
+</li>
+<li>
+<p>After starting the Eagle server, please type
http://<EAGLE_SERVER_HOST>:<PORT>/ to open the web ui of Eagle.</p>
+</li>
+</ul>
+<hr />
+<h1 id="use-eagle-web-interface">Use Eagle Web Interface</h1>
+<ul>
+<li>
+<p>This is the typical Web Interface (short for WI) after setting up your
Eagle monitoring environment. WI majorly contain the right main panel and left
function menu.</p>
+<p><img alt="Eagle Web Interface"
src="../include/images/eagle_web_interface.png" /></p>
+</li>
+</ul>
+<h2 id="home">Home</h2>
+<ul>
+<li>
+<p>This is the aggregated UI for configured sites, and the applications. It
will show those created sites created, how many application installed for each
sites, and alerts generated from that cluster. You can click âMore infoâ
link to view the details for particular site.</p>
+</li>
+<li>
+<p>The â<strong>Widgets</strong>â section is customizable; if the
application developer have its application registered to Home page, you can
find that in â<strong>Widgets</strong>â section. Please check the
application developer guide about how to register applications to home widgets.
It give you a shortcut to go directly to the application home.</p>
+</li>
+</ul>
+<h2 id="alert">Alert</h2>
+<ul>
+<li>In Alert menu, you can define the policies, list the policies and check
your alerts there. </li>
+</ul>
+<h2 id="integration">Integration</h2>
+<ul>
+<li>The integration page provides the management functionality for Eagle. You
can list the built-in applications there, create sites, and manage the
applications in your site.</li>
+</ul>
+<h2 id="sites">Sites</h2>
+<ul>
+<li>It also gives you a shortcut to particular site.</li>
+</ul>
+<hr />
+<h1 id="setup-the-monitoring-application">Setup The Monitoring Application</h1>
+<h2 id="monitoring-applications">Monitoring Applications</h2>
+<ul>
+<li>
+<p>Eagle has an extensible framework to dynamically add new monitoring
applications in Eagle environment. It also ships some built-in big data
monitoring applications.</p>
+</li>
+<li>
+<p>Go to â<strong>Integration</strong>â ->
â<strong>Applications</strong>â, it will list a set of available monitoring
applications which you can choose to monitor your services.</p>
+<p><img alt="Integation Applications"
src="../include/images/integration_applications.png" /></p>
+</li>
+<li>
+<p>The â<strong>Application</strong>â column is the display name for an
application, â<strong>Streams</strong>â is a logical name for the data
stream from the monitored source after pre-processing, which will consumed by
Alert Engine.</p>
+</li>
+<li>
+<p>At the moment, we have the below built-in applications shipped with Apache
Eagle. You can refer to the application documentation to understand how to do
the configuration for each monitoring application.</p>
+<table>
+<thead>
+<tr>
+<th>Application</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Topology Health Check</td>
+<td>This application can be used to monitor the service healthiness for HDFS,
HBase and YARN. You can get alerted once the master role or the slave role got
crashed.</td>
+</tr>
+<tr>
+<td>Hadoop JMX Metrics Monitoring</td>
+<td>This application can be used to monitor the JMX metrics data from the
master nodes of HDFS, HBase and YARN, e.g. NameNode, HBase Master and YARN
Resource Manager.</td>
+</tr>
+<tr>
+<td>HDFS Audit Log Monitor</td>
+<td>This application can be used to monitor the data operations in HDFS, to
detect sensitive data access and malicious operations; to protect from data
leak or data loss.</td>
+</tr>
+<tr>
+<td>HBase Audit Log Monitor</td>
+<td>Same as HDFS Audit Log Monitor, this application is used to monitor the
data operations in HBase.</td>
+</tr>
+<tr>
+<td>Map Reduce History Job</td>
+<td>This application is used to get the MapReduce history job counters from
YARN history server and job running history from HDFS log directory.</td>
+</tr>
+<tr>
+<td>Map Reduce Running Job</td>
+<td>This application is used to get the MapReduce running job counter
information using YARN Rest API.</td>
+</tr>
+<tr>
+<td>Hadoop Queue Monitor</td>
+<td>This application is used to get the resource scheduling and utilization
info from YARN.</td>
+</tr>
+<tr>
+<td>MR Metrics Aggregation</td>
+<td>This application is used to aggregate the job counters and some resource
utilization in a certain period of time (daily, weekly or monthly).</td>
+</tr>
+<tr>
+<td>Job Performance Monitor Web</td>
+<td>This application only contains the frontend, and depends on Map Reduce
History Job and Map Reduce Running Job.</td>
+</tr>
+<tr>
+<td>Alert Engine</td>
+<td>Alert Engine is a special application and used to process the output data
from other applications.</td>
+</tr>
+</tbody>
+</table>
+</li>
+</ul>
+<h2 id="managing-sites">Managing Sites</h2>
+<p>To enable a real monitoring use case, you have to create a site first, and
install a certain application for this site, and finally start the application.
We use site concept to group the running applications and avoid the application
conflict.</p>
+<h3 id="sites_1">Sites</h3>
+<ul>
+<li>
+<p>Go to â<strong>Integration</strong>â ->
â<strong>Sites</strong>â, there will be a table listing the managed
sites.</p>
+<p><img alt="Integration Sites" src="../include/images/integration_sites.png"
/></p>
+</li>
+</ul>
+<h3 id="create-site">Create Site</h3>
+<ul>
+<li>
+<p>Click â<strong>New Site</strong>â on the bottom right of the Sites
page. You can fill the information in site creation dialog.</p>
+<p><img alt="New Site" src="../include/images/new_site.png" /></p>
+</li>
+<li>
+<p>The â<strong>Site Id</strong>â should not be duplicated. After the
creation, you can find it in sites page.</p>
+<p><img alt="Site List" src="../include/images/site_list.png" /></p>
+</li>
+</ul>
+<h3 id="configuring-a-site">Configuring a Site</h3>
+<ul>
+<li>
+<p>By clicking â<strong>Edit</strong>â button or the Site column in Sites
table, you can have the Site configuration page, there you can install
monitoring applications.</p>
+<p><img alt="Configure Site" src="../include/images/configure_site.png" /></p>
+</li>
+</ul>
+<h2 id="install-and-run-applications-in-site">Install and Run Applications in
Site</h2>
+<ul>
+<li>
+<p>Choose the particular application which you want to install, you probably
have something to fill, e.g. the HDFS NameNode address, Zookeeper address and
port. Please check each application documentation for how to configure each
application. </p>
+</li>
+<li>
+<p>After doing the installation, you can start the application by clicking
<img alt="Start Icon" src="../include/images/start_icon.png" /> or stop the
application by <img alt="Stop Icon" src="../include/images/stop_icon.png" />.
You can check the â<strong>Status</strong>â column about the running
status. Usually, it should have â<strong>INITIALIZED</strong>â or
â<strong>RUNNING</strong>â for a healthy application.</p>
+</li>
+</ul>
+<hr />
+<h1 id="define-policies">Define Policies</h1>
+<p>After setting up the monitoring applications, you probably want to setup
some alert policies against the monitored data, so you can get notified once
any violation on the data. Eagle has a centralized place for policy
definition.</p>
+<h2 id="policies">Policies</h2>
+<ul>
+<li>
+<p>Go to â<strong>Alert</strong>â -> â<strong>Policies</strong>â,
you can check the policies defined and take control on whether to enable the
policy:</p>
+<p><img alt="Alert Policies" src="../include/images/alert_policies.png" /></p>
+</li>
+<li>
+<p>You can apply the below actions for a certain policy:</p>
+<ul>
+<li>
+<p><img alt="Start Icon" src="../include/images/start_icon.png" />: enable a
policy</p>
+</li>
+<li>
+<p><img alt="Stop Icon" src="../include/images/stop_icon.png" />: disable a
policy</p>
+</li>
+<li>
+<p><img alt="Edit Icon" src="../include/images/edit_icon.png" />: edit a
policy</p>
+</li>
+<li>
+<p><img alt="Delete Icon" src="../include/images/delete_icon.png" />: purge a
policy</p>
+</li>
+</ul>
+</li>
+</ul>
+<h2 id="define-or-edit-policies">Define or Edit Policies</h2>
+<ul>
+<li>
+<p>If you want to create a new policy, click â<strong>Alert</strong>â
-> â<strong>Define Policy</strong>â, or you can enter into the policy
definition page by editing an existing policy. After that, you can go to the
policy list to enable the policy dynamically.</p>
+<p><img alt="Define Policy" src="../include/images/alert_define_policy.png"
/></p>
+</li>
+</ul>
+<h3 id="source-stream">Source Stream</h3>
+<ul>
+<li>The source stream gives user a full view about what data stream is
available for application defined for particular site, as well as the data
structures in each data stream. Data stream name is suffixed by the site
name.</li>
+</ul>
+<h3 id="policy-name">Policy Name</h3>
+<ul>
+<li>The policy name should be globally unique.</li>
+</ul>
+<h3 id="publish-alerts">Publish Alerts</h3>
+<ul>
+<li>
+<p>In this section, you can define the alert publishment method by clicking
the â<strong>+Add Publisher</strong>â.</p>
+<p><img alt="Add Publisher" src="../include/images/add_publisher.png" /></p>
+</li>
+<li>
+<p>You can choose the publishment method from an existing policy or by
creating new publisher. </p>
+</li>
+<li>
+<p>There are four built-in publisher types:</p>
+<ul>
+<li>
+<p><strong>EmailPublisher</strong>:
org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher</p>
+</li>
+<li>
+<p><strong>KafkaPublisher</strong>:
org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher</p>
+</li>
+<li>
+<p><strong>SlackPublisher</strong>:
org.apache.eagle.alert.engine.publisher.impl.AlertSlackPublisher</p>
+</li>
+<li>
+<p><strong>EagleStoragePlugin</strong>:
org.apache.eagle.alert.engine.publisher.impl.AlertEagleStoragePlugin</p>
+</li>
+</ul>
+</li>
+</ul>
+<h3 id="policy-syntax">Policy Syntax</h3>
+<ul>
+<li>
+<p>Currently, we support SiddhiQL(please view Siddhi Query Language
Specification <a
href="https://docs.wso2.com/display/CEP300/Siddhi+Language+Specification";>here</a>)</p>
+</li>
+<li>
+<p>In order to explain how stream data is processed, let us take policy below
as an example:</p>
+<pre><code>from map_reduce_failed_job_stream[site=="sandbox" and
currentState=="FAILED"]
+select * group by jobId insert into map_reduce_failed_job_stream_out
+</code></pre>
+</li>
+<li>
+<p>This policy contains below parts:</p>
+<ul>
+<li>
+<p><strong>Source</strong>: from map_reduce_failed_job_stream</p>
+</li>
+<li>
+<p><strong>Filter</strong>: [site=="sandbox" and currentState=="FAILED"]</p>
+</li>
+<li>
+<p><strong>Projection</strong>: select *</p>
+</li>
+<li>
+<p><strong>GroupBy</strong>: group by jobId</p>
+</li>
+<li>
+<p><strong>Destination</strong>: insert into
map_reduce_failed_job_stream_out</p>
+</li>
+</ul>
+</li>
+<li>
+<p>Source Streams(schema) are defined by applications, and applications will
write stream data to data sink(currently, we support kafka as data sink).</p>
+<pre><code><streams>
+ <stream>
+ <streamId>map_reduce_failed_job_stream</streamId>
+ <description>Map Reduce Failed Job Stream</description>
+ <validate>true</validate>
+ <columns>
+ <column>
+ <name>site</name>
+ <type>string</type>
+ </column>
+ â¦...
+ <column>
+ <name>jobId</name>
+ <type>string</type>
+ <column>
+ <name>currentState</name>
+ <type>string</type>
+ </column>
+ </columns>
+ </stream>
+</streams>
+</code></pre>
+</li>
+<li>
+<p>After policy is defined, Alert engine will create siddhi execution runtime
for the policy(also load stream data schema from metadata store). Since siddhi
execution runtime knows the stream data schema, then it will process stream
data and do the calculation.</p>
+</li>
+</ul>
+<hr />
+<h1 id="monitoring-dashboard">Monitoring Dashboard</h1>
+<ul>
+<li>
+<p>After setting the sites and applications, you can find the site item from
the home page or âSitesâ menu.</p>
+</li>
+<li>
+<p>Here is a site home example. After entering the site home, the left menu
will be replaced by application dashboard links only related to that site, so
you can switch between the application dashboard quickly. In the right panel,
it contains the application icons installed in this site, but depends on if the
application has its dashboard defined. You can click the application icon or
the application links to go to the application dashboard home. Please check the
application documentation about how to use the application monitoring
dashboard.</p>
+<p><img alt="Dashboard" src="../include/images/dashboard.png" /></p>
+</li>
+</ul>
+<hr />
+<h1 id="check-the-alerts">Check The Alerts</h1>
+<ul>
+<li>
+<p>Eagle has all the alerts generated by all the applications stored in its
database, so you can check your application alerts from Eagle WI. </p>
+</li>
+<li>
+<p>Go to â<strong>Alert</strong>â -> â<strong>Alerts</strong>â, you
can find the alerts table.</p>
+<p><img alt="Alerts" src="../include/images/alert_alerts.png" /></p>
+</li>
+<li>
+<p>Also you can check more detailed information by clicking
â<strong>Detail</strong>â link for each alert item.</p>
+<p><img alt="Alert Details" src="../include/images/alert_details.png" /></p>
+</li>
+</ul>
+<hr />
+<h1 id="how-to-stream-audit-log-into-kafka">How to stream audit log into
Kafka</h1>
+<h2 id="logstash">Logstash</h2>
+<p>The sample configuration is tested with logstash-2.3.4. Logstash is
required to be installed on the namenode host.</p>
+<ul>
+<li>
+<p><strong>Step 1</strong>: Create a Kafka topic as the streaming input.</p>
+<p>Here is an sample Kafka command to create topic 'sandbox_hdfs_audit_log'</p>
+<pre><code>cd <kafka-home>
+bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1
--partitions 1 --topic sandbox_hdfs_audit_log
+</code></pre>
+</li>
+<li>
+<p><strong>Step 2</strong>: Create a Logstash configuration file under
${LOGSTASH_HOME}/conf. Here is a sample.</p>
+<pre><code>input {
+ file {
+ type => "hdp-nn-audit"
+ path => "/tmp/test/hdfs-audit.log"
+ start_position => end
+ sincedb_path => "/dev/null"
+ }
+ }
+ output {
+ if [type] == "hdp-nn-audit" {
+ kafka {
+ codec => plain {
+ format => "%{message}"
+ }
+ bootstrap_servers => "host:9092"
+ topic_id => "hdfs_audit_log"
+ acks => "0"
+ timeout_ms => 10000
+
+ send_buffer_bytes => 102400
+ client_id => "hdp-nn-audit"
+
+ workers => 10
+ compression_type => "gzip"
+ }
+ # stdout { codec => rubydebug }
+ }
+}
+</code></pre>
+</li>
+<li>
+<p><strong>Step 4</strong>: Start Logstash</p>
+<pre><code>bin/logstash -f conf/sample.conf
+</code></pre>
+</li>
+<li>
+<p><strong>Step 5</strong>: Check whether logs are flowing into the kafka
topic specified by <code>topic_id</code></p>
+</li>
+</ul>
+<h2 id="filebeat">Filebeat</h2>
+<p>The sample filebeat.yml is tested with filebeat-5.0.0-beta1-linux-x86_64.
The throughput can be up to 20K messages per second. Filebeat is required to be
installed on the namenode host.</p>
+<pre><code> filebeat.publish_async: false
+ filebeat.spool_size: 8192
+ filebeat.idle_timeout: 5s
+ max_procs: 1
+ queue_size: 1000
+
+ filebeat.prospectors:
+ - input_type: log
+ paths:
+ - /tmp/test/hdfs-audit.log
+ #tail_files: true
+ harvester_buffer_size: 8192
+
+ output.kafka:
+ enabled: true
+ hosts: ["host:9092"]
+ topic: "phx_hdfs_audit_log"
+ client_id: "client-host"
+ worker: 10
+ max_retries: 3
+ bulk_max_size: 8192
+ channel_buffer_size: 512
+ timeout: 10
+ broker_timeout: 3s
+ keep_alive: 0
+ compression: none
+ max_message_bytes: 1000000
+ required_acks: 0
+ flush_interval: 1
+
+ logging.metrics.period: 10s
+
+ processors:
+ - include_fields:
+ fields: ["message", "beat.hostname"]
+</code></pre>
+<h2 id="log4j-kafka-appender">Log4j Kafka Appender</h2>
+<p>This sample configuration is tested in HDP sandbox. <code>Restarting
namenode is required</code> after updating the log4j configuration. </p>
+<ul>
+<li>
+<p><strong>Step 1</strong>: Create a Kafka topic. Here is an example Kafka
command for creating topic "sandbox_hdfs_audit_log"</p>
+<pre><code>cd <kafka-home>
+bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1
--partitions 1 --topic sandbox_hdfs_audit_log
+</code></pre>
+</li>
+<li>
+<p><strong>Step 2</strong>: Configure $HADOOP_CONF_DIR/log4j.properties, and
add a log4j appender "KAFKA_HDFS_AUDIT" to hdfs audit logging</p>
+<pre><code>log4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender
+log4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log
+log4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.hortonworks.com:6667
+log4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer
+log4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout
+log4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8601} %p %c{2}:
%m%n
+log4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async
+#log4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1
+#log4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1
+</code></pre>
+</li>
+<li>
+<p><strong>Step 3</strong>: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and add the
reference to KAFKA_HDFS_AUDIT to HADOOP_NAMENODE_OPTS.</p>
+<pre><code>-Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT
+</code></pre>
+</li>
+<li>
+<p><strong>Step 4</strong>: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and append
the following command to it.</p>
+<pre><code>export
HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/path/to/eagle/lib/log4jkafka/lib/*
+</code></pre>
+</li>
+<li>
+<p><strong>Step 5</strong>: save the changes and restart the namenode.</p>
+</li>
+<li>
+<p><strong>Step 6</strong>: Check whether logs are flowing into Topic
sandbox_hdfs_audit_log</p>
+<pre><code>$ /usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh
--zookeeper localhost:2181 --topic sandbox_hdfs_audit_log
+</code></pre>
+</li>
+</ul>
+
+ </div>
+ </div>
+ <footer>
+
+ <div class="rst-footer-buttons" role="navigation" aria-label="footer
navigation">
+
+ <a href="../applications/" class="btn btn-neutral float-right"
title="Applications">Next <span class="icon icon-circle-arrow-right"></span></a>
+
+
+ <a href="../getting-started/" class="btn btn-neutral" title="Getting
Started"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+
+ </div>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <!-- Copyright etc -->
+
+ </div>
+
+ Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a
href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a
href="https://readthedocs.org";>Read the Docs</a>.
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+<div class="rst-versions" role="note" style="cursor: pointer">
+ <span class="rst-current-version" data-toggle="rst-current-version">
+
+ <a href="https://github.com/apache/eagle/tree/master/docs";
class="icon icon-github" style="float: left; color: #fcfcfc"> GitHub</a>
+
+
+ <span><a href="../getting-started/" style="color: #fcfcfc;">«
Previous</a></span>
+
+
+ <span style="margin-left: 15px"><a href="../applications/"
style="color: #fcfcfc">Next »</a></span>
+
+ </span>
+</div>
+
+</body>
+</html>
Propchange: eagle/site/docs/v0.5.0/using-eagle/index.html
------------------------------------------------------------------------------
svn:eol-style = native
