Author: hao
Date: Mon Apr 3 11:35:50 2017
New Revision: 1789966
URL: http://svn.apache.org/viewvc?rev=1789966&view=rev
Log:
Add docs for latest and v0.5.0
Added:
eagle/site/docs/latest/applications/
eagle/site/docs/latest/applications/index.html (with props)
eagle/site/docs/latest/css/highlight.css (with props)
eagle/site/docs/latest/developing-application/
eagle/site/docs/latest/developing-application/index.html (with props)
eagle/site/docs/latest/getting-started/
eagle/site/docs/latest/getting-started/index.html (with props)
eagle/site/docs/latest/include/images/hdfs_install_2.png (with props)
eagle/site/docs/latest/include/images/hdfs_install_3.png (with props)
eagle/site/docs/latest/include/images/hdfs_policy_1.png (with props)
eagle/site/docs/latest/include/images/health_check_installation.png (with
props)
eagle/site/docs/latest/include/images/health_check_policy.png (with props)
eagle/site/docs/latest/include/images/health_check_settings.png (with
props)
eagle/site/docs/latest/include/images/health_check_stream.png (with props)
eagle/site/docs/latest/index.html (with props)
eagle/site/docs/latest/js/highlight.pack.js (with props)
eagle/site/docs/latest/reference/
eagle/site/docs/latest/reference/index.html (with props)
eagle/site/docs/latest/underlying-design/
eagle/site/docs/latest/underlying-design/index.html (with props)
eagle/site/docs/latest/using-eagle/
eagle/site/docs/latest/using-eagle/index.html (with props)
eagle/site/docs/v0.5.0/css/highlight.css (with props)
eagle/site/docs/v0.5.0/include/images/hdfs_install_2.png (with props)
eagle/site/docs/v0.5.0/include/images/hdfs_install_3.png (with props)
eagle/site/docs/v0.5.0/include/images/hdfs_policy_1.png (with props)
eagle/site/docs/v0.5.0/include/images/health_check_installation.png (with
props)
eagle/site/docs/v0.5.0/include/images/health_check_policy.png (with props)
eagle/site/docs/v0.5.0/include/images/health_check_settings.png (with
props)
eagle/site/docs/v0.5.0/include/images/health_check_stream.png (with props)
eagle/site/docs/v0.5.0/js/highlight.pack.js (with props)
Added: eagle/site/docs/latest/applications/index.html
URL:
http://svn.apache.org/viewvc/eagle/site/docs/latest/applications/index.html?rev=1789966&view=auto
==============================================================================
--- eagle/site/docs/latest/applications/index.html (added)
+++ eagle/site/docs/latest/applications/index.html Mon Apr 3 11:35:50 2017
@@ -0,0 +1,1250 @@
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+
+
+ <title>Applications - Apache Eagle Documentation</title>
+
+
+ <link rel="shortcut icon" href="../include/images/favicon.png">
+
+
+
+ <link
href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700'
rel='stylesheet' type='text/css'>
+
+ <link rel="stylesheet" href="../css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
+ <link rel="stylesheet" href="../css/highlight.css">
+
+
+ <script>
+ // Current page data
+ var mkdocs_page_name = "Applications";
+ var mkdocs_page_input_path = "applications.md";
+ var mkdocs_page_url = "/applications/";
+ </script>
+
+ <script src="../js/jquery-2.1.1.min.js"></script>
+ <script src="../js/modernizr-2.8.3.min.js"></script>
+ <script type="text/javascript" src="../js/highlight.pack.js"></script>
+ <script src="../js/theme.js"></script>
+
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+ <div class="wy-side-nav-search">
+ <a href=".." class="icon icon-home"> Apache Eagle Documentation</a>
+ <div role="search">
+ <form id ="rtd-search-form" class="wy-form" action="../search.html"
method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ </form>
+</div>
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation"
aria-label="main navigation">
+ <ul class="current">
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="..">Home</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../getting-started/">Getting Started</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../using-eagle/">Using Eagle</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 current">
+ <a class="current" href="./">Applications</a>
+
+ <ul>
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#hdfs-data-activity-monitoring">HDFS Data Activity Monitoring</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#monitor-requirements">Monitor Requirements</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#setup-installation">Setup & Installation</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#how-to-collect-the-log">How to collect the log</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#sample-policies">Sample policies</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#1-monitor-filefolder-operations">1. monitor file/folder
operations</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#2-classify-the-filefolder-on-hdfs">2. classify the file/folder on
HDFS</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 5em;"
+
+ href="#3-classify-the-ip-zone">3. Classify the IP Zone</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#questions-on-this-application">Questions on this application</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#jmx-monitoring">JMX Monitoring</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#setup-installation_1">Setup & Installation</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#define-jmx-alert-policy">Define JMX Alert Policy</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#stream-schema">Stream Schema</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#metrics-list">Metrics List</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#job-performance-monitoring">Job Performance Monitoring</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#monitor-requirements_1">Monitor Requirements</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#applications">Applications</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#data-ingestion-and-process">Data Ingestion And Process</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#setup-installation_2">Setup & Installation</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#integration-with-alert-engine">Integration With Alert Engine</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#stream-schema_1">Stream Schema</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#topology-health-check">Topology Health Check</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#setup-installation_3">Setup & Installation</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#define-health-check-alert-policy">Define Health Check Alert
Policy</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#hadoop-queue-monitoring">Hadoop Queue Monitoring</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#setup-installation_4">Setup & Installation</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#use-of-the-application">Use of the application</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+ </ul>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../developing-application/">Developing
Application</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../underlying-design/">Underlying Design</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../reference/">Reference</a>
+
+ </li>
+<li>
+
+ </ul>
+ </div>
+
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="..">Apache Eagle Documentation</a>
+ </nav>
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+ <div role="navigation" aria-label="breadcrumbs navigation">
+ <ul class="wy-breadcrumbs">
+ <li><a href="..">Docs</a> »</li>
+
+
+
+ <li>Applications</li>
+ <li class="wy-breadcrumbs-aside">
+
+
+ <a href="https://github.com/apache/eagle/tree/master/docs";
class="icon icon-github"> Edit on GitHub</a>
+
+
+ </li>
+ </ul>
+ <hr/>
+</div>
+ <div role="main">
+ <div class="section">
+
+ <h1 id="hdfs-data-activity-monitoring">HDFS Data Activity
Monitoring</h1>
+<h2 id="monitor-requirements">Monitor Requirements</h2>
+<p>This application aims to monitor user activities on HDFS via the hdfs audit
log. Once any abnormal user activity is detected, an alert is sent in several
seconds. The whole pipeline of this application is</p>
+<ul>
+<li>
+<p>Kafka ingest: this application consumes data from Kafka. In other words,
users have to stream the log into Kafka first. </p>
+</li>
+<li>
+<p>Data re-procesing, which includes raw log parser, ip zone joiner,
sensitivity information joiner. </p>
+</li>
+<li>
+<p>Kafka sink: parsed data will flows into Kafka again, which will be consumed
by the alert engine. </p>
+</li>
+<li>
+<p>Policy evaluation: the alert engine (hosted in Alert Engine app) evaluates
each data event to check if the data violate the user defined policy. An alert
is generated if the data matches the policy.</p>
+</li>
+</ul>
+<p><img alt="HDFSAUDITLOG" src="../include/images/hdfs_audit_log.png" /></p>
+<h2 id="setup-installation">Setup & Installation</h2>
+<ul>
+<li>
+<p>Choose a site to install this application. For example 'sandbox'</p>
+</li>
+<li>
+<p>Install "Hdfs Audit Log Monitor" app step by step</p>
+<p><img alt="Install Step 2" src="../include/images/hdfs_install_1.png" /></p>
+<p><img alt="Install Step 3" src="../include/images/hdfs_install_2.png" /></p>
+<p><img alt="Install Step 4" src="../include/images/hdfs_install_3.png" /></p>
+</li>
+</ul>
+<h2 id="how-to-collect-the-log">How to collect the log</h2>
+<p>To collect the raw audit log on namenode servers, a log collector is
needed. Users can choose any tools they like. There are some common solutions
available: <a
href="https://www.elastic.co/guide/en/logstash/current/getting-started-with-logstash.html";>logstash</a>,
<a
href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html";>filebeat</a>,
log4j appender, etcs. </p>
+<p>For detailed instruction, refer to: <a
href="../using-eagle/#how-to-stream-audit-log-into-kafka">How to stream audit
log into Kafka</a></p>
+<h2 id="sample-policies">Sample policies</h2>
+<h3 id="1-monitor-filefolder-operations">1. monitor file/folder operations</h3>
+<p>Delete a file/folder on HDFS. </p>
+<pre><code>from
HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX[str:contains(src,'/tmp/test/subtest')
and ((cmd=='rename' and str:contains(dst, '.Trash')) or cmd=='delete')] select
* group by user insert into hdfs_audit_log_enriched_stream_out
+</code></pre>
+
+<p>HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX is the input stream name, and
hdfs_audit_log_enriched_stream_out is the output stream name, the content
between [] is the monitoring conditions. <code>cmd</code>, <code>src</code> and
<code>dst</code> is the fields of hdfs audit logs.</p>
+<p><img alt="Policy 1" src="../include/images/hdfs_policy_1.png" /></p>
+<h3 id="2-classify-the-filefolder-on-hdfs">2. classify the file/folder on
HDFS</h3>
+<p>Users may want to mark some folders/files on HDFS as sensitive content. For
example, by marking '/sys/soj' as "SOJ", users can monitor any operations they
care about on 'sys/soj' and its subfolders/files.</p>
+<pre><code>from HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX[sensitivityType=='SOJ'
and cmd=='delete')] select * group by user insert into
hdfs_audit_log_enriched_stream_out
+</code></pre>
+
+<p>The example policy monitors the 'delete' operation on files/subfolders
under /sys/soj. </p>
+<h3 id="3-classify-the-ip-zone">3. Classify the IP Zone</h3>
+<p>In some cases, the ips are classified into different zones. For some zone,
it may have higher secrecy. Eagle providers ways to monitor user activities on
IP level. </p>
+<pre><code>from
HDFS_AUDIT_LOG_ENRICHED_STREAM_SANDBOX[securityZone=='SECURITY' and
cmd=='delete')] select * group by user insert into
hdfs_audit_log_enriched_stream_out
+</code></pre>
+
+<p>The example policy monitors the 'delete' operation on hosts in 'SECURITY'
zone. </p>
+<h2 id="questions-on-this-application">Questions on this application</h2>
+<hr />
+<h1 id="jmx-monitoring">JMX Monitoring</h1>
+<ul>
+<li>
+<p>Application "<strong>HADOOP_JMX_METRIC_MONITOR</strong>" provide embedded
collector script to ingest hadoop/hbase jmx metric as eagle stream and provide
ability to define alert policy and detect anomaly in real-time from metric.</p>
+<table>
+<thead>
+<tr>
+<th>Fields</th>
+<th></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong>Type</strong></td>
+<td><em>HADOOP_JMX_METRIC_MONITOR</em></td>
+</tr>
+<tr>
+<td><strong>Version</strong></td>
+<td><em>0.5.0-version</em></td>
+</tr>
+<tr>
+<td><strong>Description</strong></td>
+<td><em>Collect JMX Metric and monitor in real-time</em></td>
+</tr>
+<tr>
+<td><strong>Streams</strong></td>
+<td><em>HADOOP_JMX_METRIC_STREAM</em></td>
+</tr>
+<tr>
+<td><strong>Configuration</strong></td>
+<td><em>JMX Metric Kafka Topic (default:
hadoop_jmx_metric_{SITE_ID})</em><br/><br/><em>Kafka Broker List (default:
localhost:6667)</em></td>
+</tr>
+</tbody>
+</table>
+</li>
+</ul>
+<h2 id="setup-installation_1">Setup & Installation</h2>
+<ul>
+<li>
+<p>Make sure already setup a site (here use a demo site named "sandbox").</p>
+</li>
+<li>
+<p>Install "Hadoop JMX Monitor" app in eagle server.</p>
+<p><img alt="Install Step 2" src="../include/images/install_jmx_2.png" /></p>
+</li>
+<li>
+<p>Configure Application settings.</p>
+<p><img alt="Install Step 3" src="../include/images/install_jmx_3.png" /></p>
+</li>
+<li>
+<p>Ensure a kafka topic named hadoop_jmx_metric_{SITE_ID} (In current guide,
it should be hadoop_jmx_metric_sandbox)</p>
+</li>
+<li>
+<p>Setup metric collector for monitored Hadoop/HBase using
hadoop_jmx_collector and modify the configuration.</p>
+<ul>
+<li>
+<p>Collector scripts: <a
href="https://github.com/apache/incubator-eagle/tree/master/eagle-external/hadoop_jmx_collector";>hadoop_jmx_collector</a></p>
+</li>
+<li>
+<p>Rename config-sample.json to config.json: <a
href="https://github.com/apache/incubator-eagle/blob/master/eagle-external/hadoop_jmx_collector/config-sample.json";>config-sample.json</a></p>
+<pre><code>{
+ env: {
+ site: "sandbox",
+ name_node: {
+ hosts: [
+ "sandbox.hortonworks.com"
+ ],
+ port: 50070,
+ https: false
+ },
+ resource_manager: {
+ hosts: [
+ "sandbox.hortonworks.com"
+ ],
+ port: 50030,
+ https: false
+ }
+ },
+ inputs: [{
+ component: "namenode",
+ host: "server.eagle.apache.org",
+ port: "50070",
+ https: false,
+ kafka_topic: "nn_jmx_metric_sandbox"
+ }, {
+ component: "resourcemanager",
+ host: "server.eagle.apache.org",
+ port: "8088",
+ https: false,
+ kafka_topic: "rm_jmx_metric_sandbox"
+ }, {
+ component: "datanode",
+ host: "server.eagle.apache.org",
+ port: "50075",
+ https: false,
+ kafka_topic: "dn_jmx_metric_sandbox"
+ }],
+ filter: {
+ monitoring.group.selected: [
+ "hadoop",
+ "java.lang"
+ ]
+ },
+ output: {
+ kafka: {
+ brokerList: [
+ "localhost:9092"
+ ]
+ }
+ }
+}
+</code></pre>
+</li>
+</ul>
+</li>
+<li>
+<p>Click "Install" button then you will see the
HADOOP_JMX_METRIC_STREAM_{SITE_ID} in Streams.</p>
+<p><img alt="Install Step 6" src="../include/images/install_jmx_6.png" /></p>
+</li>
+</ul>
+<h2 id="define-jmx-alert-policy">Define JMX Alert Policy</h2>
+<ol>
+<li>
+<p>Go to "Define Policy".</p>
+</li>
+<li>
+<p>Select HADOOP_JMX_METRIC_MONITOR related streams.</p>
+</li>
+<li>
+<p>Define SQL-Like policy, for example</p>
+<pre><code>from HADOOP_JMX_METRIC_STREAM_SANDBOX[metric=="cpu.usage" and value
> 0.9]
+select site,host,component,value
+insert into HADOOP_CPU_USAGE_GT_90_ALERT;
+</code></pre>
+<p>As seen in below screenshot:</p>
+</li>
+</ol>
+<p><img alt="Define JMX Alert Policy"
src="../include/images/define_jmx_alert_policy.png" /></p>
+<h2 id="stream-schema">Stream Schema</h2>
+<ul>
+<li>
+<p>Schema</p>
+<table>
+<thead>
+<tr>
+<th>Stream Name</th>
+<th>Stream Schema</th>
+<th>Time Series</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>HADOOP_JMX_METRIC_MONITOR</td>
+<td><strong>host</strong>: STRING<br/><br/><strong>timestamp</strong>:
LONG<br/><br/><strong>metric</strong>:
STRING<br/><br/><strong>component</strong>:
STRING<br/><br/><strong>site</strong>: STRING<br/><br/><strong>value</strong>:
DOUBLE</td>
+<td>True</td>
+</tr>
+</tbody>
+</table>
+</li>
+</ul>
+<h2 id="metrics-list">Metrics List</h2>
+<ul>
+<li>Please refer to the <a href="../hadoop-jmx-metrics-list.txt">Hadoop JMX
Metrics List</a> and see which metrics you're interested in.</li>
+</ul>
+<hr />
+<h1 id="job-performance-monitoring">Job Performance Monitoring</h1>
+<h2 id="monitor-requirements_1">Monitor Requirements</h2>
+<ul>
+<li>Finished/Running Job Details</li>
+<li>Job Metrics(Job Counter/Statistics) Aggregation</li>
+<li>Alerts(Job failure/Job slow)</li>
+</ul>
+<h2 id="applications">Applications</h2>
+<ul>
+<li>
+<p>Application Table</p>
+<table>
+<thead>
+<tr>
+<th>application</th>
+<th>responsibility</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Map Reduce History Job Monitoring</td>
+<td>parse mr history job logs from hdfs</td>
+</tr>
+<tr>
+<td>Map Reduce Running Job Monitoring</td>
+<td>get mr running job details from resource manager</td>
+</tr>
+<tr>
+<td>Map Reduce Metrics Aggregation</td>
+<td>aggregate metrics generated by applications above</td>
+</tr>
+</tbody>
+</table>
+</li>
+</ul>
+<h2 id="data-ingestion-and-process">Data Ingestion And Process</h2>
+<ul>
+<li>
+<p>We build storm topology to fulfill requirements for each application.</p>
+<p><img alt="topology figures" src="../include/images/jpm.jpg" /></p>
+</li>
+<li>
+<p>Map Reduce History Job Monitoring (Figure 1)</p>
+<ul>
+<li><strong>Read Spout</strong><ul>
+<li>read/parse history job logs from HDFS and flush to eagle service(storage
is Hbase)</li>
+</ul>
+</li>
+<li><strong>Sink Bolt</strong><ul>
+<li>convert parsed jobs to streams and write to data sink</li>
+</ul>
+</li>
+</ul>
+</li>
+<li>Map Reduce Running Job Monitoring (Figure 2)<ul>
+<li><strong>Read Spout</strong><ul>
+<li>fetch running job list from resource manager and emit to Parse Bolt</li>
+</ul>
+</li>
+<li><strong>Parse Bolt</strong><ul>
+<li>for each running job, fetch job detail/job counter/job configure/tasks
from resource manager</li>
+</ul>
+</li>
+</ul>
+</li>
+<li>Map Reduce Metrics Aggregation (Figure 3)<ul>
+<li><strong>Divide Spout</strong><ul>
+<li>divide time period(need to be aggregated) to small pieces and emit to
Aggregate Bolt</li>
+</ul>
+</li>
+<li><strong>Aggregate Bolt</strong><ul>
+<li>aggregate metrics for given time period received from Divide Spout</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+<h2 id="setup-installation_2">Setup & Installation</h2>
+<ul>
+<li>
+<p>Make sure already setup a site (here use a demo site named "sandbox").</p>
+</li>
+<li>
+<p>Install "Map Reduce History Job" app in eagle server(Take this application
as an example).</p>
+</li>
+<li>
+<p>Configure Application settings</p>
+<p><img alt="application configures" src="../include/images/jpm_configure.png"
/></p>
+</li>
+<li>
+<p>Ensure a kafka topic named {SITE_ID}_map_reduce_failed_job (In current
guide, it should be sandbox_map_reduce_failed_job) will be created.</p>
+</li>
+<li>
+<p>Click "Install" button then you will see the
MAP_REDUCE_FAILED_JOB_STREAM_{SITE_ID} in Alert->Streams.
+ <img alt="application configures" src="../include/images/jpm_streams.png"
/>
+ This application will write stream data to kafka topic(created by last
step)</p>
+</li>
+</ul>
+<h2 id="integration-with-alert-engine">Integration With Alert Engine</h2>
+<p>In order to integrate applications with alert engine and send alerts,
follow below steps(Take Map Reduce History Job application as an example):</p>
+<ul>
+<li>
+<p><strong>define stream and configure data sink</strong></p>
+<ul>
+<li>define stream in resource/META-INF/providers/xxxProviders.xml
+For example, MAP_REDUCE_FAILED_JOB_STREAM_{SITE_ID}</li>
+<li>configure data sink
+For example, create kafka topic {SITE_ID}_map_reduce_failed_job</li>
+</ul>
+</li>
+<li>
+<p><strong>define policy</strong></p>
+</li>
+</ul>
+<p>For example, if you want to receive map reduce job failure alerts, you can
define policies (SiddhiQL) as the following:</p>
+<pre><code class="sql">from
map_reduce_failed_job_stream[site=="sandbox" and
currentState=="FAILED"]
+select site, queue, user, jobType, jobId, submissionTime, trackingUrl,
startTime, endTime
+group by jobId insert into map_reduce_failed_job_stream_out
+</code></pre>
+
+<p><img alt="define policy" src="../include/images/jpm_define_policy.png"
/></p>
+<ul>
+<li><strong>view alerts</strong></li>
+</ul>
+<p>You can view alerts in Alert->alerts page.</p>
+<h2 id="stream-schema_1">Stream Schema</h2>
+<p>All columns above are predefined in stream map_reduce_failed_job_stream
defined in</p>
+<pre><code>eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+</code></pre>
+<p>Then, enable the policy in web ui after it's created. Eagle will schedule
it automatically.</p>
+<hr />
+<h1 id="topology-health-check">Topology Health Check</h1>
+<ul>
+<li>
+<p>Application "TOPOLOGY HEALTH CHECK" aims to monior those servies with a
master-slave structured topology and provide metrics at host level.</p>
+<table>
+<thead>
+<tr>
+<th>Fields</th>
+<th></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong>Type</strong></td>
+<td><em>TOPOLOGY_HEALTH_CHECK</em></td>
+</tr>
+<tr>
+<td><strong>Version</strong></td>
+<td><em>0.5.0-version</em></td>
+</tr>
+<tr>
+<td><strong>Description</strong></td>
+<td><em>Collect MR,HBASE,HDFS node status and cluster ratio</em></td>
+</tr>
+<tr>
+<td><strong>Streams</strong></td>
+<td><em>TOPOLOGY_HEALTH_CHECK_STREAM</em></td>
+</tr>
+<tr>
+<td><strong>Configuration</strong></td>
+<td><em>Topology Health Check Topic (default:
topology_health_check)</em><br/><br/><em>Kafka Broker List (default:
sandobox.hortonworks.com:6667)</em></td>
+</tr>
+</tbody>
+</table>
+</li>
+</ul>
+<h2 id="setup-installation_3">Setup & Installation</h2>
+<ul>
+<li>
+<p>Make sure already setup a site (here use a demo site named "sandbox").</p>
+</li>
+<li>
+<p>Install "Topology Health Check" app in eagle server.</p>
+<p><img alt="Health Check Installation"
src="../include/images/health_check_installation.png" /></p>
+</li>
+<li>
+<p>Configure Application settings.</p>
+<p><img alt="Health Check Settings"
src="../include/images/health_check_settings.png" /></p>
+</li>
+<li>
+<p>Ensure the existence of a kafka topic named topology_health_check (In
current guide, it should be topology_health_check).</p>
+</li>
+<li>
+<p>Click "Install" button then you will see the
TOPOLOGY_HEALTH_CHECK_STREAM_{SITE_ID} on "Streams" page (Streams could be
navigated in left-nav).</p>
+<p><img alt="Health Check Stream"
src="../include/images/health_check_stream.png" /></p>
+</li>
+</ul>
+<h2 id="define-health-check-alert-policy">Define Health Check Alert Policy</h2>
+<ul>
+<li>
+<p>Go to "Define Policy".</p>
+</li>
+<li>
+<p>Select TOPOLOGY_HEALTH_CHECK related streams.</p>
+</li>
+<li>
+<p>Define SQL-Like policy, for example</p>
+<pre><code>from TOPOLOGY_HEALTH_CHECK_STREAM_SANDBOX[status=='dead'] select *
insert into topology_health_check_stream_out;
+</code></pre>
+<p><img alt="Health Check Policy"
src="../include/images/health_check_policy.png" /></p>
+</li>
+</ul>
+<hr />
+<h1 id="hadoop-queue-monitoring">Hadoop Queue Monitoring</h1>
+<ul>
+<li>
+<p>This application collects metrics of Resource Manager in the following
aspects:</p>
+<ul>
+<li>
+<p>Scheduler Info of the cluster:
http://{RM_HTTP_ADDRESS}:{PORT}/ws/v1/cluster/scheduler</p>
+</li>
+<li>
+<p>Applications of the cluster:
http://{RM_HTTP_ADDRESS}:{PORT}/ws/v1/cluster/apps</p>
+</li>
+<li>
+<p>Overall metrics of the cluster:
http://{RM_HTTP_ADDRESS}:{PORT}/ws/v1/cluster/metrics</p>
+<pre><code>by version 0.5-incubating, mainly focusing at metrics
+ - `appsPending`
+ - `allocatedMB`
+ - `totalMB`
+ - `availableMB`
+ - `reservedMB`
+ - `allocatedVirtualCores`.
+</code></pre>
+</li>
+</ul>
+</li>
+</ul>
+<h2 id="setup-installation_4">Setup & Installation</h2>
+<ul>
+<li>
+<p>Make sure already setup a site (here use a demo site named "sandbox").</p>
+</li>
+<li>
+<p>From left-nav list, navigate to application managing page by
"<strong>Integration</strong>" > "<strong>Sites</strong>", and hit link
"<strong>sandbox</strong>" on right.</p>
+<p><img alt="Navigate to app mgmt"
src="../include/images/hadoop_queue_monitor_1.png" /></p>
+</li>
+<li>
+<p>Install "Hadoop Queue Monitor" by clicking "install" button of the
application.</p>
+<p><img alt="Install Hadoop Queue Monitor App"
src="../include/images/hadoop_queue_monitor_2.png" /></p>
+</li>
+<li>
+<p>In the pop-up layout, select running mode as <code>Local</code> or
<code>Cluster</code>.</p>
+<p><img alt="Select Running Mode"
src="../include/images/hadoop_queue_monitor_3.png" /></p>
+</li>
+<li>
+<p>Set the target jar of eagle's topology assembly that has existed in eagle
server, indicating the absolute path ot it. As in the following screenshot:</p>
+<p><img alt="Set Jar Path" src="../include/images/hadoop_queue_monitor_4.png"
/></p>
+</li>
+<li>
+<p>Set Resource Manager endpoint urls field, separate values with comma if
there are more than 1 url (e.g. a secondary node for HA).</p>
+<p><img alt="Set RM Endpoint"
src="../include/images/hadoop_queue_monitor_5.png" /></p>
+</li>
+<li>
+<p>Set fields "<strong>Storm Worker Number</strong>", "<strong>Parallel Tasks
Per Bolt</strong>", and "<strong>Fetching Metric Interval in Seconds</strong>",
or leave them as default if they fit your needs.</p>
+<p><img alt="Set Advanced Fields"
src="../include/images/hadoop_queue_monitor_6.png" /></p>
+</li>
+<li>
+<p>Finally, hit "<strong>Install</strong>" button to complete it.</p>
+</li>
+</ul>
+<h2 id="use-of-the-application">Use of the application</h2>
+<ul>
+<li>
+<p>There is no need to define policies for this applicatoin to work, it could
be integrated with "<strong>Job Performance Monitoring Web</strong>"
application and consequently seen on cluster dashboard, as long as the latter
application is installed too. See an exmple in the following screenshot:</p>
+<p><img alt="In Dashboard" src="../include/images/hadoop_queue_monitor_7.png"
/></p>
+</li>
+</ul>
+
+ </div>
+ </div>
+ <footer>
+
+ <div class="rst-footer-buttons" role="navigation" aria-label="footer
navigation">
+
+ <a href="../developing-application/" class="btn btn-neutral
float-right" title="Developing Application">Next <span class="icon
icon-circle-arrow-right"></span></a>
+
+
+ <a href="../using-eagle/" class="btn btn-neutral" title="Using
Eagle"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+
+ </div>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <!-- Copyright etc -->
+
+ </div>
+
+ Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a
href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a
href="https://readthedocs.org";>Read the Docs</a>.
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+<div class="rst-versions" role="note" style="cursor: pointer">
+ <span class="rst-current-version" data-toggle="rst-current-version">
+
+ <a href="https://github.com/apache/eagle/tree/master/docs";
class="icon icon-github" style="float: left; color: #fcfcfc"> GitHub</a>
+
+
+ <span><a href="../using-eagle/" style="color: #fcfcfc;">«
Previous</a></span>
+
+
+ <span style="margin-left: 15px"><a href="../developing-application/"
style="color: #fcfcfc">Next »</a></span>
+
+ </span>
+</div>
+
+</body>
+</html>
Propchange: eagle/site/docs/latest/applications/index.html
------------------------------------------------------------------------------
svn:eol-style = native
Added: eagle/site/docs/latest/css/highlight.css
URL:
http://svn.apache.org/viewvc/eagle/site/docs/latest/css/highlight.css?rev=1789966&view=auto
==============================================================================
--- eagle/site/docs/latest/css/highlight.css (added)
+++ eagle/site/docs/latest/css/highlight.css Mon Apr 3 11:35:50 2017
@@ -0,0 +1,125 @@
+/*
+This is the GitHub theme for highlight.js
+
+github.com style (c) Vasily Polovnyov <[email protected]>
+
+*/
+
+.hljs {
+ display: block;
+ overflow-x: auto;
+ padding: 0.5em;
+ color: #333;
+ -webkit-text-size-adjust: none;
+}
+
+.hljs-comment,
+.diff .hljs-header,
+.hljs-javadoc {
+ color: #998;
+ font-style: italic;
+}
+
+.hljs-keyword,
+.css .rule .hljs-keyword,
+.hljs-winutils,
+.nginx .hljs-title,
+.hljs-subst,
+.hljs-request,
+.hljs-status {
+ color: #333;
+ font-weight: bold;
+}
+
+.hljs-number,
+.hljs-hexcolor,
+.ruby .hljs-constant {
+ color: #008080;
+}
+
+.hljs-string,
+.hljs-tag .hljs-value,
+.hljs-phpdoc,
+.hljs-dartdoc,
+.tex .hljs-formula {
+ color: #d14;
+}
+
+.hljs-title,
+.hljs-id,
+.scss .hljs-preprocessor {
+ color: #900;
+ font-weight: bold;
+}
+
+.hljs-list .hljs-keyword,
+.hljs-subst {
+ font-weight: normal;
+}
+
+.hljs-class .hljs-title,
+.hljs-type,
+.vhdl .hljs-literal,
+.tex .hljs-command {
+ color: #458;
+ font-weight: bold;
+}
+
+.hljs-tag,
+.hljs-tag .hljs-title,
+.hljs-rule .hljs-property,
+.django .hljs-tag .hljs-keyword {
+ color: #000080;
+ font-weight: normal;
+}
+
+.hljs-attribute,
+.hljs-variable,
+.lisp .hljs-body,
+.hljs-name {
+ color: #008080;
+}
+
+.hljs-regexp {
+ color: #009926;
+}
+
+.hljs-symbol,
+.ruby .hljs-symbol .hljs-string,
+.lisp .hljs-keyword,
+.clojure .hljs-keyword,
+.scheme .hljs-keyword,
+.tex .hljs-special,
+.hljs-prompt {
+ color: #990073;
+}
+
+.hljs-built_in {
+ color: #0086b3;
+}
+
+.hljs-preprocessor,
+.hljs-pragma,
+.hljs-pi,
+.hljs-doctype,
+.hljs-shebang,
+.hljs-cdata {
+ color: #999;
+ font-weight: bold;
+}
+
+.hljs-deletion {
+ background: #fdd;
+}
+
+.hljs-addition {
+ background: #dfd;
+}
+
+.diff .hljs-change {
+ background: #0086b3;
+}
+
+.hljs-chunk {
+ color: #aaa;
+}
Propchange: eagle/site/docs/latest/css/highlight.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: eagle/site/docs/latest/developing-application/index.html
URL:
http://svn.apache.org/viewvc/eagle/site/docs/latest/developing-application/index.html?rev=1789966&view=auto
==============================================================================
--- eagle/site/docs/latest/developing-application/index.html (added)
+++ eagle/site/docs/latest/developing-application/index.html Mon Apr 3
11:35:50 2017
@@ -0,0 +1,857 @@
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+
+
+ <title>Developing Application - Apache Eagle Documentation</title>
+
+
+ <link rel="shortcut icon" href="../include/images/favicon.png">
+
+
+
+ <link
href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700'
rel='stylesheet' type='text/css'>
+
+ <link rel="stylesheet" href="../css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
+ <link rel="stylesheet" href="../css/highlight.css">
+
+
+ <script>
+ // Current page data
+ var mkdocs_page_name = "Developing Application";
+ var mkdocs_page_input_path = "developing-application.md";
+ var mkdocs_page_url = "/developing-application/";
+ </script>
+
+ <script src="../js/jquery-2.1.1.min.js"></script>
+ <script src="../js/modernizr-2.8.3.min.js"></script>
+ <script type="text/javascript" src="../js/highlight.pack.js"></script>
+ <script src="../js/theme.js"></script>
+
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+ <div class="wy-side-nav-search">
+ <a href=".." class="icon icon-home"> Apache Eagle Documentation</a>
+ <div role="search">
+ <form id ="rtd-search-form" class="wy-form" action="../search.html"
method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ </form>
+</div>
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation"
aria-label="main navigation">
+ <ul class="current">
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="..">Home</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../getting-started/">Getting Started</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../using-eagle/">Using Eagle</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../applications/">Applications</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 current">
+ <a class="current" href="./">Developing Application</a>
+
+ <ul>
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#introduction">Introduction</a></li>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#quick-start">Quick Start</a></li>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#create-application">Create Application</a></li>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#test-application">Test Application</a></li>
+
+
+
+
+
+
+ <li class="toctree-l3" onclick="select(this)"><a
+
+
+
+ href="#management-rest-api">Management & REST API</a></li>
+
+ <ul>
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#applicationproviderspiloader">ApplicationProviderSPILoader</a></li>
+
+
+
+
+
+
+ <li class="toctree-l4" onclick="select(this)"><a
+
+
+ style="padding-left: 4em;"
+
+ href="#application-rest-api">Application REST API</a></li>
+
+
+
+
+ </ul>
+
+
+
+
+ </ul>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../underlying-design/">Underlying Design</a>
+
+ </li>
+<li>
+
+ <li><script>
+ function select(elem) {
+ var selectedClassName = "selected";
+ $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+ $(elem).addClass(selectedClassName);
+ }
+</script>
+
+<!--
+ this macro is to fetch the first child element that has url and return the
url as the default one of the section
+ arguemnt:
+ ni: nav_item, corresponding to navigation item configured in
mkdocs.yml's "pages"
+ -->
+
+
+<!--
+ this macro generates the style for indentation on sub tocs in side nav
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!--
+ this macro shows toc items including nested tocs, toc nesting level would
depending on arguments described below
+ argument:
+ current_level: the level number of current toc item
+ max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+ <li class="toctree-l1 ">
+ <a class="" href="../reference/">Reference</a>
+
+ </li>
+<li>
+
+ </ul>
+ </div>
+
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="..">Apache Eagle Documentation</a>
+ </nav>
+
+
+ <div class="wy-nav-content">
+ <div class="rst-content">
+ <div role="navigation" aria-label="breadcrumbs navigation">
+ <ul class="wy-breadcrumbs">
+ <li><a href="..">Docs</a> »</li>
+
+
+
+ <li>Developing Application</li>
+ <li class="wy-breadcrumbs-aside">
+
+
+ <a href="https://github.com/apache/eagle/tree/master/docs";
class="icon icon-github"> Edit on GitHub</a>
+
+
+ </li>
+ </ul>
+ <hr/>
+</div>
+ <div role="main">
+ <div class="section">
+
+ <h1 id="introduction">Introduction</h1>
+<p><a href="../applications">Applications</a> in Eagle include process
component and view component. Process component normally refers to storm
topology or spark stream job which processes incoming data, while viewing
component normally refers to GUI hosted in Eagle UI. </p>
+<p><a href="../getting-started/#eagle-framework">Application Framework</a>
targets at solving the problem of managing application lifecycle and presenting
uniform views to end users.</p>
+<p>Eagle application framework is designed for end-to-end lifecycle of
applications including:</p>
+<ul>
+<li>
+<p><strong>Development</strong>: application development and framework
development</p>
+</li>
+<li>
+<p><strong>Testing</strong>.</p>
+</li>
+<li>
+<p><strong>Installation</strong>: package management with SPI/Providers.xml</p>
+</li>
+<li>
+<p><strong>Management</strong>: manage applications through REST API</p>
+</li>
+</ul>
+<hr />
+<h1 id="quick-start">Quick Start</h1>
+<ul>
+<li>
+<p>Fork and clone eagle source code repository using GIT.</p>
+<pre><code>git clone https://github.com/apache/incubator-eagle.git
+</code></pre>
+</li>
+<li>
+<p>Run Eagle Server : execute âorg.apache.eagle.server.ServerDebugâ under
eagle-server in IDE or with maven command line.</p>
+<pre><code>org.apache.eagle.server.ServerDebug
+</code></pre>
+</li>
+<li>
+<p>Access current available applications through API.</p>
+<pre><code>curl -XGET http://localhost:9090/rest/apps/providers
+</code></pre>
+</li>
+<li>
+<p>Create Site through API.</p>
+<pre><code>curl -H "Content-Type: application/json" -X POST
http://localhost:9090/rest/sites --data '{
+ "siteId":"test_site",
+ "siteName":"Test Site",
+ "description":"This is a sample site for test",
+ "context":{
+ "type":"FAKE_CLUSTER",
+ "url":"http://localhost:9090";,
+ "version":"2.6.4",
+ "additional_attr":"Some information about the face cluster site"
+ }
+}'
+</code></pre>
+</li>
+<li>
+<p>Install Application through API.</p>
+<pre><code>curl -H "Content-Type: application/json" -X POST
http://localhost:9090/rest/apps/install --data '{
+ "siteId":"test_site",
+ "appType":"EXAMPLE_APPLICATION",
+ "mode":"LOCAL"
+}'
+</code></pre>
+</li>
+<li>
+<p>Start Application (uuid means installed application uuid).</p>
+<pre><code>curl -H "Content-Type: application/json" âX POST
http://localhost:9090/rest/apps/start --data '{
+ "uuid":"9acf6792-60e8-46ea-93a6-160fb6ef0b3f"
+}'
+</code></pre>
+</li>
+<li>
+<p>Stop Application (uuid means installed application uuid).</p>
+<pre><code>curl -XPOST http://localhost:9090/rest/apps/stop '{
+ "uuid": "9acf6792-60e8-46ea-93a6-160fb6ef0b3f"
+}'
+</code></pre>
+</li>
+<li>
+<p>Uninstall Application (uuid means installed application uuid).</p>
+<pre><code>curl -XDELETE http://localhost:9090/rest/apps/uninstall '{
+ "uuid": "9acf6792-60e8-46ea-93a6-160fb6ef0b3f"
+}'
+</code></pre>
+</li>
+</ul>
+<hr />
+<h1 id="create-application">Create Application</h1>
+<p>Each application should be developed under independent modules (including
backend code and front-end code).</p>
+<p>Here is a typical code structure of a new application as following:</p>
+<pre><code>eagle-app-example/
+âââ pom.xml
+âââ src
+â âââ main
+â â âââ java
+â â â âââ org
+â â â âââ apache
+â â â âââ eagle
+â â â âââ app
+â â â âââ example
+â â â âââ ExampleApplicationProvider.java
+â â â âââ ExampleStormApplication.java
+â â âââ resources
+â â â âââ META-INF
+â â â âââ providers
+â â â â âââ
org.apache.eagle.app.example.ExampleApplicationProvider.xml
+â â â âââ services
+â â â âââ
org.apache.eagle.app.spi.ApplicationProvider
+â â âââ webapp
+â â âââ app
+â â â âââ apps
+â â â âââ example
+â â â âââ index.html
+â â âââ package.json
+â âââ test
+â âââ java
+â â âââ org
+â â âââ apache
+â â âââ eagle
+â â âââ app
+â â âââ example
+â â â âââ
ExampleApplicationProviderTest.java
+â â â âââ ExampleApplicationTest.java
+â âââ resources
+â âââ application.conf
+</code></pre>
+
+<p><strong>Eagle Example Application</strong> - <a
href="https://github.com/haoch/incubator-eagle/tree/master/eagle-examples/eagle-app-example";>eagle-app-example</a></p>
+<p><strong>Description</strong> - A typical eagle application is mainly
consisted of:</p>
+<ul>
+<li>
+<p><strong>Application</strong>: define core execution process logic
inheriting from org.apache.eagle.app.Application, which is also implemented
ApplicationTool to support Application to run as standalone process like a
Storm topology through command line.</p>
+</li>
+<li>
+<p><strong>ApplicationProvider</strong>: the interface to package application
with descriptor metadata, also used as application SPI to dynamically load new
application types.</p>
+</li>
+<li>
+<p><strong>META-INF/providers/${APP_PROVIDER_CLASS_NAME}.xml</strong>: support
to easily describe applicationâs descriptor with declarative XML like:</p>
+<pre><code><application>
+ <type>EXAMPLE_APPLICATION</type>
+ <name>Example Monitoring Application</name>
+ <version>0.5.0-incubating</version>
+ <configuration>
+ <property>
+ <name>message</name>
+ <displayName>Message</displayName>
+ <value>Hello, example application!</value>
+ <description>Just an sample configuration
property</description>
+ </property>
+ </configuration>
+ <streams>
+ <stream>
+ <streamId>SAMPLE_STREAM_1</streamId>
+ <description>Sample output stream #1</description>
+ <validate>true</validate>
+ <timeseries>true</timeseries>
+ <columns>
+ <column>
+ <name>metric</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>source</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>value</name>
+ <type>double</type>
+ <defaultValue>0.0</defaultValue>
+ </column>
+ </columns>
+ </stream>
+ <stream>
+ <streamId>SAMPLE_STREAM_2</streamId>
+ <description>Sample output stream #2</description>
+ <validate>true</validate>
+ <timeseries>true</timeseries>
+ <columns>
+ <column>
+ <name>metric</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>source</name>
+ <type>string</type>
+ </column>
+ <column>
+ <name>value</name>
+ <type>double</type>
+ <defaultValue>0.0</defaultValue>
+ </column>
+ </columns>
+ </stream>
+ </streams>
+</application>
+</code></pre>
+</li>
+<li>
+<p><strong>META-INF/services/org.apache.eagle.app.spi.ApplicationProvider</strong>:
support to dynamically scan and load extensible application provider using
java service provider.</p>
+</li>
+<li>
+<p><strong>webapp/app/apps/${APP_TYPE}</strong>: if the application has web
portal, then it could add more web code under this directory and make sure
building as following in pom.xml</p>
+<pre><code><build>
+ <resources>
+ <resource>
+ <directory>src/main/webapp/app</directory>
+ <targetPath>assets/</targetPath>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+</build>
+</code></pre>
+</li>
+</ul>
+<hr />
+<h1 id="test-application">Test Application</h1>
+<ul>
+<li>
+<p>Extend <strong>org.apache.eagle.app.test.ApplicationTestBase</strong> and
initialize injector context.</p>
+</li>
+<li>
+<p>Access shared service with <strong>@Inject</strong>.</p>
+</li>
+<li>
+<p>Test application lifecycle with related web resource.</p>
+<pre><code>@Inject private SiteResource siteResource;
+@Inject private ApplicationResource applicationResource;
+
+// Create local site
+SiteEntity siteEntity = new SiteEntity();
+siteEntity.setSiteId("test_site");
+siteEntity.setSiteName("Test Site");
+siteEntity.setDescription("Test Site for ExampleApplicationProviderTest");
+siteResource.createSite(siteEntity);
+Assert.assertNotNull(siteEntity.getUuid());
+
+ApplicationOperations.InstallOperation installOperation = new
ApplicationOperations.InstallOperation(
+ "test_site",
+ "EXAMPLE_APPLICATION",
+ ApplicationEntity.Mode.LOCAL);
+installOperation.setConfiguration(getConf());
+// Install application
+ApplicationEntity applicationEntity = applicationResource
+ .installApplication(installOperation)
+ .getData();
+// Start application
+applicationResource.startApplication(new
ApplicationOperations.StartOperation(applicationEntity.getUuid()));
+// Stop application
+applicationResource.stopApplication(new
ApplicationOperations.StopOperation(applicationEntity.getUuid()));
+// Uninstall application
+applicationResource.uninstallApplication(
+ new ApplicationOperations.UninstallOperation(applicationEntity.getUuid()));
+try {
+ applicationResource.getApplicationEntityByUUID(applicationEntity.getUuid());
+ Assert.fail("Application instance (UUID: " + applicationEntity.getUuid() +
") should have been uninstalled");
+} catch (Exception ex) {
+ // Expected exception
+}
+</code></pre>
+</li>
+</ul>
+<hr />
+<h1 id="management-rest-api">Management & REST API</h1>
+<h2 id="applicationproviderspiloader">ApplicationProviderSPILoader</h2>
+<p>Default behavior - automatically loading from class path using SPI:</p>
+<ul>
+<li>
+<p>By default, eagle will load application providers from current class
loader.</p>
+</li>
+<li>
+<p>If application.provider.dir defined, it will load from external jarsâ
class loader.</p>
+</li>
+</ul>
+<h2 id="application-rest-api">Application REST API</h2>
+<ul>
+<li>
+<p>API Table</p>
+<table>
+<thead>
+<tr>
+<th>Type</th>
+<th>Uri + Class</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong>DELETE</strong></td>
+<td>/rest/sites (org.apache.eagle.metadata.resource.SiteResource)</td>
+</tr>
+<tr>
+<td><strong>DELETE</strong></td>
+<td>/rest/sites/{siteId} (org.apache.eagle.metadata.resource.SiteResource)</td>
+</tr>
+<tr>
+<td><strong>GET</strong></td>
+<td>/rest/sites (org.apache.eagle.metadata.resource.SiteResource)</td>
+</tr>
+<tr>
+<td><strong>GET</strong></td>
+<td>/rest/sites/{siteId} (org.apache.eagle.metadata.resource.SiteResource)</td>
+</tr>
+<tr>
+<td><strong>POST</strong></td>
+<td>/rest/sites (org.apache.eagle.metadata.resource.SiteResource)</td>
+</tr>
+<tr>
+<td><strong>PUT</strong></td>
+<td>/rest/sites (org.apache.eagle.metadata.resource.SiteResource)</td>
+</tr>
+<tr>
+<td><strong>PUT</strong></td>
+<td>/rest/sites/{siteId} (org.apache.eagle.metadata.resource.SiteResource)</td>
+</tr>
+<tr>
+<td><strong>DELETE</strong></td>
+<td>/rest/apps/uninstall
(org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>GET</strong></td>
+<td>/rest/apps (org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>GET</strong></td>
+<td>/rest/apps/providers
(org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>GET</strong></td>
+<td>/rest/apps/providers/{type}
(org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>GET</strong></td>
+<td>/rest/apps/{appUuid}
(org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>POST</strong></td>
+<td>/rest/apps/install (org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>POST</strong></td>
+<td>/rest/apps/start (org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>POST</strong></td>
+<td>/rest/apps/stop (org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+<tr>
+<td><strong>PUT</strong></td>
+<td>/rest/apps/providers/reload
(org.apache.eagle.app.resource.ApplicationResource)</td>
+</tr>
+</tbody>
+</table>
+</li>
+</ul>
+
+ </div>
+ </div>
+ <footer>
+
+ <div class="rst-footer-buttons" role="navigation" aria-label="footer
navigation">
+
+ <a href="../underlying-design/" class="btn btn-neutral float-right"
title="Underlying Design">Next <span class="icon
icon-circle-arrow-right"></span></a>
+
+
+ <a href="../applications/" class="btn btn-neutral"
title="Applications"><span class="icon icon-circle-arrow-left"></span>
Previous</a>
+
+ </div>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <!-- Copyright etc -->
+
+ </div>
+
+ Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a
href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a
href="https://readthedocs.org";>Read the Docs</a>.
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+<div class="rst-versions" role="note" style="cursor: pointer">
+ <span class="rst-current-version" data-toggle="rst-current-version">
+
+ <a href="https://github.com/apache/eagle/tree/master/docs";
class="icon icon-github" style="float: left; color: #fcfcfc"> GitHub</a>
+
+
+ <span><a href="../applications/" style="color: #fcfcfc;">«
Previous</a></span>
+
+
+ <span style="margin-left: 15px"><a href="../underlying-design/"
style="color: #fcfcfc">Next »</a></span>
+
+ </span>
+</div>
+
+</body>
+</html>
Propchange: eagle/site/docs/latest/developing-application/index.html
------------------------------------------------------------------------------
svn:eol-style = native
