http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/tutorial/policy.html ---------------------------------------------------------------------- diff --git a/_site/docs/tutorial/policy.html b/_site/docs/tutorial/policy.html new file mode 100644 index 0000000..e732eb0 --- /dev/null +++ b/_site/docs/tutorial/policy.html @@ -0,0 +1,299 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Policy Tutorial</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/tutorial/policy.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/tutorial/policy.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/tutorial/policy.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/tutorial/policy.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/tutorial/policy.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/tutorial/policy.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/tutorial/policy.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/tutorial/policy.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/tutorial/policy.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/tutorial/policy.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:[email protected]"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Policy Tutorial</h1> + <p>Apache Eagle (called Eagle in the following) currently supports to customize policies for data sources for each site:</p> + +<ul> + <li>HDFS Audit Log</li> + <li>Hive<sup id="fnref:HIVE"><a href="#fn:HIVE" class="footnote">1</a></sup> Query Log</li> +</ul> + +<blockquote> + <p>NOTICE: policies are classified by sites. Please select the site first when there are multiple ones.</p> +</blockquote> + +<h3 id="how-to-define-hdfs-policy">How to define HDFS Policy?</h3> +<p>In this example we will go through the steps for creating the following HDFS policy.</p> + +<blockquote> + <p>Example Policy: Create a policy to alert when a user is trying to delete a file with sensitive data</p> +</blockquote> + +<ul> + <li> + <p><strong>Step 1</strong>: Select Source as HDFS and Stream as HDFS Audit Log</p> + + <p><img src="/images/docs/hdfs-policy1.png" alt="HDFS Policies" /></p> + </li> + <li> + <p><strong>Step 2</strong>: Eagle supports a variety of properties for match critera where users can set different values. Eagle also supports window functions to extend policies with time functions.</p> + + <pre><code>command = delete +(Eagle currently supports the following commands open, delete, copy, append, copy from local, get, move, mkdir, create, list, change permissions) + +source = /tmp/private +(Eagle supports wildcarding for property values for example /tmp/*) +</code></pre> + + <p><img src="/images/docs/hdfs-policy2.png" alt="HDFS Policies" /></p> + </li> + <li> + <p><strong>Step 3</strong>: Name your policy and select de-duplication options if you need to avoid getting duplicate alerts within a particular time window. You have an option to configure email notifications for the alerts.</p> + + <p><img src="/images/docs/hdfs-policy3.png" alt="HDFS Policies" /></p> + </li> +</ul> + +<h3 id="how-to-define-hive-policy">How to define HIVE Policy?</h3> +<p>In this example we will go thru the steps for creating the following Hive policy.</p> + +<blockquote> + <p>Example Policy: Create a policy to alert when a user is trying to select PHONE_NUMBER from a hive table with sensitive data</p> +</blockquote> + +<ul> + <li> + <p><strong>Step 1</strong>: Select Source as Hive and Stream as Hive Query Log</p> + + <p><img src="/images/docs/hive-policy1.png" alt="Hive Policies" /></p> + </li> + <li> + <p><strong>Step 2</strong>: Eagle support a variety of properties for match critera where users can set different values. Eagle also supports window functions to extend policies with time functions.</p> + + <pre><code>command = Select +(Eagle currently supports the following commands DDL statements Create, Drop, Alter, Truncate, Show) + +sensitivity type = PHONE_NUMBER +(Eagle supports classifying data in Hive with different sensitivity types. Users can use these sensitivity types to create policies) +</code></pre> + + <p><img src="/images/docs/hive-policy2.png" alt="Hive Policies" /></p> + </li> + <li> + <p><strong>Step 3</strong>: Name your policy and select de-duplication options if you need to avoid getting duplicate alerts within a particular time window. You have an option to configure email notifications for the alerts.</p> + + <p><img src="/images/docs/hive-policy3.png" alt="Hive Policies" /></p> + </li> +</ul> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:HIVE"> + <p><em>All mentions of âhiveâ on this page represent Apache Hive.</em> <a href="#fnref:HIVE" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/tutorial/site-0.3.0.html ---------------------------------------------------------------------- diff --git a/_site/docs/tutorial/site-0.3.0.html b/_site/docs/tutorial/site-0.3.0.html new file mode 100644 index 0000000..a542128 --- /dev/null +++ b/_site/docs/tutorial/site-0.3.0.html @@ -0,0 +1,341 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Site Management</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/tutorial/site-0.3.0.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:[email protected]"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Site Management</h1> + <p>Apache Eagle (called Eagle in the following) identifies different Hadoop<sup id="fnref:HADOOP"><a href="#fn:HADOOP" class="footnote">1</a></sup> environments as different sites, such as sandbox, datacenter1, datacenter2. In each site, a user can add different data sources as the monitoring targets. For each data source, a connection configuration is required.</p> + +<h4 id="step-1-add-site">Step 1: Add Site</h4> + +<p>The following is an example which creates a new site âDemoâ, and add two data sources as its monitoring targets. +<img src="/images/docs/new-site.png" alt="setup a site" /></p> + +<h4 id="step-2-add-configuration">Step 2: Add Configuration</h4> + +<p>After creating a new site, we need to edit the configuration to connect the cluster. +Here we give configuration examples for HDFS, HBASE, and Hive.</p> + +<ul> + <li> + <p>HDFS</p> + + <p><img src="/images/docs/hdfs-setup-0.3.0.png" alt="hdfs setup" /></p> + + <ul> + <li> + <p>Base case</p> + + <p>You may configure the default path for Hadoop clients to connect remote hdfs namenode.</p> + + <pre><code> {"fs.defaultFS":"hdfs://sandbox.hortonworks.com:8020"} +</code></pre> + </li> + <li> + <p>HA case</p> + + <p>Basically, you point your fs.defaultFS at your nameservice and let the client know how its configured (the backing namenodes) and how to fail over between them under the HA mode</p> + + <pre><code> {"fs.defaultFS":"hdfs://nameservice1", + "dfs.nameservices": "nameservice1", + "dfs.ha.namenodes.nameservice1":"namenode1,namenode2", + "dfs.namenode.rpc-address.nameservice1.namenode1": "hadoopnamenode01:8020", + "dfs.namenode.rpc-address.nameservice1.namenode2": "hadoopnamenode02:8020", + "dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + } +</code></pre> + </li> + <li> + <p>Kerberos-secured cluster</p> + + <p>For Kerberos-secured cluster, you need to get a keytab file and the principal from your admin, and configure âeagle.keytab.fileâ and âeagle.kerberos.principalâ to authenticate its access.</p> + + <pre><code> { "eagle.keytab.file":"/EAGLE-HOME/.keytab/eagle.keytab", + "eagle.kerberos.principal":"[email protected]" + } +</code></pre> + + <p>If there is an exception about âinvalid server principal nameâ, you may need to check the DNS resolver, or the data transfer , such as âdfs.encrypt.data.transferâ, âdfs.encrypt.data.transfer.algorithmâ, âdfs.trustedchannel.resolver.classâ, âdfs.datatransfer.client.encryptâ.</p> + </li> + </ul> + </li> + <li>Hive<sup id="fnref:HIVE"><a href="#fn:HIVE" class="footnote">2</a></sup> + <ul> + <li> + <p>Basic</p> + + <pre><code> { + "accessType": "metastoredb_jdbc", + "password": "hive", + "user": "hive", + "jdbcDriverClassName": "com.mysql.jdbc.Driver", + "jdbcUrl": "jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true" + } +</code></pre> + </li> + </ul> + </li> + <li> + <p>HBase<sup id="fnref:HBASE"><a href="#fn:HBASE" class="footnote">3</a></sup></p> + + <ul> + <li> + <p>Basic case</p> + + <p>You need to sett âhbase.zookeeper.quorumâ:âlocalhostâ property and âhbase.zookeeper.property.clientPortâ property.</p> + + <pre><code> { + "hbase.zookeeper.property.clientPort":"2181", + "hbase.zookeeper.quorum":"localhost" + } +</code></pre> + </li> + <li> + <p>Kerberos-secured cluster</p> + + <p>According to your environment, you can add or remove some of the following properties. Here is the reference.</p> + + <pre><code> { + "hbase.zookeeper.property.clientPort":"2181", + "hbase.zookeeper.quorum":"localhost", + "hbase.security.authentication":"kerberos", + "hbase.master.kerberos.principal":"hadoop/[email protected]", + "zookeeper.znode.parent":"/hbase", + "eagle.keytab.file":"/EAGLE-HOME/.keytab/eagle.keytab", + "eagle.kerberos.principal":"[email protected]" + } +</code></pre> + </li> + </ul> + </li> +</ul> + +<h4 id="step-3-checking-the-connection">Step 3: Checking the connection</h4> +<p>After the configuration is ready, you can go to <a href="/docs/tutorial/classification-0.3.0.html">classification page</a> and browse the data. If the configuration is correct, data will be ready in a few seconds.</p> + +<p>Any questions on the Kerberos configuration in Eagle, please first check <a href="/docs/FAQ.html">FAQ</a></p> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:HADOOP"> + <p><em>All mentions of âhadoopâ on this page represent Apache Hadoop.</em> <a href="#fnref:HADOOP" class="reversefootnote">↩</a></p> + </li> + <li id="fn:HIVE"> + <p><em>Apache Hive.</em> <a href="#fnref:HIVE" class="reversefootnote">↩</a></p> + </li> + <li id="fn:HBASE"> + <p><em>Apache HBase.</em> <a href="#fnref:HBASE" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/tutorial/site.html ---------------------------------------------------------------------- diff --git a/_site/docs/tutorial/site.html b/_site/docs/tutorial/site.html new file mode 100644 index 0000000..7cfdcb5 --- /dev/null +++ b/_site/docs/tutorial/site.html @@ -0,0 +1,251 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Site Management</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/tutorial/site.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/tutorial/site.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/tutorial/site.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/tutorial/site.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/tutorial/site.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/tutorial/site.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/tutorial/site.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/tutorial/site.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/tutorial/site.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/tutorial/site.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:[email protected]"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Site Management</h1> + <p>Guide to site management for versions since <strong>Apache Eagle 0.4.0-incubating</strong>.</p> + +<p><a href="/docs/tutorial/site-0.3.0.html">For Apache Eagle 0.3.0-incubating, see <em>here</em>.</a></p> + +<p>The main content in this page have two parts</p> + +<ul> + <li>How to add a new site</li> + <li>How to add an new application</li> +</ul> + +<h4 id="how-to-add-a-new-site">How to add a new site</h4> + +<p>Eagle names the cluster to monitor as a site. Users (as an admin user) can manage their sites on Eagle UI. The following example is to add a new site âtestâ.</p> + +<p><img src="/images/docs/new-site.png" alt="setup a site" /></p> + +<h4 id="how-to-add-an-new-application">How to add an new application</h4> + +<p>Once create a site, users can choose the data sources they want to monitoring by adding different applications. For example, HdfsAuditLog application monitors the hdfs audit log. Currently, Eagle supports</p> + +<ul> + <li><a href="/docs/hdfs-data-activity-monitoring.html">HDFS Data Activity Monitoring</a></li> + <li><a href="/docs/hive-query-activity-monitoring.html">HIVE Query Activity Monitoring</a></li> + <li><a href="/docs/hbase-data-activity-monitoring.html">HBASE Data Activity Monitoring</a></li> + <li><a href="/docs/mapr-integration.html">MapR FS Data Activity Monitoring</a></li> + <li><a href="/docs/jmx-metric-monitoring.html">Hadoop JMX Metrics Monitoring</a></li> +</ul> + +<p>Same as adding a site, application management also locates at the admin management page. To add an application, two steps are needed</p> + +<h4 id="step-1-go-to-site-tab-and-select-the-target-site">step 1: go to site tab, and select the target site</h4> + +<p><img src="/images/docs/new-application1.png" alt="new application step1" /></p> + +<h4 id="step-2-click-the-applications-you-want-on-the-right-side-and-save-the-page">step 2: click the applications you want on the right side and save the page</h4> + +<p><img src="/images/docs/new-application2.png" alt="new application step1" /></p> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/tutorial/topologymanagement.html ---------------------------------------------------------------------- diff --git a/_site/docs/tutorial/topologymanagement.html b/_site/docs/tutorial/topologymanagement.html new file mode 100644 index 0000000..2a95034 --- /dev/null +++ b/_site/docs/tutorial/topologymanagement.html @@ -0,0 +1,412 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Topology Management</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/tutorial/topologymanagement.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/tutorial/topologymanagement.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:[email protected]"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Topology Management</h1> + <p><em>Since Apache Eagle 0.4.0-incubating. Apache Eagle will be called Eagle in the following.</em></p> + +<blockquote> + <p>Application manager aims to manage applications on EAGLE UI. Users can easily start/start topologies remotely or locally without any shell commands. At the same, it should be capable to sync the latest status of topologies on the execution platform (e.g., Storm<sup id="fnref:STORM"><a href="#fn:STORM" class="footnote">1</a></sup> cluster).</p> +</blockquote> + +<p>This tutorial will go through all parts of application manager and then give an example to use it.</p> + +<h3 id="design">Design</h3> +<p>Application manager consists of a daemon scheduler and an execution module. The scheduler periodically loads user operations(start/stop) from database, and the execution module executes these operations. For more details, please refer to <a href="https://cwiki.apache.org/confluence/display/EAG/Application+Management";>here</a>.</p> + +<h3 id="configurations">Configurations</h3> +<p>The configuration file <code>eagle-scheduler.conf</code> defines scheduler parameters, execution platform settings and parts of default topology configuration.</p> + +<ul> + <li> + <p><strong>Scheduler properties</strong></p> + + <style> + table, td, th { + border-collapse: collapse; + border: 1px solid gray; + padding: 10px; + } + </style> + + <table> + <thead> + <tr> + <th>Property Name</th> + <th style="text-align: center">Default</th> + <th>Description</th> + </tr> + </thead> + <tbody> + <tr> + <td>appCommandLoaderEnabled</td> + <td style="text-align: center">false</td> + <td>topology management is enabled or not</td> + </tr> + <tr> + <td>appCommandLoaderIntervalSecs</td> + <td style="text-align: center">1</td> + <td>defines the interval of the scheduler loads commands</td> + </tr> + <tr> + <td>appHealthCheckIntervalSecs</td> + <td style="text-align: center">5</td> + <td>define the interval of topology health checking, which syncs the topology execution status from storm cluster to Eagle</td> + </tr> + </tbody> + </table> + </li> + <li> + <p><strong>Execution platform properties</strong></p> + + <table> + <thead> + <tr> + <th>Property Name</th> + <th style="text-align: left">Default</th> + <th>Description</th> + </tr> + </thead> + <tbody> + <tr> + <td>envContextConfig.env</td> + <td style="text-align: left">storm</td> + <td>execution environment, only storm is supported</td> + </tr> + <tr> + <td>envContextConfig.url</td> + <td style="text-align: left">http://sandbox.hortonworks.com:8744</td> + <td>storm ui</td> + </tr> + <tr> + <td>envContextConfig.nimbusHost</td> + <td style="text-align: left">sandbox.hortonworks.com</td> + <td>storm nimbus host</td> + </tr> + <tr> + <td>envContextConfig.nimbusThriftPort</td> + <td style="text-align: left">6627</td> + <td>storm nimbus thrift port</td> + </tr> + <tr> + <td>envContextConfig.jarFile</td> + <td style="text-align: left">TODO</td> + <td>storm fat jar path</td> + </tr> + </tbody> + </table> + </li> + <li> + <p><strong>Topology default properties</strong></p> + + <p>Some default topology properties are defined here.</p> + </li> +</ul> + +<h3 id="playbook">Playbook</h3> + +<ol> + <li> + <p>Editing eagle-scheduler.conf, and start Eagle service</p> + + <pre><code> # enable application manager + appCommandLoaderEnabled = true + + # provide jar path + envContextConfig.jarFile = + + # storm nimbus + envContextConfig.url = "http://sandbox.hortonworks.com:8744"; + envContextConfig.nimbusHost = "sandbox.hortonworks.com" +</code></pre> + + <p>For more configurations, please back to <a href="/docs/configuration.html">Application Configuration</a>. <br /> + After the configuration is ready, start Eagle service <code>bin/eagle-service.sh start</code>.</p> + </li> + <li> + <p>Go to admin page +<img src="/images/appManager/admin-page.png" alt="admin-page" /> +<img src="/images/appManager/topology-monitor.png" alt="topology-monitor" /></p> + </li> + <li>Go to management page, and create a topology description. There are three required fields + <ul> + <li>name: topology name</li> + <li>type: topology type [CLASS, DYNAMIC]</li> + <li>execution entry: either the class which implements interface TopologyExecutable or eagle <a href="https://github.com/apache/eagle/blob/master/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf";>DSL</a> based topology definition +<img src="/images/appManager/topology-description.png" alt="topology-description" /></li> + </ul> + </li> + <li> + <p>Back to monitoring page, and choose the site/application to deploy the topology +<img src="/images/appManager/topology-execution.png" alt="topology-execution" /></p> + </li> + <li> + <p>Go to site page, and add topology configurations.</p> + + <p><strong>NOTICE</strong> topology configurations defined here are REQUIRED an extra prefix <code>.app</code></p> + + <p>Blow are some example configurations for [site=sandbox, applicatoin=hbaseSecurityLog].</p> + + <pre><code> classification.hbase.zookeeper.property.clientPort=2181 + classification.hbase.zookeeper.quorum=sandbox.hortonworks.com + + app.envContextConfig.env=storm + app.envContextConfig.mode=cluster + + app.dataSourceConfig.topic=sandbox_hbase_security_log + app.dataSourceConfig.zkConnection=sandbox.hortonworks.com:2181 + app.dataSourceConfig.zkConnectionTimeoutMS=15000 + app.dataSourceConfig.brokerZkPath=/brokers + app.dataSourceConfig.fetchSize=1048586 + app.dataSourceConfig.transactionZKServers=sandbox.hortonworks.com + app.dataSourceConfig.transactionZKPort=2181 + app.dataSourceConfig.transactionZKRoot=/consumers + app.dataSourceConfig.consumerGroupId=eagle.hbasesecurity.consumer + app.dataSourceConfig.transactionStateUpdateMS=2000 + app.dataSourceConfig.deserializerClass=org.apache.eagle.security.hbase.parse.HbaseAuditLogKafkaDeserializer + + app.eagleProps.site=sandbox + app.eagleProps.application=hbaseSecurityLog + app.eagleProps.dataJoinPollIntervalSec=30 + app.eagleProps.mailHost=some.mail.server + app.eagleProps.mailSmtpPort=25 + app.eagleProps.mailDebug=true + app.eagleProps.eagleService.host=localhost + app.eagleProps.eagleService.port=9099 + app.eagleProps.eagleService.username=admin + app.eagleProps.eagleService.password=secret +</code></pre> + + <p><img src="/images/appManager/topology-configuration-1.png" alt="topology-configuration-1" /> +<img src="/images/appManager/topology-configuration-2.png" alt="topology-configuration-2" /></p> + </li> + <li> + <p>Go to monitoring page, and start topologies +<img src="/images/appManager/start-topology-1.png" alt="start-topology-1" /> +<img src="/images/appManager/start-topology-2.png" alt="start-topology-2" /></p> + </li> + <li>stop topologies on monitoring page +<img src="/images/appManager/stop-topology-1.png" alt="stop-topology-1" /> +<img src="/images/appManager/stop-topology-2.png" alt="stop-topology-2" /> +<img src="/images/appManager/stop-topology-3.png" alt="stop-topology-3" /></li> +</ol> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:STORM"> + <p><em>All mentions of âstormâ on this page represent Apache Storm.</em> <a href="#fnref:STORM" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/tutorial/userprofile.html ---------------------------------------------------------------------- diff --git a/_site/docs/tutorial/userprofile.html b/_site/docs/tutorial/userprofile.html new file mode 100644 index 0000000..fbf55e6 --- /dev/null +++ b/_site/docs/tutorial/userprofile.html @@ -0,0 +1,297 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - User Profile Tutorial</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/tutorial/userprofile.html" id="">Introduction</a></li> + + <li class="sidenavli "><a href="/docs/usecases.html" data-permalink="/docs/tutorial/userprofile.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/tutorial/userprofile.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/tutorial/userprofile.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/tutorial/userprofile.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/tutorial/userprofile.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/tutorial/userprofile.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/tutorial/userprofile.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/tutorial/userprofile.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/tutorial/userprofile.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:[email protected]"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">User Profile Tutorial</h1> + <p>This document will introduce how to start the online processing on user profiles. Assume Apache Eagle has been installed and <a href="http://sandbox.hortonworks.com:9099/eagle-service";>Eagle service</a> +is started.</p> + +<h3 id="user-profile-offline-training">User Profile Offline Training</h3> + +<ul> + <li> + <p><strong>Step 1</strong>: Start Apache Spark if not started +<img src="/images/docs/start-spark.png" alt="Start Spark" /></p> + </li> + <li> + <p><strong>Step 2</strong>: start offline scheduler</p> + + <ul> + <li> + <p>Option 1: command line</p> + + <pre><code>$ cd <eagle-home>/bin +$ bin/eagle-userprofile-scheduler.sh --site sandbox start +</code></pre> + </li> + <li> + <p>Option 2: start via Apache Ambari + <img src="/images/docs/offline-userprofile.png" alt="Click "ops"" /></p> + </li> + </ul> + </li> + <li> + <p><strong>Step 3</strong>: generate a model</p> + + <p><img src="/images/docs/userProfile1.png" alt="Click "ops"" /> + <img src="/images/docs/userProfile2.png" alt="Click "Update Now"" /> + <img src="/images/docs/userProfile3.png" alt="Click "Confirm"" /> + <img src="/images/docs/userProfile4.png" alt="Check" /></p> + </li> +</ul> + +<h3 id="user-profile-online-detection">User Profile Online Detection</h3> + +<p>Two options to start the topology are provided.</p> + +<ul> + <li> + <p><strong>Option 1</strong>: command line</p> + + <p>submit userProfiles topology if itâs not on <a href="http://sandbox.hortonworks.com:8744";>topology UI</a></p> + + <pre><code>$ bin/eagle-topology.sh --main org.apache.eagle.security.userprofile.UserProfileDetectionMain --config conf/sandbox-userprofile-topology.conf start +</code></pre> + </li> + <li> + <p><strong>Option 2</strong>: Apache Ambari</p> + + <p><img src="/images/docs/online-userprofile.png" alt="Online userProfiles" /></p> + </li> +</ul> + +<h3 id="evaluate-user-profile-in-sandbox">Evaluate User Profile in Sandbox</h3> + +<ol> + <li>Prepare sample data for ML training and validation sample data + <ul> + <li>a. Download following sample data to be used for training</li> + <li><a href="/data/user1.hdfs-audit.2015-10-11-00.txt"><code>user1.hdfs-audit.2015-10-11-00.txt</code></a></li> + <li><a href="/data/user1.hdfs-audit.2015-10-11-01.txt"><code>user1.hdfs-audit.2015-10-11-01.txt</code></a></li> + <li>b. Downlaod <a href="/data/userprofile-validate.txt"><code>userprofile-validate.txt</code></a>file which contains data points that you can try to test the models</li> + </ul> + </li> + <li>Copy the files (downloaded in the previous step) into a location in sandbox +For example: <code>/usr/hdp/current/eagle/lib/userprofile/data/</code></li> + <li>Modify <code><Eagle-home>/conf/sandbox-userprofile-scheduler.conf </code> +update <code>training-audit-path</code> to set to the path for training data sample (the path you used for Step 1.a) +update detection-audit-path to set to the path for validation (the path you used for Step 1.b)</li> + <li>Run ML training program from eagle UI</li> + <li> + <p>Produce Apache Kafka data using the contents from validate file (Step 1.b) +Run the command (assuming the eagle configuration uses Kafka topic <code>sandbox_hdfs_audit_log</code>)</p> + + <pre><code> ./kafka-console-producer.sh --broker-list sandbox.hortonworks.com:6667 --topic sandbox_hdfs_audit_log +</code></pre> + </li> + <li>Paste few lines of data from file validate onto kafka-console-producer +Check <a href="http://localhost:9099/eagle-service/#/dam/alertList";>http://localhost:9099/eagle-service/#/dam/alertList</a> for generated alerts</li> +</ol> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/eagle/blob/d6fed677/_site/docs/usecases.html ---------------------------------------------------------------------- diff --git a/_site/docs/usecases.html b/_site/docs/usecases.html new file mode 100644 index 0000000..056c0ce --- /dev/null +++ b/_site/docs/usecases.html @@ -0,0 +1,288 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + + <title>Eagle - Use Cases</title> + <meta name="description" content="Eagle - Analyze Big Data Platforms for Security and Performance"> + + <meta name="keywords" content="Eagle, Hadoop, Security, Real Time"> + <meta name="author" content="eBay Inc."> + + <meta charset="utf-8"> + <meta name="viewport" content="initial-scale=1"> + + <link rel="stylesheet" href="/css/animate.css"> + <link rel="stylesheet" href="/css/bootstrap.min.css"> + + <link rel="stylesheet" href="/css/font-awesome.min.css"> + + <link rel="stylesheet" href="/css/misc.css"> + <link rel="stylesheet" href="/css/style.css"> + <link rel="stylesheet" href="/css/styles.css"> + <link rel="stylesheet" href="/css/main.css"> + <link rel="alternate" type="application/rss+xml" title="Eagle" href="http://goeagle.io/feed.xml"; /> + <link rel="shortcut icon" href="/images/favicon.png"> + + <!-- Baidu Analytics Tracking--> + <script> + var _hmt = _hmt || []; + (function() { + var hm = document.createElement("script"); + hm.src = "//hm.baidu.com/hm.js?fedc55df2ea52777a679192e8f849ece"; + var s = document.getElementsByTagName("script")[0]; + s.parentNode.insertBefore(hm, s); + })(); + </script> + + <!-- Google Analytics Tracking --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + ga('create', 'UA-68929805-1', 'auto'); + ga('send', 'pageview'); + </script> +</head> +<body> +<!-- header start --> +<div id="home_page"> + <div class="topbar"> + <div class="container"> + <div class="row" > + <nav class="navbar navbar-default"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> + <a class="navbar-brand" href="/"><img src="/images/logo2.png" height="44px" style="margin-top:-7px"></a> </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <!-- <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right" id="top-menu"> + <li><a class="menu" href="/#home_page">HOME</a></li> + <li><a class="menu" href="/docs/">DOCS</a></li> + <li><a class="menu" href="/#about_page">ABOUT</a></li> + <li><a class="menu" href="/#diagram_page">ARCHITECTURE</a></li> + <li><a class="menu" href="/#modules_page">MODULES</a></li> + <li><a class="menu" href="/#usecase_page">USE CASES</a></li> + <li> + </li> + </ul> --> + </div> + </div> + <!-- /.container-fluid --> + </nav> + </div> + </div> + </div> + <div class="headerimage"> + <div class="flexslider"> + <ul class="slides"> + <li><img src="/images/slider/4.jpg" alt="Slide 1"></li> + </ul> + </div> + </div> + <div class="particles" style="height:40%"> </div><!---particles--> + <div class="slider-caption" style="top:80px;"> + <div class="homewrapper"> + <div class="hometitle"> + <a href="/"> + <img src="/images/feather.png" height="80px"> + </a> + </div> + <div class="hometext"> + <h2>Analyze Big Data Platforms For Security and Performance</h2> + <div class="social-buttons"> + <a href="https://github.com/apache/eagle";><i class="fa fa-github"></i></a> + <a href="http://twitter.com/TheApacheEagle";><i class="fa fa-twitter"></i></a> + <a href="https://www.facebook.com/TheApacheEagle/";><i class="fa fa-facebook"></i></a> + <a href="#"><i class="fa fa-weixin"></i></a> + <!-- <a href="https://www.weibo.com/ApacheEagle/";><i class="fa fa-weibo"></i></a> --> + </div> + </div> + </div> + </div> +</div> +<!-- header end --> +<div class="container-fluid page-content"> + <div class="row"> + <div class="col-md-10 col-md-offset-1"> + <!-- sidebar --> + <div class="col-xs-6 col-sm-3" id="sidebar" role="navigation"> + <ul class="nav" id="adminnav"> + + <li class="heading">Getting Started</li> + + <li class="sidenavli "><a href="/docs/index.html" data-permalink="/docs/usecases.html" id="">Introduction</a></li> + + <li class="sidenavli current"><a href="/docs/usecases.html" data-permalink="/docs/usecases.html" id="">Use Cases</a></li> + + <li class="sidenavli "><a href="/docs/terminology.html" data-permalink="/docs/usecases.html" id="">Terminology</a></li> + + <li class="sidenavli "><a href="/docs/ecosystem.html" data-permalink="/docs/usecases.html" id="">Ecosystem</a></li> + + <li class="sidenavli "><a href="/docs/community.html" data-permalink="/docs/usecases.html" id="">Community</a></li> + + <li class="sidenavli "><a href="/docs/FAQ.html" data-permalink="/docs/usecases.html" id="">FAQ</a></li> + + <li class="divider"></li> + + <li class="heading">Documentations</li> + + <li class="sidenavli "><a href="/docs/latest/" data-permalink="/docs/usecases.html" id="">Latest version (v0.5.0)</a></li> + + <li class="divider"></li> + + <li class="heading">Download</li> + + <li class="sidenavli "><a href="/docs/download-latest.html" data-permalink="/docs/usecases.html" id="">Latest version (v0.5.0)</a></li> + + <li class="sidenavli "><a href="/docs/download.html" data-permalink="/docs/usecases.html" id="">Archived</a></li> + + <li class="divider"></li> + + <li class="heading">Supplement</li> + + <li class="sidenavli "><a href="/docs/security.html" data-permalink="/docs/usecases.html" id="">Security</a></li> + + <li class="divider"></li> + + <li class="sidenavli"> + <a href="mailto:[email protected]"; target="_blank">Need Help?</a> + </li> + </ul> + </div> + <div class="col-xs-6 col-sm-9 page-main-content" style="margin-left: -15px" id="loadcontent"> + <h1 class="page-header" style="margin-top: 0px">Use Cases</h1> + <h3 id="data-activity-monitoring">Data Activity Monitoring</h3> + +<ul> + <li> + <p>Data activity represents how user explores data provided by big data platforms. Analyzing data activity and alerting for insecure access are fundamental requirements for securing enterprise data. As data volume is increasing exponentially with Hadoop<sup id="fnref:HADOOP"><a href="#fn:HADOOP" class="footnote">1</a></sup>, Hive<sup id="fnref:HIVE"><a href="#fn:HIVE" class="footnote">2</a></sup>, Spark<sup id="fnref:SPARK"><a href="#fn:SPARK" class="footnote">3</a></sup> technology, understanding data activities for every user becomes extremely hard, let alone to alert for a single malicious event in real time among petabytes streaming data per day.</p> + </li> + <li> + <p>Securing enterprise data starts from understanding data activities for every user. Apache Eagle (called Eagle in the following) has integrated with many popular big data platforms e.g. Hadoop, Hive, Spark, Cassandra<sup id="fnref:CASSANDRA"><a href="#fn:CASSANDRA" class="footnote">4</a></sup> etc. With Eagle user can browse data hierarchy, mark sensitive data and then create comprehensive policy to alert for insecure data access.</p> + </li> +</ul> + +<h3 id="job-performance-analytics">Job Performance Analytics</h3> + +<ul> + <li> + <p>Running map/reduce job is the most popular way people use to analyze data in Hadoop system. Analyzing job performance and providing tuning suggestions are critical for Hadoop system stability, job SLA and resource usage etc.</p> + </li> + <li> + <p>Eagle analyzes job performance with two complementing approaches. First Eagle periodically takes snapshots for all running jobs with YARN API, secondly Eagle continuously reads job lifecycle events immediately after the job is completed. With the two approaches, Eagle can analyze single jobâs trend, data skew problem, failure reasons etc. More interestingly, Eagle can analyze whole Hadoop clusterâs performance by taking into account all jobs.</p> + </li> +</ul> + +<h3 id="node-anomaly-detection">Node Anomaly Detection</h3> + +<ul> + <li> + <p>One of practical benefits from analyzing map/reduce job is node anomaly detection. Big data platform like Hadoop may involve thousands of nodes for supporting multi-tenant jobs. One bad node may not crash whole cluster thanks to failure tolerance design, but may affect specific jobs and cause a lot of rescheduling, job delay and hurt stability of whole cluster etc.</p> + </li> + <li> + <p>Eagle developed out-of-the-box algorithm to compare task failure ratio for each node in a large cluster. If one node continues to fail running tasks, it may have potential issues, sometimes one of its disks is full or fails etc. In a nutshell, if one node behaves very differently from all other nodes within one large cluster, this node is anomalous and we should take action.</p> + </li> +</ul> + +<h3 id="cluster-performance-analytics">Cluster Performance Analytics</h3> + +<ul> + <li> + <p>It is critical to understand why a cluster performs bad. Is that because of some crazy jobs recently onboarded, or huge amount of tiny files, or namenode performance degrading?</p> + </li> + <li> + <p>Eagle in realtime calculates resource usage per minute out of individual jobs, e.g. CPU, memory, HDFS IO bytes, HDFS IO numOps etc. and also collects namenode JMX metrics. Correlating them together will easily help system adminstrator find root cause for cluster slowness.</p> + </li> +</ul> + +<h3 id="cluster-resource-usage-trend">Cluster Resource Usage Trend</h3> + +<ul> + <li> + <p>YARN manages resource allocation through queue in a large Hadoop cluster. Cluster resource usage is exactly reflected by overall queue usage.</p> + </li> + <li> + <p>Eagle in realtime collects queue statistics and provide insights of cluster resource usage.</p> + </li> +</ul> + +<hr /> + +<h4 id="footnotes"><em>Footnotes</em></h4> + +<div class="footnotes"> + <ol> + <li id="fn:HADOOP"> + <p><em>All mentions of âhadoopâ on this page represent Apache Hadoop.</em> <a href="#fnref:HADOOP" class="reversefootnote">↩</a></p> + </li> + <li id="fn:HIVE"> + <p><em>All mentions of âhiveâ on this page represent Apache Hive.</em> <a href="#fnref:HIVE" class="reversefootnote">↩</a></p> + </li> + <li id="fn:SPARK"> + <p><em>All mentions of âsparkâ on this page represent Apache Spark.</em> <a href="#fnref:SPARK" class="reversefootnote">↩</a></p> + </li> + <li id="fn:CASSANDRA"> + <p><em>Apache Cassandra.</em> <a href="#fnref:CASSANDRA" class="reversefootnote">↩</a></p> + </li> + </ol> +</div> + + </div><!--end of loadcontent--> + </div> + <!--end of centered content--> + </div> +</div> +<!--end of container--> + +<!-- footer start --> +<div class="footerwrapper"> + <div class="container"> + <div class="row"> + <div class="col-md-12"><div style="margin-left:auto; margin-right:auto; text-align:center;font-size: 12px"> +<div> +</div> +<div> +<a href="http://www.apache.org";> +<img id="asf-logo" alt="Apache Software Foundation" src="/images/apache-logo-small.gif"> +</a> +</div> +<div> +Copyright © 2015 <a href="http://www.apache.org";>The Apache Software Foundation</a>, Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. +</div> +<div> +Apache Eagle, Eagle, Apache Hadoop, Hadoop, Apache HBase, HBase, Apache Hive, Hive, Apache Ambari, Ambari, Apache Spark, Spark, Apache Kafka, Kafka, Apache Storm, Storm, Apache Maven, Maven, Apache Tomcat, Tomcat, Apache Derby, Derby, Apache Cassandra, Cassandra, Apache ZooKeeper, ZooKeeper, Apache, the Apache feather logo, and the Apache project logo are trademarks of The Apache Software Foundation. +</div> + </div></div> + </div> + </div> +</div> +<!-- footer end --> + +<!-- JavaScripts --> +<script src="/js/jquery-1.11.1.min.js"></script> +<script src="/js/jquery.singlePageNav.js"></script> +<script src="/js/jquery.flexslider.js"></script> +<script src="/js/modernizr.min.js"></script> +<script src="/js/svg.js"></script> +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> + +<script> + /************** FlexSlider *********************/ + $('.flexslider').flexslider({ + animation: "fade", + directionNav: false + }); +</script> +</body> +</html>
