Added: zeppelin/site/docs/0.7.2/install/yarn_install.html URL: http://svn.apache.org/viewvc/zeppelin/site/docs/0.7.2/install/yarn_install.html?rev=1798613&view=auto ============================================================================== --- zeppelin/site/docs/0.7.2/install/yarn_install.html (added) +++ zeppelin/site/docs/0.7.2/install/yarn_install.html Tue Jun 13 16:55:32 2017 @@ -0,0 +1,387 @@ + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Apache Zeppelin 0.7.2 Documentation: Install Zeppelin to connect with existing YARN cluster</title> + <meta name="description" content="This page describes how to pre-configure a bare metal node, configure Apache Zeppelin and connect it to existing YARN cluster running Hortonworks flavour of Hadoop."> + <meta name="author" content="The Apache Software Foundation"> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"> + + <!-- Le styles --> + <link href="/docs/0.7.2/assets/themes/zeppelin/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/docs/0.7.2/assets/themes/zeppelin/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/docs/0.7.2/assets/themes/zeppelin/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- Js --> + <script src="https://code.jquery.com/jquery-1.10.2.min.js";></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/bootstrap/js/bootstrap.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/docs.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/anchor.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/toc.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/lunr.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/search.js"></script> + + <!-- atom & rss feed --> + <link href="/docs/0.7.2/atom.xml" type="application/atom+xml" rel="alternate" title="Sitewide ATOM Feed"> + <link href="/docs/0.7.2/rss.xml" type="application/rss+xml" rel="alternate" title="Sitewide RSS Feed"> + </head> + + <body> + + <div id="menu" class="navbar navbar-inverse navbar-fixed-top" role="navigation"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-brand"> + <a class="navbar-brand-main" href="http://zeppelin.apache.org";> + <img src="/assets/themes/zeppelin/img/zeppelin_logo.png" width="50" alt="I'm zeppelin"> + <span style="vertical-align:middle">Zeppelin</span> + </a> + <a class="navbar-brand-version" href="/docs/0.7.2"> + <span><small>0.7.2</small></span> + </a> + </div> + </div> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Quick Start <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.7.2/index.html">What is Apache Zeppelin ?</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Getting Started</b><span></li> + <li><a href="/docs/0.7.2/install/install.html">Install</a></li> + <li><a href="/docs/0.7.2/install/configuration.html">Configuration</a></li> + <li><a href="/docs/0.7.2/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li> + <li><a href="/docs/0.7.2/quickstart/tutorial.html">Tutorial</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Basic Feature Guide</b><span></li> + <li><a href="/docs/0.7.2/manual/dynamicform.html">Dynamic Form</a></li> + <li><a href="/docs/0.7.2/manual/publish.html">Publish your Paragraph</a></li> + <li><a href="/docs/0.7.2/manual/notebookashomepage.html">Customize Zeppelin Homepage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>More</b><span></li> + <li><a href="/docs/0.7.2/install/upgrade.html">Upgrade Zeppelin Version</a></li> + <li><a href="/docs/0.7.2/install/build.html">Build from source</a></li> + <li><a href="/docs/0.7.2/quickstart/install_with_flink_and_spark_cluster.html">Install Zeppelin with Flink and Spark Clusters Tutorial</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li><a href="/docs/0.7.2/manual/interpreters.html">Overview</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Usage</b><span></li> + <li><a href="/docs/0.7.2/manual/interpreterinstallation.html">Interpreter Installation</a></li> + <!--<li><a href="/docs/0.7.2/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>--> + <li><a href="/docs/0.7.2/manual/dependencymanagement.html">Interpreter Dependency Management</a></li> + <li><a href="/docs/0.7.2/manual/userimpersonation.html">Interpreter User Impersonation</a></li> + <li><a href="/docs/0.7.2/manual/interpreterexechooks.html">Interpreter Execution Hooks (Experimental)</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Available Interpreters</b><span></li> + <li><a href="/docs/0.7.2/interpreter/alluxio.html">Alluxio</a></li> + <li><a href="/docs/0.7.2/interpreter/beam.html">Beam</a></li> + <li><a href="/docs/0.7.2/interpreter/bigquery.html">BigQuery</a></li> + <li><a href="/docs/0.7.2/interpreter/cassandra.html">Cassandra</a></li> + <li><a href="/docs/0.7.2/interpreter/elasticsearch.html">Elasticsearch</a></li> + <li><a href="/docs/0.7.2/interpreter/flink.html">Flink</a></li> + <li><a href="/docs/0.7.2/interpreter/geode.html">Geode</a></li> + <li><a href="/docs/0.7.2/interpreter/hbase.html">HBase</a></li> + <li><a href="/docs/0.7.2/interpreter/hdfs.html">HDFS</a></li> + <li><a href="/docs/0.7.2/interpreter/hive.html">Hive</a></li> + <li><a href="/docs/0.7.2/interpreter/ignite.html">Ignite</a></li> + <li><a href="/docs/0.7.2/interpreter/jdbc.html">JDBC</a></li> + <li><a href="/docs/0.7.2/interpreter/kylin.html">Kylin</a></li> + <li><a href="/docs/0.7.2/interpreter/lens.html">Lens</a></li> + <li><a href="/docs/0.7.2/interpreter/livy.html">Livy</a></li> + <li><a href="/docs/0.7.2/interpreter/markdown.html">Markdown</a></li> + <li><a href="/docs/0.7.2/interpreter/pig.html">Pig</a></li> + <li><a href="/docs/0.7.2/interpreter/python.html">Python</a></li> + <li><a href="/docs/0.7.2/interpreter/postgresql.html">Postgresql, HAWQ</a></li> + <li><a href="/docs/0.7.2/interpreter/r.html">R</a></li> + <li><a href="/docs/0.7.2/interpreter/scalding.html">Scalding</a></li> + <li><a href="/docs/0.7.2/interpreter/scio.html">Scio</a></li> + <li><a href="/docs/0.7.2/interpreter/shell.html">Shell</a></li> + <li><a href="/docs/0.7.2/interpreter/spark.html">Spark</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Display System <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li class="title"><span><b>Basic Display System</b><span></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#text">Text</a></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#html">Html</a></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#table">Table</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Angular API</b><span></li> + <li><a href="/docs/0.7.2/displaysystem/back-end-angular.html">Angular (backend API)</a></li> + <li><a href="/docs/0.7.2/displaysystem/front-end-angular.html">Angular (frontend API)</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">More<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu" style="right: 0; left: auto;"> + <li class="title"><span><b>Notebook Storage</b><span></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#storage-in-zeppelinhub">ZeppelinHub Storage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>REST API</b><span></li> + <li><a href="/docs/0.7.2/rest-api/rest-interpreter.html">Interpreter API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-notebook.html">Notebook API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-notebookRepo.html">Notebook Repository API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-configuration.html">Configuration API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-credential.html">Credential API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-helium.html">Helium API</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Security</b><span></li> + <li><a href="/docs/0.7.2/security/shiroauthentication.html">Shiro Authentication</a></li> + <li><a href="/docs/0.7.2/security/notebook_authorization.html">Notebook Authorization</a></li> + <li><a href="/docs/0.7.2/security/datasource_authorization.html">Data Source Authorization</a></li> + <li><a href="/docs/0.7.2/security/helium_authorization.html">Helium Authorization</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Advanced</b><span></li> + <li><a href="/docs/0.7.2/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li> + <li><a href="/docs/0.7.2/install/cdh.html">Zeppelin on CDH</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Contibute</b><span></li> + <li><a href="/docs/0.7.2/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li> + <li><a href="/docs/0.7.2/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization (Experimental)</a></li> + <li><a href="/docs/0.7.2/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li> + <li><a href="/docs/0.7.2/development/howtocontribute.html">How to contribute (code)</a></li> + <li><a href="/docs/0.7.2/development/howtocontributewebsite.html">How to contribute (website)</a></li> + </ul> + </li> + <li> + <a href="/docs/0.7.2/search.html" class="nav-search-link"> + <span class="fa fa-search nav-search-icon"></span> + </a> + </li> + </ul> + </nav><!--/.navbar-collapse --> + </div> + </div> + + + + <div class="content"> + +<!--<div class="hero-unit Install Zeppelin to connect with existing YARN cluster"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<h2>Introduction</h2> + +<p>This page describes how to pre-configure a bare metal node, configure Zeppelin and connect it to existing YARN cluster running Hortonworks flavour of Hadoop. It also describes steps to configure Spark interpreter of Zeppelin.</p> + +<h2>Prepare Node</h2> + +<h3>Zeppelin user (Optional)</h3> + +<p>This step is optional, however its nice to run Zeppelin under its own user. In case you do not like to use Zeppelin (hope not) the user could be deleted along with all the packages that were installed for Zeppelin, Zeppelin binary itself and associated directories.</p> + +<p>Create a zeppelin user and switch to zeppelin user or if zeppelin user is already created then login as zeppelin.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">useradd zeppelin +su - zeppelin +whoami +</code></pre></div> +<p>Assuming a zeppelin user is created then running whoami command must return</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">zeppelin +</code></pre></div> +<p>Its assumed in the rest of the document that zeppelin user is indeed created and below installation instructions are performed as zeppelin user.</p> + +<h3>List of Prerequisites</h3> + +<ul> +<li>CentOS 6.x, Mac OSX, Ubuntu 14.X</li> +<li>Java 1.7</li> +<li>Hadoop client</li> +<li>Spark</li> +<li>Internet connection is required.</li> +</ul> + +<p>It's assumed that the node has CentOS 6.x installed on it. Although any version of Linux distribution should work fine.</p> + +<h4>Hadoop client</h4> + +<p>Zeppelin can work with multiple versions & distributions of Hadoop. A complete list is available <a href="https://github.com/apache/zeppelin#build";>here</a>. This document assumes Hadoop 2.7.x client libraries including configuration files are installed on Zeppelin node. It also assumes /etc/hadoop/conf contains various Hadoop configuration files. The location of Hadoop configuration files may vary, hence use appropriate location.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">hadoop version +Hadoop 2.7.1.2.3.1.0-2574 +Subversion g...@github.com:hortonworks/hadoop.git -r f66cf95e2e9367a74b0ec88b2df33458b6cff2d0 +Compiled by jenkins on 2015-07-25T22:36Z +Compiled with protoc 2.5.0 +From <span class="nb">source </span>with checksum 54f9bbb4492f92975e84e390599b881d +This <span class="nb">command </span>was run using /usr/hdp/2.3.1.0-2574/hadoop/lib/hadoop-common-2.7.1.2.3.1.0-2574.jar +</code></pre></div> +<h4>Spark</h4> + +<p>Spark is supported out of the box and to take advantage of this, you need to Download appropriate version of Spark binary packages from <a href="http://spark.apache.org/downloads.html";>Spark Download page</a> and unzip it. +Zeppelin can work with multiple versions of Spark. A complete list is available <a href="https://github.com/apache/zeppelin#build";>here</a>. +This document assumes Spark 1.6.0 is installed at /usr/lib/spark.</p> + +<blockquote> +<p>Note: Spark should be installed on the same node as Zeppelin.</p> + +<p>Note: Spark's pre-built package for CDH 4 doesn't support yarn.</p> +</blockquote> + +<h4>Zeppelin</h4> + +<p>Checkout source code from <a href="https://github.com/apache/zeppelin.git";>git://git.apache.org/zeppelin.git</a> or download binary package from <a href="https://zeppelin.apache.org/download.html";>Download page</a>. +You can refer <a href="install.html">Install</a> page for the details. +This document assumes that Zeppelin is located under <code>/home/zeppelin/zeppelin</code>.</p> + +<h2>Zeppelin Configuration</h2> + +<p>Zeppelin configuration needs to be modified to connect to YARN cluster. Create a copy of zeppelin environment shell script.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">cp /home/zeppelin/zeppelin/conf/zeppelin-env.sh.template /home/zeppelin/zeppelin/conf/zeppelin-env.sh +</code></pre></div> +<p>Set the following properties</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">export </span><span class="nv">JAVA_HOME</span><span class="o">=</span><span class="s2">"/usr/java/jdk1.7.0_79"</span> +<span class="nb">export </span><span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span><span class="s2">"/etc/hadoop/conf"</span> +<span class="nb">export </span><span class="nv">ZEPPELIN_JAVA_OPTS</span><span class="o">=</span><span class="s2">"-Dhdp.version=2.3.1.0-2574"</span> +<span class="nb">export </span><span class="nv">SPARK_HOME</span><span class="o">=</span><span class="s2">"/usr/lib/spark"</span> +</code></pre></div> +<p>As /etc/hadoop/conf contains various configurations of YARN cluster, Zeppelin can now submit Spark/Hive jobs on YARN cluster form its web interface. The value of hdp.version is set to 2.3.1.0-2574. This can be obtained by running the following command</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">hdp-select status hadoop-client <span class="p">|</span> sed <span class="s1">'s/hadoop-client - \(.*\)/\1/'</span> +<span class="c"># It returned 2.3.1.0-2574</span> +</code></pre></div> +<h2>Start/Stop</h2> + +<h3>Start Zeppelin</h3> +<div class="highlight"><pre><code class="text language-text" data-lang="text">cd /home/zeppelin/zeppelin +bin/zeppelin-daemon.sh start +</code></pre></div> +<p>After successful start, visit http://[zeppelin-server-host-name]:8080 with your web browser.</p> + +<h3>Stop Zeppelin</h3> +<div class="highlight"><pre><code class="text language-text" data-lang="text">bin/zeppelin-daemon.sh stop +</code></pre></div> +<h2>Interpreter</h2> + +<p>Zeppelin provides various distributed processing frameworks to process data that ranges from Spark, JDBC, Ignite and Lens to name a few. This document describes to configure JDBC & Spark interpreters.</p> + +<h3>Hive</h3> + +<p>Zeppelin supports Hive through JDBC interpreter. You might need the information to use Hive and can find in your hive-site.xml</p> + +<p>Once Zeppelin server has started successfully, visit http://[zeppelin-server-host-name]:8080 with your web browser. Click on Interpreter tab next to Notebook dropdown. Look for Hive configurations and set them appropriately. Set them as per Hive installation on YARN cluster. +Click on Save button. Once these configurations are updated, Zeppelin will prompt you to restart the interpreter. Accept the prompt and the interpreter will reload the configurations.</p> + +<h3>Spark</h3> + +<p>It was assumed that 1.6.0 version of Spark is installed at /usr/lib/spark. Look for Spark configurations and click edit button to add the following properties</p> + +<table class="table-configuration"> + <tr> + <th>Property Name</th> + <th>Property Value</th> + <th>Remarks</th> + </tr> + <tr> + <td>master</td> + <td>yarn-client</td> + <td>In yarn-client mode, the driver runs in the client process, and the application master is only used for requesting resources from YARN.</td> + </tr> + <tr> + <td>spark.driver.extraJavaOptions</td> + <td>-Dhdp.version=2.3.1.0-2574</td> + <td></td> + </tr> + <tr> + <td>spark.yarn.am.extraJavaOptions</td> + <td>-Dhdp.version=2.3.1.0-2574</td> + <td></td> + </tr> +</table> + +<p>Click on Save button. Once these configurations are updated, Zeppelin will prompt you to restart the interpreter. Accept the prompt and the interpreter will reload the configurations.</p> + +<p>Spark & Hive notebooks can be written with Zeppelin now. The resulting Spark & Hive jobs will run on configured YARN cluster.</p> + +<h2>Debug</h2> + +<p>Zeppelin does not emit any kind of error messages on web interface when notebook/paragraph is run. If a paragraph fails it only displays ERROR. The reason for failure needs to be looked into log files which is present in logs directory under zeppelin installation base directory. Zeppelin creates a log file for each kind of interpreter.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="o">[</span>zeppelin@zeppelin-3529 logs<span class="o">]</span><span class="nv">$ </span><span class="nb">pwd</span> +/home/zeppelin/zeppelin/logs +<span class="o">[</span>zeppelin@zeppelin-3529 logs<span class="o">]</span><span class="nv">$ </span>ls -l +total 844 +-rw-rw-r-- 1 zeppelin zeppelin 14648 Aug 3 14:45 zeppelin-interpreter-hive-zeppelin-zeppelin-3529.log +-rw-rw-r-- 1 zeppelin zeppelin 625050 Aug 3 16:05 zeppelin-interpreter-spark-zeppelin-zeppelin-3529.log +-rw-rw-r-- 1 zeppelin zeppelin 200394 Aug 3 21:15 zeppelin-zeppelin-zeppelin-3529.log +-rw-rw-r-- 1 zeppelin zeppelin 16162 Aug 3 14:03 zeppelin-zeppelin-zeppelin-3529.out +<span class="o">[</span>zeppelin@zeppelin-3529 logs<span class="o">]</span><span class="err">$</span> +</code></pre></div> + </div> +</div> + + + <hr> + <footer> + <!-- <p>© 2017 The Apache Software Foundation</p>--> + </footer> + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-45176241-5', 'zeppelin.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + </body> +</html> +
Added: zeppelin/site/docs/0.7.2/interpreter/alluxio.html URL: http://svn.apache.org/viewvc/zeppelin/site/docs/0.7.2/interpreter/alluxio.html?rev=1798613&view=auto ============================================================================== --- zeppelin/site/docs/0.7.2/interpreter/alluxio.html (added) +++ zeppelin/site/docs/0.7.2/interpreter/alluxio.html Tue Jun 13 16:55:32 2017 @@ -0,0 +1,485 @@ + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Apache Zeppelin 0.7.2 Documentation: Alluxio Interpreter for Apache Zeppelin</title> + <meta name="description" content="Alluxio is a memory-centric distributed storage system enabling reliable data sharing at memory-speed across cluster frameworks."> + <meta name="author" content="The Apache Software Foundation"> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"> + + <!-- Le styles --> + <link href="/docs/0.7.2/assets/themes/zeppelin/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/docs/0.7.2/assets/themes/zeppelin/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/docs/0.7.2/assets/themes/zeppelin/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- Js --> + <script src="https://code.jquery.com/jquery-1.10.2.min.js";></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/bootstrap/js/bootstrap.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/docs.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/anchor.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/toc.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/lunr.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/search.js"></script> + + <!-- atom & rss feed --> + <link href="/docs/0.7.2/atom.xml" type="application/atom+xml" rel="alternate" title="Sitewide ATOM Feed"> + <link href="/docs/0.7.2/rss.xml" type="application/rss+xml" rel="alternate" title="Sitewide RSS Feed"> + </head> + + <body> + + <div id="menu" class="navbar navbar-inverse navbar-fixed-top" role="navigation"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-brand"> + <a class="navbar-brand-main" href="http://zeppelin.apache.org";> + <img src="/assets/themes/zeppelin/img/zeppelin_logo.png" width="50" alt="I'm zeppelin"> + <span style="vertical-align:middle">Zeppelin</span> + </a> + <a class="navbar-brand-version" href="/docs/0.7.2"> + <span><small>0.7.2</small></span> + </a> + </div> + </div> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Quick Start <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.7.2/index.html">What is Apache Zeppelin ?</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Getting Started</b><span></li> + <li><a href="/docs/0.7.2/install/install.html">Install</a></li> + <li><a href="/docs/0.7.2/install/configuration.html">Configuration</a></li> + <li><a href="/docs/0.7.2/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li> + <li><a href="/docs/0.7.2/quickstart/tutorial.html">Tutorial</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Basic Feature Guide</b><span></li> + <li><a href="/docs/0.7.2/manual/dynamicform.html">Dynamic Form</a></li> + <li><a href="/docs/0.7.2/manual/publish.html">Publish your Paragraph</a></li> + <li><a href="/docs/0.7.2/manual/notebookashomepage.html">Customize Zeppelin Homepage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>More</b><span></li> + <li><a href="/docs/0.7.2/install/upgrade.html">Upgrade Zeppelin Version</a></li> + <li><a href="/docs/0.7.2/install/build.html">Build from source</a></li> + <li><a href="/docs/0.7.2/quickstart/install_with_flink_and_spark_cluster.html">Install Zeppelin with Flink and Spark Clusters Tutorial</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li><a href="/docs/0.7.2/manual/interpreters.html">Overview</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Usage</b><span></li> + <li><a href="/docs/0.7.2/manual/interpreterinstallation.html">Interpreter Installation</a></li> + <!--<li><a href="/docs/0.7.2/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>--> + <li><a href="/docs/0.7.2/manual/dependencymanagement.html">Interpreter Dependency Management</a></li> + <li><a href="/docs/0.7.2/manual/userimpersonation.html">Interpreter User Impersonation</a></li> + <li><a href="/docs/0.7.2/manual/interpreterexechooks.html">Interpreter Execution Hooks (Experimental)</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Available Interpreters</b><span></li> + <li><a href="/docs/0.7.2/interpreter/alluxio.html">Alluxio</a></li> + <li><a href="/docs/0.7.2/interpreter/beam.html">Beam</a></li> + <li><a href="/docs/0.7.2/interpreter/bigquery.html">BigQuery</a></li> + <li><a href="/docs/0.7.2/interpreter/cassandra.html">Cassandra</a></li> + <li><a href="/docs/0.7.2/interpreter/elasticsearch.html">Elasticsearch</a></li> + <li><a href="/docs/0.7.2/interpreter/flink.html">Flink</a></li> + <li><a href="/docs/0.7.2/interpreter/geode.html">Geode</a></li> + <li><a href="/docs/0.7.2/interpreter/hbase.html">HBase</a></li> + <li><a href="/docs/0.7.2/interpreter/hdfs.html">HDFS</a></li> + <li><a href="/docs/0.7.2/interpreter/hive.html">Hive</a></li> + <li><a href="/docs/0.7.2/interpreter/ignite.html">Ignite</a></li> + <li><a href="/docs/0.7.2/interpreter/jdbc.html">JDBC</a></li> + <li><a href="/docs/0.7.2/interpreter/kylin.html">Kylin</a></li> + <li><a href="/docs/0.7.2/interpreter/lens.html">Lens</a></li> + <li><a href="/docs/0.7.2/interpreter/livy.html">Livy</a></li> + <li><a href="/docs/0.7.2/interpreter/markdown.html">Markdown</a></li> + <li><a href="/docs/0.7.2/interpreter/pig.html">Pig</a></li> + <li><a href="/docs/0.7.2/interpreter/python.html">Python</a></li> + <li><a href="/docs/0.7.2/interpreter/postgresql.html">Postgresql, HAWQ</a></li> + <li><a href="/docs/0.7.2/interpreter/r.html">R</a></li> + <li><a href="/docs/0.7.2/interpreter/scalding.html">Scalding</a></li> + <li><a href="/docs/0.7.2/interpreter/scio.html">Scio</a></li> + <li><a href="/docs/0.7.2/interpreter/shell.html">Shell</a></li> + <li><a href="/docs/0.7.2/interpreter/spark.html">Spark</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Display System <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li class="title"><span><b>Basic Display System</b><span></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#text">Text</a></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#html">Html</a></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#table">Table</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Angular API</b><span></li> + <li><a href="/docs/0.7.2/displaysystem/back-end-angular.html">Angular (backend API)</a></li> + <li><a href="/docs/0.7.2/displaysystem/front-end-angular.html">Angular (frontend API)</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">More<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu" style="right: 0; left: auto;"> + <li class="title"><span><b>Notebook Storage</b><span></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#storage-in-zeppelinhub">ZeppelinHub Storage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>REST API</b><span></li> + <li><a href="/docs/0.7.2/rest-api/rest-interpreter.html">Interpreter API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-notebook.html">Notebook API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-notebookRepo.html">Notebook Repository API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-configuration.html">Configuration API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-credential.html">Credential API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-helium.html">Helium API</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Security</b><span></li> + <li><a href="/docs/0.7.2/security/shiroauthentication.html">Shiro Authentication</a></li> + <li><a href="/docs/0.7.2/security/notebook_authorization.html">Notebook Authorization</a></li> + <li><a href="/docs/0.7.2/security/datasource_authorization.html">Data Source Authorization</a></li> + <li><a href="/docs/0.7.2/security/helium_authorization.html">Helium Authorization</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Advanced</b><span></li> + <li><a href="/docs/0.7.2/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li> + <li><a href="/docs/0.7.2/install/cdh.html">Zeppelin on CDH</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Contibute</b><span></li> + <li><a href="/docs/0.7.2/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li> + <li><a href="/docs/0.7.2/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization (Experimental)</a></li> + <li><a href="/docs/0.7.2/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li> + <li><a href="/docs/0.7.2/development/howtocontribute.html">How to contribute (code)</a></li> + <li><a href="/docs/0.7.2/development/howtocontributewebsite.html">How to contribute (website)</a></li> + </ul> + </li> + <li> + <a href="/docs/0.7.2/search.html" class="nav-search-link"> + <span class="fa fa-search nav-search-icon"></span> + </a> + </li> + </ul> + </nav><!--/.navbar-collapse --> + </div> + </div> + + + + <div class="content"> + +<!--<div class="hero-unit Alluxio Interpreter for Apache Zeppelin"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<h1>Alluxio Interpreter for Apache Zeppelin</h1> + +<div id="toc"></div> + +<h2>Overview</h2> + +<p><a href="http://alluxio.org/";>Alluxio</a> is a memory-centric distributed storage system enabling reliable data sharing at memory-speed across cluster frameworks.</p> + +<h2>Configuration</h2> + +<table class="table-configuration"> + <tr> + <th>Name</th> + <th>Class</th> + <th>Description</th> + </tr> + <tr> + <td>alluxio.master.hostname</td> + <td>localhost</td> + <td>Alluxio master hostname</td> + </tr> + <tr> + <td>alluxio.master.port</td> + <td>19998</td> + <td>Alluxio master port</td> + </tr> +</table> + +<h2>Enabling Alluxio Interpreter</h2> + +<p>In a notebook, to enable the <strong>Alluxio</strong> interpreter, click on the <strong>Gear</strong> icon and select <strong>Alluxio</strong>.</p> + +<h2>Using the Alluxio Interpreter</h2> + +<p>In a paragraph, use <code>%alluxio</code> to select the <strong>Alluxio</strong> interpreter and then input all commands.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">%alluxio +<span class="nb">help</span> +</code></pre></div> +<blockquote> +<p><strong>Tip :</strong> Use ( Ctrl + . ) for autocompletion.</p> +</blockquote> + +<h2>Interpreter Commands</h2> + +<p>The <strong>Alluxio</strong> interpreter accepts the following commands.</p> + +<p><center> + <table class="table-configuration"> + <tr> + <th>Operation</th> + <th>Syntax</th> + <th>Description</th> + </tr> + <tr> + <td>cat</td> + <td>cat "path"</td> + <td>Print the content of the file to the console.</td> + </tr> + <tr> + <td>chgrp</td> + <td>chgrp "group" "path"</td> + <td>Change the group of the directory or file.</td> + </tr> + <tr> + <td>chmod</td> + <td>chmod "permission" "path"</td> + <td>Change the permission of the directory or file.</td> + </tr> + <tr> + <td>chown</td> + <td>chown "owner" "path"</td> + <td>Change the owner of the directory or file.</td> + </tr> + <tr> + <td>copyFromLocal</td> + <td>copyFromLocal "source path" "remote path"</td> + <td>Copy the specified file specified by "source path" to the path specified by "remote path". + This command will fail if "remote path" already exists.</td> + </tr> + <tr> + <td>copyToLocal</td> + <td>copyToLocal "remote path" "local path"</td> + <td>Copy the specified file from the path specified by "remote path" to a local destination.</td> + </tr> + <tr> + <td>count</td> + <td>count "path"</td> + <td>Display the number of folders and files matching the specified prefix in "path".</td> + </tr> + <tr> + <td>du</td> + <td>du "path"</td> + <td>Display the size of a file or a directory specified by the input path.</td> + </tr> + <tr> + <td>fileInfo</td> + <td>fileInfo "path"</td> + <td>Print the information of the blocks of a specified file.</td> + </tr> + <tr> + <td>free</td> + <td>free "path"</td> + <td>Free a file or all files under a directory from Alluxio. If the file/directory is also + in under storage, it will still be available there.</td> + </tr> + <tr> + <td>getCapacityBytes</td> + <td>getCapacityBytes</td> + <td>Get the capacity of the AlluxioFS.</td> + </tr> + <tr> + <td>getUsedBytes</td> + <td>getUsedBytes</td> + <td>Get number of bytes used in the AlluxioFS.</td> + </tr> + <tr> + <td>load</td> + <td>load "path"</td> + <td>Load the data of a file or a directory from under storage into Alluxio.</td> + </tr> + <tr> + <td>loadMetadata</td> + <td>loadMetadata "path"</td> + <td>Load the metadata of a file or a directory from under storage into Alluxio.</td> + </tr> + <tr> + <td>location</td> + <td>location "path"</td> + <td>Display a list of hosts that have the file data.</td> + </tr> + <tr> + <td>ls</td> + <td>ls "path"</td> + <td>List all the files and directories directly under the given path with information such as + size.</td> + </tr> + <tr> + <td>mkdir</td> + <td>mkdir "path1" ... "pathn"</td> + <td>Create directory(ies) under the given paths, along with any necessary parent directories. + Multiple paths separated by spaces or tabs. This command will fail if any of the given paths + already exist.</td> + </tr> + <tr> + <td>mount</td> + <td>mount "path" "uri"</td> + <td>Mount the underlying file system path "uri" into the Alluxio namespace as "path". The "path" + is assumed not to exist and is created by the operation. No data or metadata is loaded from under + storage into Alluxio. After a path is mounted, operations on objects under the mounted path are + mirror to the mounted under storage.</td> + </tr> + <tr> + <td>mv</td> + <td>mv "source" "destination"</td> + <td>Move a file or directory specified by "source" to a new location "destination". This command + will fail if "destination" already exists.</td> + </tr> + <tr> + <td>persist</td> + <td>persist "path"</td> + <td>Persist a file or directory currently stored only in Alluxio to the underlying file system.</td> + </tr> + <tr> + <td>pin</td> + <td>pin "path"</td> + <td>Pin the given file to avoid evicting it from memory. If the given path is a directory, it + recursively pins all the files contained and any new files created within this directory.</td> + </tr> + <tr> + <td>report</td> + <td>report "path"</td> + <td>Report to the master that a file is lost.</td> + </tr> + <tr> + <td>rm</td> + <td>rm "path"</td> + <td>Remove a file. This command will fail if the given path is a directory rather than a file.</td> + </tr> + <tr> + <td>setTtl</td> + <td>setTtl "time"</td> + <td>Set the TTL (time to live) in milliseconds to a file.</td> + </tr> + <tr> + <td>tail</td> + <td>tail "path"</td> + <td>Print the last 1KB of the specified file to the console.</td> + </tr> + <tr> + <td>touch</td> + <td>touch "path"</td> + <td>Create a 0-byte file at the specified location.</td> + </tr> + <tr> + <td>unmount</td> + <td>unmount "path"</td> + <td>Unmount the underlying file system path mounted in the Alluxio namespace as "path". Alluxio + objects under "path" are removed from Alluxio, but they still exist in the previously mounted + under storage.</td> + </tr> + <tr> + <td>unpin</td> + <td>unpin "path"</td> + <td>Unpin the given file to allow Alluxio to evict this file again. If the given path is a + directory, it recursively unpins all files contained and any new files created within this + directory.</td> + </tr> + <tr> + <td>unsetTtl</td> + <td>unsetTtl</td> + <td>Remove the TTL (time to live) setting from a file.</td> + </tr><br> + </table> +</center></p> + +<h2>How to test it's working</h2> + +<p>Be sure to have configured correctly the Alluxio interpreter, then open a new paragraph and type one of the above commands.</p> + +<p>Below a simple example to show how to interact with Alluxio interpreter. +Following steps are performed:</p> + +<ul> +<li>using sh interpreter a new text file is created on local machine</li> +<li>using Alluxio interpreter: + +<ul> +<li>is listed the content of the afs (Alluxio File System) root</li> +<li>the file previously created is copied to afs</li> +<li>is listed again the content of the afs root to check the existence of the new copied file</li> +<li>is showed the content of the copied file (using the tail command)</li> +<li>the file previously copied to afs is copied to local machine</li> +</ul></li> +<li> using sh interpreter it's checked the existence of the new file copied from Alluxio and its content is showed</li> +</ul> + +<p><center> + <img src="../assets/themes/zeppelin/img/docs-img/alluxio-example.png" alt="Alluxio Interpreter Example"> +</center></p> + + </div> +</div> + + + <hr> + <footer> + <!-- <p>© 2017 The Apache Software Foundation</p>--> + </footer> + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-45176241-5', 'zeppelin.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + </body> +</html> + Added: zeppelin/site/docs/0.7.2/interpreter/beam.html URL: http://svn.apache.org/viewvc/zeppelin/site/docs/0.7.2/interpreter/beam.html?rev=1798613&view=auto ============================================================================== --- zeppelin/site/docs/0.7.2/interpreter/beam.html (added) +++ zeppelin/site/docs/0.7.2/interpreter/beam.html Tue Jun 13 16:55:32 2017 @@ -0,0 +1,345 @@ + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Apache Zeppelin 0.7.2 Documentation: Beam interpreter in Apache Zeppelin</title> + <meta name="description" content="Apache Beam is an open source, unified programming model that you can use to create a data processing pipeline."> + <meta name="author" content="The Apache Software Foundation"> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"> + + <!-- Le styles --> + <link href="/docs/0.7.2/assets/themes/zeppelin/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/docs/0.7.2/assets/themes/zeppelin/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/docs/0.7.2/assets/themes/zeppelin/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- Js --> + <script src="https://code.jquery.com/jquery-1.10.2.min.js";></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/bootstrap/js/bootstrap.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/docs.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/anchor.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/toc.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/lunr.min.js"></script> + <script src="/docs/0.7.2/assets/themes/zeppelin/js/search.js"></script> + + <!-- atom & rss feed --> + <link href="/docs/0.7.2/atom.xml" type="application/atom+xml" rel="alternate" title="Sitewide ATOM Feed"> + <link href="/docs/0.7.2/rss.xml" type="application/rss+xml" rel="alternate" title="Sitewide RSS Feed"> + </head> + + <body> + + <div id="menu" class="navbar navbar-inverse navbar-fixed-top" role="navigation"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-brand"> + <a class="navbar-brand-main" href="http://zeppelin.apache.org";> + <img src="/assets/themes/zeppelin/img/zeppelin_logo.png" width="50" alt="I'm zeppelin"> + <span style="vertical-align:middle">Zeppelin</span> + </a> + <a class="navbar-brand-version" href="/docs/0.7.2"> + <span><small>0.7.2</small></span> + </a> + </div> + </div> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Quick Start <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="/docs/0.7.2/index.html">What is Apache Zeppelin ?</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Getting Started</b><span></li> + <li><a href="/docs/0.7.2/install/install.html">Install</a></li> + <li><a href="/docs/0.7.2/install/configuration.html">Configuration</a></li> + <li><a href="/docs/0.7.2/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li> + <li><a href="/docs/0.7.2/quickstart/tutorial.html">Tutorial</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Basic Feature Guide</b><span></li> + <li><a href="/docs/0.7.2/manual/dynamicform.html">Dynamic Form</a></li> + <li><a href="/docs/0.7.2/manual/publish.html">Publish your Paragraph</a></li> + <li><a href="/docs/0.7.2/manual/notebookashomepage.html">Customize Zeppelin Homepage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>More</b><span></li> + <li><a href="/docs/0.7.2/install/upgrade.html">Upgrade Zeppelin Version</a></li> + <li><a href="/docs/0.7.2/install/build.html">Build from source</a></li> + <li><a href="/docs/0.7.2/quickstart/install_with_flink_and_spark_cluster.html">Install Zeppelin with Flink and Spark Clusters Tutorial</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li><a href="/docs/0.7.2/manual/interpreters.html">Overview</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Usage</b><span></li> + <li><a href="/docs/0.7.2/manual/interpreterinstallation.html">Interpreter Installation</a></li> + <!--<li><a href="/docs/0.7.2/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>--> + <li><a href="/docs/0.7.2/manual/dependencymanagement.html">Interpreter Dependency Management</a></li> + <li><a href="/docs/0.7.2/manual/userimpersonation.html">Interpreter User Impersonation</a></li> + <li><a href="/docs/0.7.2/manual/interpreterexechooks.html">Interpreter Execution Hooks (Experimental)</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Available Interpreters</b><span></li> + <li><a href="/docs/0.7.2/interpreter/alluxio.html">Alluxio</a></li> + <li><a href="/docs/0.7.2/interpreter/beam.html">Beam</a></li> + <li><a href="/docs/0.7.2/interpreter/bigquery.html">BigQuery</a></li> + <li><a href="/docs/0.7.2/interpreter/cassandra.html">Cassandra</a></li> + <li><a href="/docs/0.7.2/interpreter/elasticsearch.html">Elasticsearch</a></li> + <li><a href="/docs/0.7.2/interpreter/flink.html">Flink</a></li> + <li><a href="/docs/0.7.2/interpreter/geode.html">Geode</a></li> + <li><a href="/docs/0.7.2/interpreter/hbase.html">HBase</a></li> + <li><a href="/docs/0.7.2/interpreter/hdfs.html">HDFS</a></li> + <li><a href="/docs/0.7.2/interpreter/hive.html">Hive</a></li> + <li><a href="/docs/0.7.2/interpreter/ignite.html">Ignite</a></li> + <li><a href="/docs/0.7.2/interpreter/jdbc.html">JDBC</a></li> + <li><a href="/docs/0.7.2/interpreter/kylin.html">Kylin</a></li> + <li><a href="/docs/0.7.2/interpreter/lens.html">Lens</a></li> + <li><a href="/docs/0.7.2/interpreter/livy.html">Livy</a></li> + <li><a href="/docs/0.7.2/interpreter/markdown.html">Markdown</a></li> + <li><a href="/docs/0.7.2/interpreter/pig.html">Pig</a></li> + <li><a href="/docs/0.7.2/interpreter/python.html">Python</a></li> + <li><a href="/docs/0.7.2/interpreter/postgresql.html">Postgresql, HAWQ</a></li> + <li><a href="/docs/0.7.2/interpreter/r.html">R</a></li> + <li><a href="/docs/0.7.2/interpreter/scalding.html">Scalding</a></li> + <li><a href="/docs/0.7.2/interpreter/scio.html">Scio</a></li> + <li><a href="/docs/0.7.2/interpreter/shell.html">Shell</a></li> + <li><a href="/docs/0.7.2/interpreter/spark.html">Spark</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Display System <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li class="title"><span><b>Basic Display System</b><span></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#text">Text</a></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#html">Html</a></li> + <li><a href="/docs/0.7.2/displaysystem/basicdisplaysystem.html#table">Table</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Angular API</b><span></li> + <li><a href="/docs/0.7.2/displaysystem/back-end-angular.html">Angular (backend API)</a></li> + <li><a href="/docs/0.7.2/displaysystem/front-end-angular.html">Angular (frontend API)</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">More<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu" style="right: 0; left: auto;"> + <li class="title"><span><b>Notebook Storage</b><span></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li> + <li><a href="/docs/0.7.2/storage/storage.html#storage-in-zeppelinhub">ZeppelinHub Storage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>REST API</b><span></li> + <li><a href="/docs/0.7.2/rest-api/rest-interpreter.html">Interpreter API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-notebook.html">Notebook API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-notebookRepo.html">Notebook Repository API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-configuration.html">Configuration API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-credential.html">Credential API</a></li> + <li><a href="/docs/0.7.2/rest-api/rest-helium.html">Helium API</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Security</b><span></li> + <li><a href="/docs/0.7.2/security/shiroauthentication.html">Shiro Authentication</a></li> + <li><a href="/docs/0.7.2/security/notebook_authorization.html">Notebook Authorization</a></li> + <li><a href="/docs/0.7.2/security/datasource_authorization.html">Data Source Authorization</a></li> + <li><a href="/docs/0.7.2/security/helium_authorization.html">Helium Authorization</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Advanced</b><span></li> + <li><a href="/docs/0.7.2/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li> + <li><a href="/docs/0.7.2/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li> + <li><a href="/docs/0.7.2/install/cdh.html">Zeppelin on CDH</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span><b>Contibute</b><span></li> + <li><a href="/docs/0.7.2/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li> + <li><a href="/docs/0.7.2/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization (Experimental)</a></li> + <li><a href="/docs/0.7.2/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li> + <li><a href="/docs/0.7.2/development/howtocontribute.html">How to contribute (code)</a></li> + <li><a href="/docs/0.7.2/development/howtocontributewebsite.html">How to contribute (website)</a></li> + </ul> + </li> + <li> + <a href="/docs/0.7.2/search.html" class="nav-search-link"> + <span class="fa fa-search nav-search-icon"></span> + </a> + </li> + </ul> + </nav><!--/.navbar-collapse --> + </div> + </div> + + + + <div class="content"> + +<!--<div class="hero-unit Beam interpreter in Apache Zeppelin"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<h1>Beam interpreter for Apache Zeppelin</h1> + +<div id="toc"></div> + +<h2>Overview</h2> + +<p><a href="http://beam.incubator.apache.org";>Apache Beam</a> is an open source unified platform for data processing pipelines. A pipeline can be build using one of the Beam SDKs. +The execution of the pipeline is done by different Runners. Currently, Beam supports Apache Flink Runner, Apache Spark Runner, and Google Dataflow Runner.</p> + +<h2>How to use</h2> + +<p>Basically, you can write normal Beam java code where you can determine the Runner. You should write the main method inside a class becuase the interpreter invoke this main to execute the pipeline. Unlike Zeppelin normal pattern, each paragraph is considered as a separate job, there isn't any relation to any other paragraph.</p> + +<p>The following is a demonstration of a word count example with data represented in array of strings +But it can read data from files by replacing <code>Create.of(SENTENCES).withCoder(StringUtf8Coder.of())</code> with <code>TextIO.Read.from("path/to/filename.txt")</code></p> +<div class="highlight"><pre><code class="java language-java" data-lang="java"><span class="o">%</span><span class="n">beam</span> + +<span class="c1">// most used imports</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.coders.StringUtf8Coder</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.transforms.Create</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">java.io.Serializable</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">java.util.Arrays</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">java.util.List</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">java.util.ArrayList</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.api.java.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.api.java.function.Function</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.SparkConf</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.streaming.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.SparkContext</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.runners.direct.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.runners.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.options.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.runners.spark.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.runners.spark.io.ConsoleIO</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.runners.flink.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.runners.flink.examples.WordCount.Options</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.Pipeline</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.io.TextIO</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.options.PipelineOptionsFactory</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.transforms.Count</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.transforms.DoFn</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.transforms.MapElements</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.transforms.ParDo</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.transforms.SimpleFunction</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.values.KV</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.beam.sdk.options.PipelineOptions</span><span class="o">;</span> + +<span class="kd">public</span> <span class="kd">class</span> <span class="nc">MinimalWordCount</span> <span class="o">{</span> + <span class="kd">static</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">s</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><>();</span> + + <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span><span class="o">[]</span> <span class="n">SENTENCES_ARRAY</span> <span class="o">=</span> <span class="k">new</span> <span class="n">String</span><span class="o">[]</span> <span class="o">{</span> + <span class="s">"Hadoop is the Elephant King!"</span><span class="o">,</span> + <span class="s">"A yellow and elegant thing."</span><span class="o">,</span> + <span class="s">"He never forgets"</span><span class="o">,</span> + <span class="s">"Useful data, or lets"</span><span class="o">,</span> + <span class="s">"An extraneous element cling!"</span><span class="o">,</span> + <span class="s">"A wonderful king is Hadoop."</span><span class="o">,</span> + <span class="s">"The elephant plays well with Sqoop."</span><span class="o">,</span> + <span class="s">"But what helps him to thrive"</span><span class="o">,</span> + <span class="s">"Are Impala, and Hive,"</span><span class="o">,</span> + <span class="s">"And HDFS in the group."</span><span class="o">,</span> + <span class="s">"Hadoop is an elegant fellow."</span><span class="o">,</span> + <span class="s">"An elephant gentle and mellow."</span><span class="o">,</span> + <span class="s">"He never gets mad,"</span><span class="o">,</span> + <span class="s">"Or does anything bad,"</span><span class="o">,</span> + <span class="s">"Because, at his core, he is yellow"</span><span class="o">,</span> + <span class="o">};</span> + <span class="kd">static</span> <span class="kd">final</span> <span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">SENTENCES</span> <span class="o">=</span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span><span class="n">SENTENCES_ARRAY</span><span class="o">);</span> + <span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="o">{</span> + <span class="n">Options</span> <span class="n">options</span> <span class="o">=</span> <span class="n">PipelineOptionsFactory</span><span class="o">.</span><span class="na">create</span><span class="o">().</span><span class="na">as</span><span class="o">(</span><span class="n">Options</span><span class="o">.</span><span class="na">class</span><span class="o">);</span> + <span class="n">options</span><span class="o">.</span><span class="na">setRunner</span><span class="o">(</span><span class="n">FlinkRunner</span><span class="o">.</span><span class="na">class</span><span class="o">);</span> + <span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="n">options</span><span class="o">);</span> + <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Create</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">SENTENCES</span><span class="o">).</span><span class="na">withCoder</span><span class="o">(</span><span class="n">StringUtf8Coder</span><span class="o">.</span><span class="na">of</span><span class="o">()))</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="s">"ExtractWords"</span><span class="o">,</span> <span class="n">ParDo</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="k">new</span> <span class="n">DoFn</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">>()</span> <span class="o">{</span> + <span class="nd">@Override</span> + <span class="kd">public</span> <span class="kt">void</span> <span class="nf">processElement</span><span class="o">(</span><span class="n">ProcessContext</span> <span class="n">c</span><span class="o">)</span> <span class="o">{</span> + <span class="k">for</span> <span class="o">(</span><span class="n">String</span> <span class="n">word</span> <span class="o">:</span> <span class="n">c</span><span class="o">.</span><span class="na">element</span><span class="o">().</span><span class="na">split</span><span class="o">(</span><span class="s">"[^a-zA-Z']+"</span><span class="o">))</span> <span class="o">{</span> + <span class="k">if</span> <span class="o">(!</span><span class="n">word</span><span class="o">.</span><span class="na">isEmpty</span><span class="o">())</span> <span class="o">{</span> + <span class="n">c</span><span class="o">.</span><span class="na">output</span><span class="o">(</span><span class="n">word</span><span class="o">);</span> + <span class="o">}</span> + <span class="o">}</span> + <span class="o">}</span> + <span class="o">}))</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Count</span><span class="o">.<</span><span class="n">String</span><span class="o">></span> <span class="n">perElement</span><span class="o">())</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="s">"FormatResults"</span><span class="o">,</span> <span class="n">ParDo</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="k">new</span> <span class="n">DoFn</span><span class="o"><</span><span class="n">KV</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Long</span><span class="o">>,</span> <span class="n">String</span><span class="o">>()</span> <span class="o">{</span> + <span class="nd">@Override</span> + <span class="kd">public</span> <span class="kt">void</span> <span class="nf">processElement</span><span class="o">(</span><span class="n">DoFn</span><span class="o"><</span><span class="n">KV</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Long</span><span class="o">>,</span> <span class="n">String</span><span class="o">>.</span><span class="na">ProcessContext</span> <span class="n">arg0</span><span class="o">)</span> + <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span> + <span class="n">s</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="s">"\n"</span> <span class="o">+</span> <span class="n">arg0</span><span class="o">.</span><span class="na">element</span><span class="o">().</span><span class="na">getKey</span><span class="o">()</span> <span class="o">+</span> <span class="s">"\t"</span> <span class="o">+</span> <span class="n">arg0</span><span class="o">.</span><span class="na">element</span><span class="o">().</span><span class="na">getValue</span><span class="o">());</span> + <span class="o">}</span> + <span class="o">}));</span> + <span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">();</span> + <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">"%table word\tcount"</span><span class="o">);</span> + <span class="k">for</span> <span class="o">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="o">;</span> <span class="n">i</span> <span class="o"><</span> <span class="n">s</span><span class="o">.</span><span class="na">size</span><span class="o">();</span> <span class="n">i</span><span class="o">++)</span> <span class="o">{</span> + <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">print</span><span class="o">(</span><span class="n">s</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="n">i</span><span class="o">));</span> + <span class="o">}</span> + + <span class="o">}</span> +<span class="o">}</span> +</code></pre></div> + </div> +</div> + + + <hr> + <footer> + <!-- <p>© 2017 The Apache Software Foundation</p>--> + </footer> + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-45176241-5', 'zeppelin.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + </body> +</html> +
