Repository: zeppelin Updated Branches: refs/heads/master df7dd5c37 -> 4efb39f45
[ZEPPELIN-1046] bin/install-interpreter.sh for netinst package ### What is this PR for? Implementation of bin/install-interpreter.sh for netinst package which suggested in the [discussion](http://apache-zeppelin-users-incubating-mailing-list.75479.x6.nabble.com/Ask-opinion-regarding-0-6-0-release-package-tp3298p3314.html). Some usages will be ``` # download all interpreters provided by Apache Zeppelin project bin/install-interpreter.sh --all # download an interpreter with name (for example markdown interpreter) bin/install-interpreter.sh --name md # download an (3rd party) interpreter with specific maven artifact name bin/install-interpreter.sh --name md -t org.apache.zeppelin:zeppelin-markdown:0.6.0-SNAPSHOT ``` If it looks fine, i'll continue the work (refactor code, and add test) ### What type of PR is it? Feature ### Todos * [x] - working implementation * [x] - refactor * [x] - add test ### What is the Jira issue? * Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/ * Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533] ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: Lee moon soo <[email protected]> Author: AhyoungRyu <[email protected]> Closes #1042 from Leemoonsoo/netinst and squashes the following commits: f81d16e [Lee moon soo] address mina's comment 049bc89 [Lee moon soo] Update docs 7307c67 [Lee moon soo] Merge remote-tracking branch 'AhyoungRyu/netinst-docs' into netinst 7e749ad [Lee moon soo] Address mina's comment 0eedd2a [AhyoungRyu] Address @minahlee feedback 13f2d04 [Lee moon soo] generate netinst package 03c664e [AhyoungRyu] Add a new line 5d0a971 [AhyoungRyu] Revert install.md to latest version 13899fb [AhyoungRyu] Reorganize interpreter installation docs 4c1f029 [Lee moon soo] Proxy support 9079580 [Lee moon soo] fix artifact name 1077296 [Lee moon soo] update test aebca17 [Lee moon soo] Add docs d547551 [Lee moon soo] Remove test entries 6ee06b8 [Lee moon soo] Make DependencyResolver in zeppelin-interpreter module not aware of ZEPPELIN_HOME 7b1b36a [Lee moon soo] update usage 49f0568 [Lee moon soo] Add conf/interpreter-list 1b558fd [Lee moon soo] update some text ec7d152 [Lee moon soo] add tip 2c81a3f [Lee moon soo] update 78a7c52 [Lee moon soo] Refactor and add test 47f5706 [Lee moon soo] Install multiple interpreters at once 38e2556 [Lee moon soo] Initial implementation of install-interpreter.sh Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4efb39f4 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4efb39f4 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4efb39f4 Branch: refs/heads/master Commit: 4efb39f45079b018d9a7907870c8e435924c63f8 Parents: df7dd5c Author: Lee moon soo <[email protected]> Authored: Thu Jun 23 15:04:43 2016 -0700 Committer: Mina Lee <[email protected]> Committed: Thu Jun 23 20:58:10 2016 -0700 ---------------------------------------------------------------------- bin/install-interpreter.sh | 45 +++ conf/interpreter-list | 35 +++ dev/create_release.sh | 1 + docs/_includes/themes/zeppelin/_navigation.html | 6 +- docs/install/install.md | 5 +- docs/manual/interpreterinstallation.md | 164 ++++++++++ .../dep/AbstractDependencyResolver.java | 12 + .../apache/zeppelin/dep/DependencyResolver.java | 25 +- .../zeppelin/dep/DependencyResolverTest.java | 31 +- .../zeppelin/conf/ZeppelinConfiguration.java | 4 + .../interpreter/InterpreterFactory.java | 6 +- .../interpreter/install/InstallInterpreter.java | 301 +++++++++++++++++++ .../install/InstallInterpreterTest.java | 86 ++++++ 13 files changed, 680 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/bin/install-interpreter.sh ---------------------------------------------------------------------- diff --git a/bin/install-interpreter.sh b/bin/install-interpreter.sh new file mode 100755 index 0000000..06be75c --- /dev/null +++ b/bin/install-interpreter.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Run Zeppelin +# + +bin=$(dirname "${BASH_SOURCE-$0}") +bin=$(cd "${bin}">/dev/null; pwd) + +. "${bin}/common.sh" + + +ZEPPELIN_INSTALL_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.install.InstallInterpreter +ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/install-interpreter.log" +JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" +fi +addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" +fi +addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" + +addJarInDir "${ZEPPELIN_HOME}/lib" + +CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +$ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_INSTALL_INTERPRETER_MAIN ${@} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/conf/interpreter-list ---------------------------------------------------------------------- diff --git a/conf/interpreter-list b/conf/interpreter-list new file mode 100644 index 0000000..a2f6426 --- /dev/null +++ b/conf/interpreter-list @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# [name] [maven artifact] [description] + +alluxio org.apache.zeppelin:zeppelin-alluxio:0.6.0 Alluxio interpreter +angular org.apache.zeppelin:zeppelin-angular:0.6.0 HTML and AngularJS view rendering +cassandra org.apache.zeppelin:zeppelin-cassandra:0.6.0 Cassandra interpreter +elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter +file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter +flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter +hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter +ignite org.apache.zeppelin:zeppelin-ignite:0.6.0 Ignite interpreter +jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.0 Jdbc interpreter +kylin org.apache.zeppelin:zeppelin-kylin:0.6.0 Kylin interpreter +lens org.apache.zeppelin:zeppelin-lens:0.6.0 Lens interpreter +livy org.apache.zeppelin:zeppelin-livy:0.6.0 Livy interpreter +md org.apache.zeppelin:zeppelin-markdown:0.6.0 Markdown support +postgresql org.apache.zeppelin:zeppelin-postgresql:0.6.0 Postgresql interpreter +python org.apache.zeppelin:zeppelin-python:0.6.0 Python interpreter +shell org.apache.zeppelin:zeppelin-shell:0.6.0 Shell command http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/dev/create_release.sh ---------------------------------------------------------------------- diff --git a/dev/create_release.sh b/dev/create_release.sh index f1bba0d..2363d90 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -103,6 +103,7 @@ function make_binary_release() { git_clone make_source_package make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" +make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell" # remove non release files and dirs rm -rf "${WORKING_DIR}/zeppelin" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/docs/_includes/themes/zeppelin/_navigation.html ---------------------------------------------------------------------- diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 40b9fb6..498a2d3 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -21,8 +21,8 @@ <li><a href="{{BASE_PATH}}/index.html">What is Apache Zeppelin ?</a></li> <li role="separator" class="divider"></li> <li class="title"><span><b>Getting Started</b><span></li> - <li><a href="{{BASE_PATH}}/install/install.html">Quick Start</a></li> - <li><a href="{{BASE_PATH}}/install/install.html#zeppelin-configuration">Configuration</a></li> + <li><a href="{{BASE_PATH}}/install/install.html">Install</a></li> + <li><a href="{{BASE_PATH}}/install/install.html#apache-zeppelin-configuration">Configuration</a></li> <li><a href="{{BASE_PATH}}/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li> <li><a href="{{BASE_PATH}}/quickstart/tutorial.html">Tutorial</a></li> <li role="separator" class="divider"></li> @@ -42,6 +42,7 @@ <li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li> <li role="separator" class="divider"></li> <li class="title"><span><b>Usage</b><span></li> + <li><a href="{{BASE_PATH}}/manual/interpreterinstallation.html">Interpreter Installation</a></li> <li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li> <li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li> <li role="separator" class="divider"></li> @@ -110,3 +111,4 @@ </nav><!--/.navbar-collapse --> </div> </div> + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/docs/install/install.md ---------------------------------------------------------------------- diff --git a/docs/install/install.md b/docs/install/install.md index 93d9feb..55741dc 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -60,6 +60,9 @@ Although it can be unstable somehow since it is on development status, you can e ### Downloading Binary Package If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html). + +If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`. + After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. ### Building from Source @@ -388,4 +391,4 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z <td>1024000</td> <td>Size in characters of the maximum text message to be received by websocket.</td> </tr> -</table> +</table> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/docs/manual/interpreterinstallation.md ---------------------------------------------------------------------- diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md new file mode 100644 index 0000000..d522e62 --- /dev/null +++ b/docs/manual/interpreterinstallation.md @@ -0,0 +1,164 @@ +--- +layout: page +title: "Interpreter Installation" +description: "" +group: manual +--- +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +{% include JB/setup %} + +# Interpreter Installation + +Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters. + +## Community managed interpreters +Apache Zeppelin provides several interpreters as [community managed interpreters](#available-community-managed-interpreters). +If you downloaded `netinst` binary package, you need to install by using below commands. + +#### Install all community managed interpreters + +``` +./bin/install-interpreter.sh --all +``` + +#### Install specific interpreters + +``` +./bin/install-interpreter.sh --name md,shell,jdbc,python +``` + +You can get full list of community managed interpreters by running + +``` +./bin/install-interpreter.sh --list +``` + +Once you have installed interpreters, you need to restart Zeppelin. And then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +## 3rd party interpreters + +You can also install 3rd party interpreters located in the maven repository by using below commands. + +#### Install 3rd party interpreters + +``` +./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1 +``` + +The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. + +Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration). +And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +#### Install multiple 3rd party interpreters at once + +``` +./bin/install-interpreter.sh --name interpreter1,interpreter2 --artifact groupId1:artifact1:version1,groupId2:artifact2:version2 +``` + +`--name` and `--artifact` arguments will recieve comma separated list. + +## Available community managed interpreters + +You can also find the below community managed interpreter list in `conf/interpreter-list` file. +<table class="table-configuration"> + <tr> + <th>Name</th> + <th>Maven Artifact</th> + <th>Description</th> + </tr> + <tr> + <td>alluxio</td> + <td>org.apache.zeppelin:zeppelin-alluxio:0.6.0</td> + <td>Alluxio interpreter</td> + </tr> + <tr> + <td>angular</td> + <td>org.apache.zeppelin:zeppelin-angular:0.6.0</td> + <td>HTML and AngularJS view rendering</td> + </tr> + <tr> + <td>cassandra</td> + <td>org.apache.zeppelin:zeppelin-cassandra:0.6.0</td> + <td>Cassandra interpreter</td> + </tr> + <tr> + <td>elasticsearch</td> + <td>org.apache.zeppelin:zeppelin-elasticsearch:0.6.0</td> + <td>Elasticsearch interpreter</td> + </tr> + <tr> + <td>file</td> + <td>org.apache.zeppelin:zeppelin-file:0.6.0</td> + <td>HDFS file interpreter</td> + </tr> + <tr> + <td>flink</td> + <td>org.apache.zeppelin:zeppelin-flink:0.6.0</td> + <td>Flink interpreter</td> + </tr> + <tr> + <td>hbase</td> + <td>org.apache.zeppelin:zeppelin-hbase:0.6.0</td> + <td>Hbase interpreter</td> + </tr> + <tr> + <td>ignite</td> + <td>org.apache.zeppelin:zeppelin-ignite:0.6.0</td> + <td>Ignite interpreter</td> + </tr> + <tr> + <td>jdbc</td> + <td>org.apache.zeppelin:zeppelin-jdbc:0.6.0</td> + <td>Jdbc interpreter</td> + </tr> + <tr> + <td>kylin</td> + <td>org.apache.zeppelin:zeppelin-kylin:0.6.0</td> + <td>Kylin interpreter</td> + </tr> + <tr> + <td>lens</td> + <td>org.apache.zeppelin:zeppelin-lens:0.6.0</td> + <td>Lens interpreter</td> + </tr> + <tr> + <td>livy</td> + <td>org.apache.zeppelin:zeppelin-livy:0.6.0</td> + <td>Livy interpreter</td> + </tr> + <tr> + <td>md</td> + <td>org.apache.zeppelin:zeppelin-markdown:0.6.0</td> + <td>Markdown support</td> + </tr> + <tr> + <td>postgresql</td> + <td>org.apache.zeppelin:zeppelin-postgresql:0.6.0</td> + <td>Postgresql interpreter</td> + </tr> + <tr> + <td>python</td> + <td>org.apache.zeppelin:zeppelin-python:0.6.0</td> + <td>Python interpreter</td> + </tr> + <tr> + <td>shell</td> + <td>org.apache.zeppelin:zeppelin-shell:0.6.0</td> + <td>Shell command</td> + </tr> +</table> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java index b22941e..1e0844e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.dep; +import java.net.URL; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; @@ -25,6 +26,7 @@ import java.util.List; import org.sonatype.aether.RepositorySystem; import org.sonatype.aether.RepositorySystemSession; import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.Proxy; import org.sonatype.aether.repository.RemoteRepository; import org.sonatype.aether.repository.RepositoryPolicy; import org.sonatype.aether.resolution.ArtifactResult; @@ -44,6 +46,16 @@ public abstract class AbstractDependencyResolver { repos.add(Booter.newLocalRepository()); } + public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) { + Authentication auth = new Authentication(proxyUser, proxyPassword); + Proxy proxy = new Proxy(proxyUrl.getProtocol(), proxyUrl.getHost(), proxyUrl.getPort(), auth); + synchronized (repos) { + for (RemoteRepository repo : repos) { + repo.setProxy(proxy); + } + } + } + public List<RemoteRepository> getRepos() { return this.repos; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index 60ef1f9..214175a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -19,6 +19,7 @@ package org.apache.zeppelin.dep; import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -63,11 +64,6 @@ public class DependencyResolver extends AbstractDependencyResolver { return load(artifact, new LinkedList<String>()); } - public List<File> load(String artifact, String destPath) - throws RepositoryException, IOException { - return load(artifact, new LinkedList<String>(), destPath); - } - public synchronized List<File> load(String artifact, Collection<String> excludes) throws RepositoryException, IOException { if (StringUtils.isBlank(artifact)) { @@ -85,25 +81,20 @@ public class DependencyResolver extends AbstractDependencyResolver { return libs; } } - - public List<File> load(String artifact, Collection<String> excludes, String destPath) + + public List<File> load(String artifact, File destPath) throws IOException, RepositoryException { + return load(artifact, new LinkedList<String>(), destPath); + } + + public List<File> load(String artifact, Collection<String> excludes, File destPath) throws RepositoryException, IOException { List<File> libs = new LinkedList<File>(); if (StringUtils.isNotBlank(artifact)) { libs = load(artifact, excludes); - // find home dir - String home = System.getenv("ZEPPELIN_HOME"); - if (home == null) { - home = System.getProperty("zeppelin.home"); - } - if (home == null) { - home = ".."; - } - for (File srcFile : libs) { - File destFile = new File(home + "/" + destPath, srcFile.getName()); + File destFile = new File(destPath, srcFile.getName()); if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) { FileUtils.copyFile(srcFile, destFile); logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java index a8664ef..af2c7ff 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java @@ -33,27 +33,20 @@ import org.sonatype.aether.RepositoryException; public class DependencyResolverTest { private static DependencyResolver resolver; private static String testPath; - private static String testCopyPath; - private static String home; - + private static File testCopyPath; + private static File tmpDir; + @BeforeClass public static void setUp() throws Exception { - testPath = "test-repo"; - testCopyPath = "test-copy-repo"; + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + testPath = tmpDir.getAbsolutePath() + "/test-repo"; + testCopyPath = new File(tmpDir, "test-copy-repo"); resolver = new DependencyResolver(testPath); - home = System.getenv("ZEPPELIN_HOME"); - if (home == null) { - home = System.getProperty("zeppelin.home"); - } - if (home == null) { - home = ".."; - } } @AfterClass public static void tearDown() throws Exception { - FileUtils.deleteDirectory(new File(home + "/" + testPath)); - FileUtils.deleteDirectory(new File(home + "/" + testCopyPath)); + FileUtils.deleteDirectory(tmpDir); } @Rule @@ -78,19 +71,19 @@ public class DependencyResolverTest { public void testLoad() throws Exception { // basic load resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 4); - FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + assertEquals(testCopyPath.list().length, 4); + FileUtils.cleanDirectory(testCopyPath); // load with exclusions parameter resolver.load("com.databricks:spark-csv_2.10:1.3.0", Collections.singletonList("org.scala-lang:scala-library"), testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 3); - FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + assertEquals(testCopyPath.list().length, 3); + FileUtils.cleanDirectory(testCopyPath); // load from added repository resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/";, false); resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 8); + assertEquals(testCopyPath.list().length, 8); // load invalid artifact resolver.delRepo("sonatype"); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 45fbba4..0a7b8c0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -346,6 +346,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP); } + public String getInterpreterListPath() { + return getRelativeDir(String.format("%s/interpreter-list", getConfDir())); + } + public String getInterpreterDir() { return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index bad18c0..aeb7818 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -350,15 +350,17 @@ public class InterpreterFactory implements InterpreterGroupFactory { List<Dependency> deps = intSetting.getDependencies(); if (deps != null) { for (Dependency d: deps) { + File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + if (d.getExclusions() != null) { depResolver.load( d.getGroupArtifactVersion(), d.getExclusions(), - conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + new File(destDir, intSetting.id())); } else { depResolver.load( d.getGroupArtifactVersion(), - conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + new File(destDir, intSetting.id())); } } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java new file mode 100644 index 0000000..da67d9f --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.install; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Logger; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.util.Util; +import org.sonatype.aether.RepositoryException; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Commandline utility to install interpreter from maven repository + */ +public class InstallInterpreter { + private final File interpreterListFile; + private final File interpreterBaseDir; + private final List<AvailableInterpreterInfo> availableInterpreters; + private final String localRepoDir; + private URL proxyUrl; + private String proxyUser; + private String proxyPassword; + + /** + * + * @param interpreterListFile + * @param interpreterBaseDir interpreter directory for installing binaries + * @throws IOException + */ + public InstallInterpreter(File interpreterListFile, File interpreterBaseDir, String localRepoDir) + throws IOException { + this.interpreterListFile = interpreterListFile; + this.interpreterBaseDir = interpreterBaseDir; + this.localRepoDir = localRepoDir; + availableInterpreters = new LinkedList<AvailableInterpreterInfo>(); + readAvailableInterpreters(); + } + + + /** + * Information for available informations + */ + private static class AvailableInterpreterInfo { + public final String name; + public final String artifact; + public final String description; + + public AvailableInterpreterInfo(String name, String artifact, String description) { + this.name = name; + this.artifact = artifact; + this.description = description; + } + } + + private void readAvailableInterpreters() throws IOException { + if (!interpreterListFile.isFile()) { + System.err.println("Can't find interpreter list " + interpreterListFile.getAbsolutePath()); + return; + } + String text = FileUtils.readFileToString(interpreterListFile); + String[] lines = text.split("\n"); + + Pattern pattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(.*)"); + + int lineNo = 0; + for (String line : lines) { + lineNo++; + if (line == null || line.length() == 0 || line.startsWith("#")) { + continue; + } + + Matcher match = pattern.matcher(line); + if (match.groupCount() != 3) { + System.err.println("Error on line " + lineNo + ", " + line); + continue; + } + + match.find(); + + String name = match.group(1); + String artifact = match.group(2); + String description = match.group(3); + + availableInterpreters.add(new AvailableInterpreterInfo(name, artifact, description)); + } + } + + public List<AvailableInterpreterInfo> list() { + for (AvailableInterpreterInfo info : availableInterpreters) { + System.out.println(info.name + "\t\t\t" + info.description); + } + + return availableInterpreters; + } + + public void installAll() { + for (AvailableInterpreterInfo info : availableInterpreters) { + install(info.name, info.artifact); + } + } + + public void install(String [] names) { + for (String name : names) { + install(name); + } + } + + public void install(String name) { + // find artifact name + for (AvailableInterpreterInfo info : availableInterpreters) { + if (name.equals(info.name)) { + install(name, info.artifact); + return; + } + } + + throw new RuntimeException("Can't find interpreter '" + name + "'"); + } + + public void install(String [] names, String [] artifacts) { + if (names.length != artifacts.length) { + throw new RuntimeException("Length of given names and artifacts are different"); + } + + for (int i = 0; i < names.length; i++) { + install(names[i], artifacts[i]); + } + } + + public void install(String name, String artifact) { + DependencyResolver depResolver = new DependencyResolver(localRepoDir); + if (proxyUrl != null) { + depResolver.setProxy(proxyUrl, proxyUser, proxyPassword); + } + + File installDir = new File(interpreterBaseDir, name); + if (installDir.exists()) { + System.err.println("Directory " + installDir.getAbsolutePath() + " already exists. Skipping"); + return; + } + + System.out.println("Install " + name + "(" + artifact + ") to " + + installDir.getAbsolutePath() + " ... "); + + try { + depResolver.load(artifact, installDir); + System.out.println("Interpreter " + name + " installed under " + + installDir.getAbsolutePath() + "."); + } catch (RepositoryException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) { + this.proxyUrl = proxyUrl; + this.proxyUser = proxyUser; + this.proxyPassword = proxyPassword; + } + + public static void usage() { + System.out.println("Options"); + System.out.println(" -l, --list List available interpreters"); + System.out.println(" -a, --all Install all available interpreters"); + System.out.println(" -n, --name [NAMES] Install interpreters (comma separated " + + "list)" + + "e.g. md,shell,jdbc,python,angular"); + System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names" + + ". " + + "(comma separated list correspond to --name) " + + "e.g. customGroup:customArtifact:customVersion"); + System.out.println(" --proxy-url [url] (Optional) proxy url. http(s)://host:port"); + System.out.println(" --proxy-user [user] (Optional) proxy user"); + System.out.println(" --proxy-password [password] (Optional) proxy password"); + } + + public static void main(String [] args) throws IOException { + if (args.length == 0) { + usage(); + return; + } + + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + InstallInterpreter installer = new InstallInterpreter( + new File(conf.getInterpreterListPath()), + new File(conf.getInterpreterDir()), + conf.getInterpreterLocalRepoPath()); + + String names = null; + String artifacts = null; + URL proxyUrl = null; + String proxyUser = null; + String proxyPassword = null; + boolean all = false; + + for (int i = 0; i < args.length; i++) { + String arg = args[i].toLowerCase(Locale.US); + switch (arg) { + case "--list": + case "-l": + installer.list(); + System.exit(0); + break; + case "--all": + case "-a": + all = true; + break; + case "--name": + case "-n": + names = args[++i]; + break; + case "--artifact": + case "-t": + artifacts = args[++i]; + break; + case "--version": + case "-v": + Util.getVersion(); + break; + case "--proxy-url": + proxyUrl = new URL(args[++i]); + break; + case "--proxy-user": + proxyUser = args[++i]; + break; + case "--proxy-password": + proxyPassword = args[++i]; + break; + case "--help": + case "-h": + usage(); + System.exit(0); + break; + default: + System.out.println("Unknown option " + arg); + } + } + + if (proxyUrl != null) { + installer.setProxy(proxyUrl, proxyUser, proxyPassword); + } + + if (all) { + installer.installAll(); + System.exit(0); + } + + if (names != null) { + if (artifacts != null) { + installer.install(names.split(","), artifacts.split(",")); + startTip(); + configurationTip(); + interpreterSettingTip(); + } else { + installer.install(names.split(",")); + startTip(); + interpreterSettingTip(); + } + } + } + + private static void startTip() { + System.out.println(""); + } + + private static void configurationTip() { + System.out.println("Add interpreter class name to '" + + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName() + "' property " + + "in your conf/zeppelin-site.xml file"); + } + + private static void interpreterSettingTip() { + System.out.println("Create interpreter setting in 'Interpreter' menu on GUI." + + " And then you can bind interpreter on your notebook"); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4efb39f4/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java new file mode 100644 index 0000000..e934f1a --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java @@ -0,0 +1,86 @@ +package org.apache.zeppelin.interpreter.install; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class InstallInterpreterTest { + private File tmpDir; + private InstallInterpreter installer; + private File interpreterBaseDir; + + @Before + public void setUp() throws IOException { + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + new File(tmpDir, "conf").mkdirs(); + interpreterBaseDir = new File(tmpDir, "interpreter"); + File localRepoDir = new File(tmpDir, "local-repo"); + interpreterBaseDir.mkdir(); + localRepoDir.mkdir(); + + File interpreterListFile = new File(tmpDir, "conf/interpreter-list"); + + + // create interpreter list file + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); + + String interpreterList = ""; + interpreterList += "intp1 org.apache.commons:commons-csv:1.1 test interpreter 1\n"; + interpreterList += "intp2 org.apache.commons:commons-math3:3.6.1 test interpreter 2\n"; + + FileUtils.writeStringToFile(new File(tmpDir, "conf/interpreter-list"), interpreterList); + + installer = new InstallInterpreter(interpreterListFile, interpreterBaseDir, localRepoDir + .getAbsolutePath()); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + + @Test + public void testList() { + assertEquals(2, installer.list().size()); + } + + @Test + public void install() { + assertEquals(0, interpreterBaseDir.listFiles().length); + + installer.install("intp1"); + assertTrue(new File(interpreterBaseDir, "intp1").isDirectory()); + } + + @Test + public void installAll() { + installer.installAll(); + assertTrue(new File(interpreterBaseDir, "intp1").isDirectory()); + assertTrue(new File(interpreterBaseDir, "intp2").isDirectory()); + } +}
