This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push: new b9006a2 ZEPPELIN-4227. Move StandardInterpreterLauncher and SparkInterpreterLauncher into zeppelin-zengine b9006a2 is described below commit b9006a247043aa3151a798d10c85b7edf6ea9fe3 Author: Jeff Zhang <zjf...@apache.org> AuthorDate: Sun Jul 7 22:25:17 2019 +0800 ZEPPELIN-4227. Move StandardInterpreterLauncher and SparkInterpreterLauncher into zeppelin-zengine ### What is this PR for? We should include these default interpreter launcher into zeppelin-zengine, so that user can use zeppelin even without building zeppelin-plugins module. ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-4227 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Closes #3421 from zjffdu/ZEPPELIN-4227 and squashes the following commits: ee1c7bcc1 [Jeff Zhang] ZEPPELIN-4227. Move StandardInterpreterLauncher and SparkInterpreterLauncher into zeppelin-zengine --- .travis.yml | 4 +- zeppelin-plugins/launcher/cluster/pom.xml | 7 +-- zeppelin-plugins/launcher/docker/pom.xml | 5 -- zeppelin-plugins/launcher/spark/pom.xml | 67 ---------------------- zeppelin-plugins/launcher/standard/pom.xml | 49 ---------------- zeppelin-plugins/pom.xml | 2 - .../apache/zeppelin/rest/NotebookRestApiTest.java | 4 +- .../launcher/SparkInterpreterLauncher.java | 0 .../launcher/StandardInterpreterLauncher.java | 0 .../org/apache/zeppelin/plugin/PluginManager.java | 13 +++++ .../launcher/SparkInterpreterLauncherTest.java | 9 +++ .../launcher/StandardInterpreterLauncherTest.java | 0 12 files changed, 27 insertions(+), 133 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef2193a..2838e1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,7 +86,7 @@ matrix: - sudo: required jdk: "openjdk8" dist: xenial - env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev -Pexamples -Pspark-scala-2.11" BUILD_FLAG="clean install -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/JdbcIntegrationTest.java,**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" + env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev -Pexamples -Pspark-scala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat -DskipTests" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/JdbcIntegrationTest.java,**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for spark 2.3 - jdk: "openjdk8" @@ -98,7 +98,7 @@ matrix: # Test interpreter modules - jdk: "openjdk8" dist: xenial - env: PYTHON="3" SPARKR="true" SCALA_VER="2.10" TENSORFLOW="1.0.0" PROFILE="-Pscala-2.10" BUILD_FLAG="clean install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,zeppelin-interpreter-api,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" + env: PYTHON="3" SPARKR="true" SCALA_VER="2.10" TENSORFLOW="1.0.0" PROFILE="-Pscala-2.10" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,zeppelin-interpreter-api,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" # Run Spark integration test and unit test separately for each spark version diff --git a/zeppelin-plugins/launcher/cluster/pom.xml b/zeppelin-plugins/launcher/cluster/pom.xml index bf24546..08b03be 100644 --- a/zeppelin-plugins/launcher/cluster/pom.xml +++ b/zeppelin-plugins/launcher/cluster/pom.xml @@ -39,15 +39,10 @@ <properties> <plugin.name>Launcher/ClusterInterpreterLauncher</plugin.name> </properties> - + <dependencies> <dependency> <groupId>org.apache.zeppelin</groupId> - <artifactId>launcher-standard</artifactId> - <version>0.9.0-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>org.apache.zeppelin</groupId> <artifactId>launcher-docker</artifactId> <version>0.9.0-SNAPSHOT</version> </dependency> diff --git a/zeppelin-plugins/launcher/docker/pom.xml b/zeppelin-plugins/launcher/docker/pom.xml index 6d029fe..b9a47cb 100644 --- a/zeppelin-plugins/launcher/docker/pom.xml +++ b/zeppelin-plugins/launcher/docker/pom.xml @@ -42,11 +42,6 @@ <dependencies> <dependency> - <groupId>org.apache.zeppelin</groupId> - <artifactId>launcher-spark</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>com.hubspot.jinjava</groupId> <artifactId>jinjava</artifactId> <version>2.4.12</version> diff --git a/zeppelin-plugins/launcher/spark/pom.xml b/zeppelin-plugins/launcher/spark/pom.xml deleted file mode 100644 index 189c27f..0000000 --- a/zeppelin-plugins/launcher/spark/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> - - <modelVersion>4.0.0</modelVersion> - - <parent> - <artifactId>zengine-plugins-parent</artifactId> - <groupId>org.apache.zeppelin</groupId> - <version>0.9.0-SNAPSHOT</version> - <relativePath>../../../zeppelin-plugins</relativePath> - </parent> - - <groupId>org.apache.zeppelin</groupId> - <artifactId>launcher-spark</artifactId> - <packaging>jar</packaging> - <version>0.9.0-SNAPSHOT</version> - <name>Zeppelin: Plugin SparkInterpreterLauncher</name> - <description>Spark Launcher implementation based on shell script interpreter.sh</description> - - <properties> - <plugin.name>Launcher/SparkInterpreterLauncher</plugin.name> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.zeppelin</groupId> - <artifactId>launcher-standard</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <artifactId>maven-dependency-plugin</artifactId> - </plugin> - - <plugin> - <artifactId>maven-surefire-plugin</artifactId> - <version>${plugin.surefire.version}</version> - <configuration> - <environmentVariables> - <ZEPPELIN_HOME>${project.basedir}/../../..</ZEPPELIN_HOME> - </environmentVariables> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/zeppelin-plugins/launcher/standard/pom.xml b/zeppelin-plugins/launcher/standard/pom.xml deleted file mode 100644 index c8c68ce..0000000 --- a/zeppelin-plugins/launcher/standard/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> - - <modelVersion>4.0.0</modelVersion> - - <parent> - <artifactId>zengine-plugins-parent</artifactId> - <groupId>org.apache.zeppelin</groupId> - <version>0.9.0-SNAPSHOT</version> - <relativePath>../../../zeppelin-plugins</relativePath> - </parent> - - <groupId>org.apache.zeppelin</groupId> - <artifactId>launcher-standard</artifactId> - <packaging>jar</packaging> - <version>0.9.0-SNAPSHOT</version> - <name>Zeppelin: Plugin StandardLauncher</name> - <description>Launcher implementation based on shell script interpreter.sh</description> - - <properties> - <plugin.name>Launcher/StandardInterpreterLauncher</plugin.name> - </properties> - - <build> - <plugins> - <plugin> - <artifactId>maven-dependency-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/zeppelin-plugins/pom.xml b/zeppelin-plugins/pom.xml index dea3a47..bca6421 100644 --- a/zeppelin-plugins/pom.xml +++ b/zeppelin-plugins/pom.xml @@ -43,9 +43,7 @@ <module>notebookrepo/zeppelin-hub</module> <module>notebookrepo/filesystem</module> - <module>launcher/standard</module> <module>launcher/k8s-standard</module> - <module>launcher/spark</module> <module>launcher/cluster</module> <module>launcher/docker</module> </modules> diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java index 1835b51..079fe5f 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java @@ -422,8 +422,8 @@ public class NotebookRestApiTest extends AbstractTestRestApi { assertEquals(resp.get("status"), "OK"); post2.releaseConnection(); - assertEquals(Job.Status.FINISHED, p1.getStatus()); - assertEquals(Job.Status.FINISHED, p2.getStatus()); + assertEquals(p1.getReturn().toString(), Job.Status.FINISHED, p1.getStatus()); + assertEquals(p2.getReturn().toString(), Job.Status.FINISHED, p2.getStatus()); assertNotNull(p2.getReturn()); assertEquals("abc\n", p2.getReturn().message().get(0).getData()); } finally { diff --git a/zeppelin-plugins/launcher/spark/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java similarity index 100% rename from zeppelin-plugins/launcher/spark/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java diff --git a/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java similarity index 100% rename from zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java index c3a2b9b..cdcfed0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java @@ -140,6 +140,19 @@ public class PluginManager { return cachedLaunchers.get(launcherPlugin); } LOGGER.info("Loading Interpreter Launcher Plugin: " + launcherPlugin); + // load plugin from classpath directly first for these builtin InterpreterLauncher. + // If fails, then try to load it from plugin folder. + try { + InterpreterLauncher launcher = (InterpreterLauncher) + (Class.forName("org.apache.zeppelin.interpreter.launcher." + launcherPlugin)) + .getConstructor(ZeppelinConfiguration.class, RecoveryStorage.class) + .newInstance(zConf, recoveryStorage); + return launcher; + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException + | NoSuchMethodException | InvocationTargetException e) { + LOGGER.warn("Fail to instantiate InterpreterLauncher from classpath directly:" + launcherPlugin, e); + } + URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir, "Launcher", launcherPlugin); String pluginClass = "org.apache.zeppelin.interpreter.launcher." + launcherPlugin; InterpreterLauncher launcher = null; diff --git a/zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java similarity index 97% rename from zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java rename to zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java index 118e7d4..8500de6 100644 --- a/zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java @@ -25,7 +25,10 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; import org.apache.zeppelin.util.Util; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -37,6 +40,8 @@ import static org.junit.Assert.assertTrue; public class SparkInterpreterLauncherTest { + private static final Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterLauncher.class); + private String sparkHome; private String zeppelinHome; @@ -47,7 +52,11 @@ public class SparkInterpreterLauncherTest { } sparkHome = DownloadUtils.downloadSpark("2.3.2"); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), + new File("..").getAbsolutePath()); + zeppelinHome = ZeppelinConfiguration.create().getZeppelinHome(); + LOGGER.info("ZEPPELIN_HOME: " + zeppelinHome); } @Test diff --git a/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java similarity index 100% rename from zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java rename to zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java