ZEPPELIN-546 Enables interpreter library loading from maven repository ### What is this PR for? This PR enables library loading from maven repository to load interpreter binaries. To leverage current spark interpreter's library loading, moved org.apache.zeppelin.spark.dep(under spark) to org.apache.zeppelin.dep(under zeppelin-interpreter). Making REST API and loading interpreter property on runtime will be the next step to complete [ZEPPELIN-546](https://issues.apache.org/jira/browse/ZEPPELIN-546?jql=project%20%3D%20ZEPPELIN) and will be handled in different PR.
### What type of PR is it? Feature ### Is there a relevant Jira issue? [ZEPPELIN-546](https://issues.apache.org/jira/browse/ZEPPELIN-546?jql=project%20%3D%20ZEPPELIN) ### Questions: * Does the licenses files need update? No (dependencies added to `zeppelin-interpreter/pom.xml` is already used in other module) * Is there breaking changes for older versions? No * Does this needs documentation? Yes (will be addressed in another PR) Author: Mina Lee <[email protected]> Closes #590 from minahlee/ZEPPELIN-546 and squashes the following commits: cd190d4 [Mina Lee] [ZEPPELIN-546] Refactoring 72ad12c [Mina Lee] [ZEPPELIN-546] Add test 039f5fa [Mina Lee] [ZEPPELIN-546] Fix tests 4baf271 [Mina Lee] [ZEPPELIN-546] Pass DependencyResolver to InterpreterFactory aab2d04 [Mina Lee] [ZEPPELIN-546] Load interpreter from maven repository - Move org.apache.zeppelin.spark.dep package from zeppelin-spark to zeppelin-interpreter - Rename DependencyResolver/DependencyContext to SparkDependencyResolver/SparkDependencyContext - Add general DependencyResolver Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/bc715511 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/bc715511 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/bc715511 Branch: refs/heads/master Commit: bc715511404985b60c4cd302be305bf8fd917bf7 Parents: 0c42f43 Author: Mina Lee <[email protected]> Authored: Tue Jan 12 16:12:38 2016 -0800 Committer: Lee moon soo <[email protected]> Committed: Tue Jan 12 19:17:50 2016 -0800 ---------------------------------------------------------------------- .../apache/zeppelin/spark/DepInterpreter.java | 36 +- .../zeppelin/spark/PySparkInterpreter.java | 4 +- .../apache/zeppelin/spark/SparkInterpreter.java | 14 +- .../apache/zeppelin/spark/ZeppelinContext.java | 6 +- .../org/apache/zeppelin/spark/dep/Booter.java | 72 ---- .../apache/zeppelin/spark/dep/Dependency.java | 90 ----- .../zeppelin/spark/dep/DependencyContext.java | 179 --------- .../zeppelin/spark/dep/DependencyResolver.java | 385 ------------------- .../apache/zeppelin/spark/dep/Repository.java | 80 ---- .../zeppelin/spark/dep/RepositoryListener.java | 118 ------ .../spark/dep/RepositorySystemFactory.java | 66 ---- .../spark/dep/SparkDependencyContext.java | 181 +++++++++ .../spark/dep/SparkDependencyResolver.java | 351 +++++++++++++++++ .../zeppelin/spark/dep/TransferListener.java | 145 ------- .../spark/dep/DependencyResolverTest.java | 52 --- .../spark/dep/SparkDependencyResolverTest.java | 52 +++ zeppelin-interpreter/pom.xml | 117 ++++++ .../dep/AbstractDependencyResolver.java | 70 ++++ .../java/org/apache/zeppelin/dep/Booter.java | 72 ++++ .../org/apache/zeppelin/dep/Dependency.java | 90 +++++ .../apache/zeppelin/dep/DependencyContext.java | 148 +++++++ .../apache/zeppelin/dep/DependencyResolver.java | 163 ++++++++ .../org/apache/zeppelin/dep/Repository.java | 80 ++++ .../apache/zeppelin/dep/RepositoryListener.java | 118 ++++++ .../zeppelin/dep/RepositorySystemFactory.java | 66 ++++ .../apache/zeppelin/dep/TransferListener.java | 145 +++++++ .../zeppelin/dep/DependencyResolverTest.java | 62 +++ .../apache/zeppelin/server/ZeppelinServer.java | 5 +- .../zeppelin/conf/ZeppelinConfiguration.java | 1 + .../interpreter/InterpreterFactory.java | 12 +- .../interpreter/InterpreterFactoryTest.java | 62 +-- .../notebook/NoteInterpreterLoaderTest.java | 2 +- .../apache/zeppelin/notebook/NotebookTest.java | 4 +- .../notebook/repo/NotebookRepoSyncTest.java | 2 +- .../notebook/repo/VFSNotebookRepoTest.java | 2 +- 35 files changed, 1806 insertions(+), 1246 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 7a17aa0..a4fdae3 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -40,7 +40,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.spark.dep.DependencyContext; +import org.apache.zeppelin.spark.dep.SparkDependencyContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonatype.aether.resolution.ArtifactResolutionException; @@ -69,7 +69,9 @@ public class DepInterpreter extends Interpreter { "spark", DepInterpreter.class.getName(), new InterpreterPropertyBuilder() - .add("zeppelin.dep.localrepo", "local-repo", "local repository for dependency loader") + .add("zeppelin.dep.localrepo", + getSystemDefault("ZEPPELIN_DEP_LOCALREPO", null, "local-repo"), + "local repository for dependency loader") .add("zeppelin.dep.additionalRemoteRepository", "spark-packages,http://dl.bintray.com/spark-packages/maven,false;";, "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.") @@ -79,7 +81,7 @@ public class DepInterpreter extends Interpreter { private SparkIMain intp; private ByteArrayOutputStream out; - private DependencyContext depc; + private SparkDependencyContext depc; private SparkJLineCompletion completor; private SparkILoop interpreter; static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class); @@ -88,10 +90,30 @@ public class DepInterpreter extends Interpreter { super(property); } - public DependencyContext getDependencyContext() { + public SparkDependencyContext getDependencyContext() { return depc; } + public static String getSystemDefault( + String envName, + String propertyName, + String defaultValue) { + + if (envName != null && !envName.isEmpty()) { + String envValue = System.getenv().get(envName); + if (envValue != null) { + return envValue; + } + } + + if (propertyName != null && !propertyName.isEmpty()) { + String propValue = System.getProperty(propertyName); + if (propValue != null) { + return propValue; + } + } + return defaultValue; + } @Override public void close() { @@ -152,16 +174,16 @@ public class DepInterpreter extends Interpreter { intp.setContextClassLoader(); intp.initializeSynchronous(); - depc = new DependencyContext(getProperty("zeppelin.dep.localrepo"), + depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), getProperty("zeppelin.dep.additionalRemoteRepository")); completor = new SparkJLineCompletion(intp); - intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); Map<String, Object> binder = (Map<String, Object>) getValue("_binder"); binder.put("depc", depc); intp.interpret("@transient val z = " - + "_binder.get(\"depc\").asInstanceOf[org.apache.zeppelin.spark.dep.DependencyContext]"); + + "_binder.get(\"depc\")" + + ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]"); } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index 0bfad6a..8c4ba87 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -54,7 +54,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.WrappedInterpreter; -import org.apache.zeppelin.spark.dep.DependencyContext; +import org.apache.zeppelin.spark.dep.SparkDependencyContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,7 +127,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand URL [] urls = new URL[0]; if (depInterpreter != null) { - DependencyContext depc = depInterpreter.getDependencyContext(); + SparkDependencyContext depc = depInterpreter.getDependencyContext(); if (depc != null) { List<File> files = depc.getFiles(); List<URL> urlList = new LinkedList<URL>(); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 2bf7a6b..d975791 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -55,8 +55,8 @@ import org.apache.zeppelin.interpreter.InterpreterUtils; import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.apache.zeppelin.spark.dep.DependencyContext; -import org.apache.zeppelin.spark.dep.DependencyResolver; +import org.apache.zeppelin.spark.dep.SparkDependencyContext; +import org.apache.zeppelin.spark.dep.SparkDependencyResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,7 +117,7 @@ public class SparkInterpreter extends Interpreter { private SparkContext sc; private ByteArrayOutputStream out; private SQLContext sqlc; - private DependencyResolver dep; + private SparkDependencyResolver dep; private SparkJLineCompletion completor; private JobProgressListener sparkListener; @@ -222,9 +222,9 @@ public class SparkInterpreter extends Interpreter { return sqlc; } - public DependencyResolver getDependencyResolver() { + public SparkDependencyResolver getDependencyResolver() { if (dep == null) { - dep = new DependencyResolver(intp, + dep = new SparkDependencyResolver(intp, sc, getProperty("zeppelin.dep.localrepo"), getProperty("zeppelin.dep.additionalRemoteRepository")); @@ -427,7 +427,7 @@ public class SparkInterpreter extends Interpreter { // add dependency from DepInterpreter DepInterpreter depInterpreter = getDepInterpreter(); if (depInterpreter != null) { - DependencyContext depc = depInterpreter.getDependencyContext(); + SparkDependencyContext depc = depInterpreter.getDependencyContext(); if (depc != null) { List<File> files = depc.getFiles(); if (files != null) { @@ -536,7 +536,7 @@ public class SparkInterpreter extends Interpreter { // add jar if (depInterpreter != null) { - DependencyContext depc = depInterpreter.getDependencyContext(); + SparkDependencyContext depc = depInterpreter.getDependencyContext(); if (depc != null) { List<File> files = depc.getFilesDist(); if (files != null) { http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index 5ec38d4..af806bf 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -43,7 +43,7 @@ import org.apache.zeppelin.display.Input.ParamOption; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.spark.dep.DependencyResolver; +import org.apache.zeppelin.spark.dep.SparkDependencyResolver; import scala.Tuple2; import scala.Unit; @@ -53,14 +53,14 @@ import scala.collection.Iterable; * Spark context for zeppelin. */ public class ZeppelinContext extends HashMap<String, Object> { - private DependencyResolver dep; + private SparkDependencyResolver dep; private PrintStream out; private InterpreterContext interpreterContext; private int maxResult; public ZeppelinContext(SparkContext sc, SQLContext sql, InterpreterContext interpreterContext, - DependencyResolver dep, PrintStream printStream, + SparkDependencyResolver dep, PrintStream printStream, int maxResult) { this.sc = sc; this.sqlContext = sql; http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java deleted file mode 100644 index 1b7a6d6..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import java.io.File; - -import org.apache.maven.repository.internal.MavenRepositorySystemSession; -import org.sonatype.aether.RepositorySystem; -import org.sonatype.aether.RepositorySystemSession; -import org.sonatype.aether.repository.LocalRepository; -import org.sonatype.aether.repository.RemoteRepository; - -/** - * Manage mvn repository. - */ -public class Booter { - public static RepositorySystem newRepositorySystem() { - return RepositorySystemFactory.newRepositorySystem(); - } - - public static RepositorySystemSession newRepositorySystemSession( - RepositorySystem system, String localRepoPath) { - MavenRepositorySystemSession session = new MavenRepositorySystemSession(); - - // find homedir - String home = System.getenv("ZEPPELIN_HOME"); - if (home == null) { - home = System.getProperty("zeppelin.home"); - } - if (home == null) { - home = ".."; - } - - String path = home + "/" + localRepoPath; - - LocalRepository localRepo = - new LocalRepository(new File(path).getAbsolutePath()); - session.setLocalRepositoryManager(system.newLocalRepositoryManager(localRepo)); - - // session.setTransferListener(new ConsoleTransferListener()); - // session.setRepositoryListener(new ConsoleRepositoryListener()); - - // uncomment to generate dirty trees - // session.setDependencyGraphTransformer( null ); - - return session; - } - - public static RemoteRepository newCentralRepository() { - return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/";); - } - - public static RemoteRepository newLocalRepository() { - return new RemoteRepository("local", - "default", "file://" + System.getProperty("user.home") + "/.m2/repository"); - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/Dependency.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/Dependency.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/Dependency.java deleted file mode 100644 index ca92893..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/Dependency.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import java.util.LinkedList; -import java.util.List; - -/** - * - */ -public class Dependency { - private String groupArtifactVersion; - private boolean local = false; - private List<String> exclusions; - - - public Dependency(String groupArtifactVersion) { - this.groupArtifactVersion = groupArtifactVersion; - exclusions = new LinkedList<String>(); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Dependency)) { - return false; - } else { - return ((Dependency) o).groupArtifactVersion.equals(groupArtifactVersion); - } - } - - /** - * Don't add artifact into SparkContext (sc.addJar()) - * @return - */ - public Dependency local() { - local = true; - return this; - } - - public Dependency excludeAll() { - exclude("*"); - return this; - } - - /** - * - * @param exclusions comma or newline separated list of "groupId:ArtifactId" - * @return - */ - public Dependency exclude(String exclusions) { - for (String item : exclusions.split(",|\n")) { - this.exclusions.add(item); - } - - return this; - } - - - public String getGroupArtifactVersion() { - return groupArtifactVersion; - } - - public boolean isDist() { - return !local; - } - - public List<String> getExclusions() { - return exclusions; - } - - public boolean isLocalFsArtifact() { - int numSplits = groupArtifactVersion.split(":").length; - return !(numSplits >= 3 && numSplits <= 6); - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java deleted file mode 100644 index 834e518..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import java.io.File; -import java.net.MalformedURLException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.HashMap; - -import org.sonatype.aether.RepositorySystem; -import org.sonatype.aether.RepositorySystemSession; -import org.sonatype.aether.artifact.Artifact; -import org.sonatype.aether.collection.CollectRequest; -import org.sonatype.aether.graph.DependencyFilter; -import org.sonatype.aether.repository.RemoteRepository; -import org.sonatype.aether.repository.Authentication; -import org.sonatype.aether.resolution.ArtifactResolutionException; -import org.sonatype.aether.resolution.ArtifactResult; -import org.sonatype.aether.resolution.DependencyRequest; -import org.sonatype.aether.resolution.DependencyResolutionException; -import org.sonatype.aether.util.artifact.DefaultArtifact; -import org.sonatype.aether.util.artifact.JavaScopes; -import org.sonatype.aether.util.filter.DependencyFilterUtils; -import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; - - -/** - * - */ -public class DependencyContext { - List<Dependency> dependencies = new LinkedList<Dependency>(); - List<Repository> repositories = new LinkedList<Repository>(); - - List<File> files = new LinkedList<File>(); - List<File> filesDist = new LinkedList<File>(); - private RepositorySystem system = Booter.newRepositorySystem(); - private RepositorySystemSession session; - private RemoteRepository mavenCentral = Booter.newCentralRepository(); - private RemoteRepository mavenLocal = Booter.newLocalRepository(); - private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>(); - - public DependencyContext(String localRepoPath, String additionalRemoteRepository) { - session = Booter.newRepositorySystemSession(system, localRepoPath); - addRepoFromProperty(additionalRemoteRepository); - } - - public Dependency load(String lib) { - Dependency dep = new Dependency(lib); - - if (dependencies.contains(dep)) { - dependencies.remove(dep); - } - dependencies.add(dep); - return dep; - } - - public Repository addRepo(String name) { - Repository rep = new Repository(name); - repositories.add(rep); - return rep; - } - - public void reset() { - dependencies = new LinkedList<Dependency>(); - repositories = new LinkedList<Repository>(); - - files = new LinkedList<File>(); - filesDist = new LinkedList<File>(); - } - - private void addRepoFromProperty(String listOfRepo) { - if (listOfRepo != null) { - String[] repos = listOfRepo.split(";"); - for (String repo : repos) { - String[] parts = repo.split(","); - if (parts.length == 3) { - String id = parts[0].trim(); - String url = parts[1].trim(); - boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); - if (id.length() > 1 && url.length() > 1) { - RemoteRepository rr = new RemoteRepository(id, "default", url); - rr.setPolicy(isSnapshot, null); - additionalRepos.add(rr); - } - } - } - } - } - - /** - * fetch all artifacts - * @return - * @throws MalformedURLException - * @throws ArtifactResolutionException - * @throws DependencyResolutionException - */ - public List<File> fetch() throws MalformedURLException, - DependencyResolutionException, ArtifactResolutionException { - - for (Dependency dep : dependencies) { - if (!dep.isLocalFsArtifact()) { - List<ArtifactResult> artifacts = fetchArtifactWithDep(dep); - for (ArtifactResult artifact : artifacts) { - if (dep.isDist()) { - filesDist.add(artifact.getArtifact().getFile()); - } - files.add(artifact.getArtifact().getFile()); - } - } else { - if (dep.isDist()) { - filesDist.add(new File(dep.getGroupArtifactVersion())); - } - files.add(new File(dep.getGroupArtifactVersion())); - } - } - - return files; - } - - private List<ArtifactResult> fetchArtifactWithDep(Dependency dep) - throws DependencyResolutionException, ArtifactResolutionException { - Artifact artifact = new DefaultArtifact( - DependencyResolver.inferScalaVersion(dep.getGroupArtifactVersion())); - - DependencyFilter classpathFlter = DependencyFilterUtils - .classpathFilter(JavaScopes.COMPILE); - PatternExclusionsDependencyFilter exclusionFilter = new PatternExclusionsDependencyFilter( - DependencyResolver.inferScalaVersion(dep.getExclusions())); - - CollectRequest collectRequest = new CollectRequest(); - collectRequest.setRoot(new org.sonatype.aether.graph.Dependency(artifact, - JavaScopes.COMPILE)); - - collectRequest.addRepository(mavenCentral); - collectRequest.addRepository(mavenLocal); - for (RemoteRepository repo : additionalRepos) { - collectRequest.addRepository(repo); - } - for (Repository repo : repositories) { - RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl()); - rr.setPolicy(repo.isSnapshot(), null); - Authentication auth = repo.getAuthentication(); - if (auth != null) { - rr.setAuthentication(auth); - } - collectRequest.addRepository(rr); - } - - DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, - DependencyFilterUtils.andFilter(exclusionFilter, classpathFlter)); - - return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); - } - - public List<File> getFiles() { - return files; - } - - public List<File> getFilesDist() { - return filesDist; - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java deleted file mode 100644 index eed3924..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URL; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.spark.SparkContext; -import org.apache.spark.repl.SparkIMain; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.RepositorySystem; -import org.sonatype.aether.RepositorySystemSession; -import org.sonatype.aether.artifact.Artifact; -import org.sonatype.aether.collection.CollectRequest; -import org.sonatype.aether.graph.Dependency; -import org.sonatype.aether.graph.DependencyFilter; -import org.sonatype.aether.repository.RemoteRepository; -import org.sonatype.aether.resolution.ArtifactResult; -import org.sonatype.aether.resolution.DependencyRequest; -import org.sonatype.aether.util.artifact.DefaultArtifact; -import org.sonatype.aether.util.artifact.JavaScopes; -import org.sonatype.aether.util.filter.DependencyFilterUtils; -import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; - -import scala.Some; -import scala.collection.IndexedSeq; -import scala.reflect.io.AbstractFile; -import scala.tools.nsc.Global; -import scala.tools.nsc.backend.JavaPlatform; -import scala.tools.nsc.util.ClassPath; -import scala.tools.nsc.util.MergedClassPath; - -/** - * Deps resolver. - * Add new dependencies from mvn repo (at runetime) to Zeppelin. - */ -public class DependencyResolver { - Logger logger = LoggerFactory.getLogger(DependencyResolver.class); - private Global global; - private SparkIMain intp; - private SparkContext sc; - private RepositorySystem system = Booter.newRepositorySystem(); - private List<RemoteRepository> repos = new LinkedList<RemoteRepository>(); - private RepositorySystemSession session; - private DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter( - JavaScopes.COMPILE, - JavaScopes.PROVIDED, - JavaScopes.RUNTIME, - JavaScopes.SYSTEM); - - private final String[] exclusions = new String[] {"org.scala-lang:scala-library", - "org.scala-lang:scala-compiler", - "org.scala-lang:scala-reflect", - "org.scala-lang:scalap", - "org.apache.zeppelin:zeppelin-zengine", - "org.apache.zeppelin:zeppelin-spark", - "org.apache.zeppelin:zeppelin-server"}; - - public DependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath, - String additionalRemoteRepository) { - this.intp = intp; - this.global = intp.global(); - this.sc = sc; - session = Booter.newRepositorySystemSession(system, localRepoPath); - repos.add(Booter.newCentralRepository()); // add maven central - repos.add(Booter.newLocalRepository()); - addRepoFromProperty(additionalRemoteRepository); - } - - public void addRepo(String id, String url, boolean snapshot) { - synchronized (repos) { - delRepo(id); - RemoteRepository rr = new RemoteRepository(id, "default", url); - rr.setPolicy(snapshot, null); - repos.add(rr); - } - } - - public RemoteRepository delRepo(String id) { - synchronized (repos) { - Iterator<RemoteRepository> it = repos.iterator(); - if (it.hasNext()) { - RemoteRepository repo = it.next(); - if (repo.getId().equals(id)) { - it.remove(); - return repo; - } - } - } - return null; - } - - private void addRepoFromProperty(String listOfRepo) { - if (listOfRepo != null) { - String[] repos = listOfRepo.split(";"); - for (String repo : repos) { - String[] parts = repo.split(","); - if (parts.length == 3) { - String id = parts[0].trim(); - String url = parts[1].trim(); - boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); - if (id.length() > 1 && url.length() > 1) { - addRepo(id, url, isSnapshot); - } - } - } - } - } - - private void updateCompilerClassPath(URL[] urls) throws IllegalAccessException, - IllegalArgumentException, InvocationTargetException { - - JavaPlatform platform = (JavaPlatform) global.platform(); - MergedClassPath<AbstractFile> newClassPath = mergeUrlsIntoClassPath(platform, urls); - - Method[] methods = platform.getClass().getMethods(); - for (Method m : methods) { - if (m.getName().endsWith("currentClassPath_$eq")) { - m.invoke(platform, new Some(newClassPath)); - break; - } - } - - // NOTE: Must use reflection until this is exposed/fixed upstream in Scala - List<String> classPaths = new LinkedList<String>(); - for (URL url : urls) { - classPaths.add(url.getPath()); - } - - // Reload all jars specified into our compiler - global.invalidateClassPathEntries(scala.collection.JavaConversions.asScalaBuffer(classPaths) - .toList()); - } - - // Until spark 1.1.x - // check https://github.com/apache/spark/commit/191d7cf2a655d032f160b9fa181730364681d0e7 - private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException, - IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.classLoader().getParent(); - Method addURL; - addURL = cl.getClass().getDeclaredMethod("addURL", new Class[] {URL.class}); - addURL.setAccessible(true); - for (URL url : urls) { - addURL.invoke(cl, url); - } - } - - private void updateRuntimeClassPath_2_x(URL[] urls) throws SecurityException, - IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.classLoader().getParent(); - Method addURL; - addURL = cl.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class}); - addURL.setAccessible(true); - for (URL url : urls) { - addURL.invoke(cl, url); - } - } - - private MergedClassPath<AbstractFile> mergeUrlsIntoClassPath(JavaPlatform platform, URL[] urls) { - IndexedSeq<ClassPath<AbstractFile>> entries = - ((MergedClassPath<AbstractFile>) platform.classPath()).entries(); - List<ClassPath<AbstractFile>> cp = new LinkedList<ClassPath<AbstractFile>>(); - - for (int i = 0; i < entries.size(); i++) { - cp.add(entries.apply(i)); - } - - for (URL url : urls) { - AbstractFile file; - if ("file".equals(url.getProtocol())) { - File f = new File(url.getPath()); - if (f.isDirectory()) { - file = AbstractFile.getDirectory(scala.reflect.io.File.jfile2path(f)); - } else { - file = AbstractFile.getFile(scala.reflect.io.File.jfile2path(f)); - } - } else { - file = AbstractFile.getURL(url); - } - - ClassPath<AbstractFile> newcp = platform.classPath().context().newClassPath(file); - - // distinct - if (cp.contains(newcp) == false) { - cp.add(newcp); - } - } - - return new MergedClassPath(scala.collection.JavaConversions.asScalaBuffer(cp).toIndexedSeq(), - platform.classPath().context()); - } - - public List<String> load(String artifact, - boolean addSparkContext) throws Exception { - return load(artifact, new LinkedList<String>(), addSparkContext); - } - - public List<String> load(String artifact, Collection<String> excludes, - boolean addSparkContext) throws Exception { - if (StringUtils.isBlank(artifact)) { - // Should throw here - throw new RuntimeException("Invalid artifact to load"); - } - - // <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version> - int numSplits = artifact.split(":").length; - if (numSplits >= 3 && numSplits <= 6) { - return loadFromMvn(artifact, excludes, addSparkContext); - } else { - loadFromFs(artifact, addSparkContext); - LinkedList<String> libs = new LinkedList<String>(); - libs.add(artifact); - return libs; - } - } - - private void loadFromFs(String artifact, boolean addSparkContext) throws Exception { - File jarFile = new File(artifact); - - intp.global().new Run(); - - if (sc.version().startsWith("1.1")) { - updateRuntimeClassPath_1_x(new URL[] {jarFile.toURI().toURL()}); - } else { - updateRuntimeClassPath_2_x(new URL[] {jarFile.toURI().toURL()}); - } - - if (addSparkContext) { - sc.addJar(jarFile.getAbsolutePath()); - } - } - - private List<String> loadFromMvn(String artifact, Collection<String> excludes, - boolean addSparkContext) throws Exception { - List<String> loadedLibs = new LinkedList<String>(); - Collection<String> allExclusions = new LinkedList<String>(); - allExclusions.addAll(excludes); - allExclusions.addAll(Arrays.asList(exclusions)); - - List<ArtifactResult> listOfArtifact; - listOfArtifact = getArtifactsWithDep(artifact, allExclusions); - - Iterator<ArtifactResult> it = listOfArtifact.iterator(); - while (it.hasNext()) { - Artifact a = it.next().getArtifact(); - String gav = a.getGroupId() + ":" + a.getArtifactId() + ":" + a.getVersion(); - for (String exclude : allExclusions) { - if (gav.startsWith(exclude)) { - it.remove(); - break; - } - } - } - - List<URL> newClassPathList = new LinkedList<URL>(); - List<File> files = new LinkedList<File>(); - for (ArtifactResult artifactResult : listOfArtifact) { - logger.info("Load " + artifactResult.getArtifact().getGroupId() + ":" - + artifactResult.getArtifact().getArtifactId() + ":" - + artifactResult.getArtifact().getVersion()); - newClassPathList.add(artifactResult.getArtifact().getFile().toURI().toURL()); - files.add(artifactResult.getArtifact().getFile()); - loadedLibs.add(artifactResult.getArtifact().getGroupId() + ":" - + artifactResult.getArtifact().getArtifactId() + ":" - + artifactResult.getArtifact().getVersion()); - } - - intp.global().new Run(); - if (sc.version().startsWith("1.1")) { - updateRuntimeClassPath_1_x(newClassPathList.toArray(new URL[0])); - } else { - updateRuntimeClassPath_2_x(newClassPathList.toArray(new URL[0])); - } - updateCompilerClassPath(newClassPathList.toArray(new URL[0])); - - if (addSparkContext) { - for (File f : files) { - sc.addJar(f.getAbsolutePath()); - } - } - - return loadedLibs; - } - - /** - * - * @param dependency - * @param excludes list of pattern can either be of the form groupId:artifactId - * @return - * @throws Exception - */ - public List<ArtifactResult> getArtifactsWithDep(String dependency, - Collection<String> excludes) throws Exception { - Artifact artifact = new DefaultArtifact(inferScalaVersion(dependency)); - DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter( JavaScopes.COMPILE ); - PatternExclusionsDependencyFilter exclusionFilter = - new PatternExclusionsDependencyFilter(inferScalaVersion(excludes)); - - CollectRequest collectRequest = new CollectRequest(); - collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE)); - - synchronized (repos) { - for (RemoteRepository repo : repos) { - collectRequest.addRepository(repo); - } - } - DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, - DependencyFilterUtils.andFilter(exclusionFilter, classpathFlter)); - return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); - } - - public static Collection<String> inferScalaVersion(Collection<String> artifact) { - List<String> list = new LinkedList<String>(); - for (String a : artifact) { - list.add(inferScalaVersion(a)); - } - return list; - } - - public static String inferScalaVersion(String artifact) { - int pos = artifact.indexOf(":"); - if (pos < 0 || pos + 2 >= artifact.length()) { - // failed to infer - return artifact; - } - - if (':' == artifact.charAt(pos + 1)) { - String restOfthem = ""; - String versionSep = ":"; - - String groupId = artifact.substring(0, pos); - int nextPos = artifact.indexOf(":", pos + 2); - if (nextPos < 0) { - if (artifact.charAt(artifact.length() - 1) == '*') { - nextPos = artifact.length() - 1; - versionSep = ""; - restOfthem = "*"; - } else { - versionSep = ""; - nextPos = artifact.length(); - } - } - - String artifactId = artifact.substring(pos + 2, nextPos); - if (nextPos < artifact.length()) { - if (!restOfthem.equals("*")) { - restOfthem = artifact.substring(nextPos + 1); - } - } - - String [] version = scala.util.Properties.versionNumberString().split("[.]"); - String scalaVersion = version[0] + "." + version[1]; - - return groupId + ":" + artifactId + "_" + scalaVersion + versionSep + restOfthem; - } else { - return artifact; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/Repository.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/Repository.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/Repository.java deleted file mode 100644 index aee56b5..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/Repository.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; -import org.sonatype.aether.repository.Authentication; -/** - * - * - */ -public class Repository { - private boolean snapshot = false; - private String name; - private String url; - private String username = null; - private String password = null; - - public Repository(String name){ - this.name = name; - } - - public Repository url(String url) { - this.url = url; - return this; - } - - public Repository snapshot() { - snapshot = true; - return this; - } - - public boolean isSnapshot() { - return snapshot; - } - - public String getName() { - return name; - } - - public String getUrl() { - return url; - } - - public Repository username(String username) { - this.username = username; - return this; - } - - public Repository password(String password) { - this.password = password; - return this; - } - - public Repository credentials(String username, String password) { - this.username = username; - this.password = password; - return this; - } - - protected Authentication getAuthentication() { - Authentication auth = null; - if (this.username != null && this.password != null) { - auth = new Authentication(this.username, this.password); - } - return auth; - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositoryListener.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositoryListener.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositoryListener.java deleted file mode 100644 index 86b3334..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositoryListener.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.AbstractRepositoryListener; -import org.sonatype.aether.RepositoryEvent; - -/** - * Simple listener that print log. - */ -public class RepositoryListener extends AbstractRepositoryListener { - Logger logger = LoggerFactory.getLogger(RepositoryListener.class); - - public RepositoryListener() {} - - @Override - public void artifactDeployed(RepositoryEvent event) { - logger.info("Deployed " + event.getArtifact() + " to " + event.getRepository()); - } - - @Override - public void artifactDeploying(RepositoryEvent event) { - logger.info("Deploying " + event.getArtifact() + " to " + event.getRepository()); - } - - @Override - public void artifactDescriptorInvalid(RepositoryEvent event) { - logger.info("Invalid artifact descriptor for " + event.getArtifact() + ": " - + event.getException().getMessage()); - } - - @Override - public void artifactDescriptorMissing(RepositoryEvent event) { - logger.info("Missing artifact descriptor for " + event.getArtifact()); - } - - @Override - public void artifactInstalled(RepositoryEvent event) { - logger.info("Installed " + event.getArtifact() + " to " + event.getFile()); - } - - @Override - public void artifactInstalling(RepositoryEvent event) { - logger.info("Installing " + event.getArtifact() + " to " + event.getFile()); - } - - @Override - public void artifactResolved(RepositoryEvent event) { - logger.info("Resolved artifact " + event.getArtifact() + " from " + event.getRepository()); - } - - @Override - public void artifactDownloading(RepositoryEvent event) { - logger.info("Downloading artifact " + event.getArtifact() + " from " + event.getRepository()); - } - - @Override - public void artifactDownloaded(RepositoryEvent event) { - logger.info("Downloaded artifact " + event.getArtifact() + " from " + event.getRepository()); - } - - @Override - public void artifactResolving(RepositoryEvent event) { - logger.info("Resolving artifact " + event.getArtifact()); - } - - @Override - public void metadataDeployed(RepositoryEvent event) { - logger.info("Deployed " + event.getMetadata() + " to " + event.getRepository()); - } - - @Override - public void metadataDeploying(RepositoryEvent event) { - logger.info("Deploying " + event.getMetadata() + " to " + event.getRepository()); - } - - @Override - public void metadataInstalled(RepositoryEvent event) { - logger.info("Installed " + event.getMetadata() + " to " + event.getFile()); - } - - @Override - public void metadataInstalling(RepositoryEvent event) { - logger.info("Installing " + event.getMetadata() + " to " + event.getFile()); - } - - @Override - public void metadataInvalid(RepositoryEvent event) { - logger.info("Invalid metadata " + event.getMetadata()); - } - - @Override - public void metadataResolved(RepositoryEvent event) { - logger.info("Resolved metadata " + event.getMetadata() + " from " + event.getRepository()); - } - - @Override - public void metadataResolving(RepositoryEvent event) { - logger.info("Resolving metadata " + event.getMetadata() + " from " + event.getRepository()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositorySystemFactory.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositorySystemFactory.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositorySystemFactory.java deleted file mode 100644 index cc0740d..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/RepositorySystemFactory.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import org.apache.maven.repository.internal.DefaultServiceLocator; -import org.apache.maven.wagon.Wagon; -import org.apache.maven.wagon.providers.http.HttpWagon; -import org.apache.maven.wagon.providers.http.LightweightHttpWagon; -import org.sonatype.aether.RepositorySystem; -import org.sonatype.aether.connector.file.FileRepositoryConnectorFactory; -import org.sonatype.aether.connector.wagon.WagonProvider; -import org.sonatype.aether.connector.wagon.WagonRepositoryConnectorFactory; -import org.sonatype.aether.spi.connector.RepositoryConnectorFactory; - -/** - * Get maven repository instance. - */ -public class RepositorySystemFactory { - public static RepositorySystem newRepositorySystem() { - DefaultServiceLocator locator = new DefaultServiceLocator(); - locator.addService(RepositoryConnectorFactory.class, FileRepositoryConnectorFactory.class); - locator.addService(RepositoryConnectorFactory.class, WagonRepositoryConnectorFactory.class); - locator.setServices(WagonProvider.class, new ManualWagonProvider()); - - return locator.getService(RepositorySystem.class); - } - - /** - * ManualWagonProvider - */ - public static class ManualWagonProvider implements WagonProvider { - - @Override - public Wagon lookup(String roleHint) throws Exception { - if ("http".equals(roleHint)) { - return new LightweightHttpWagon(); - } - - if ("https".equals(roleHint)) { - return new HttpWagon(); - } - - return null; - } - - @Override - public void release(Wagon arg0) { - - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java new file mode 100644 index 0000000..1b20b0f --- /dev/null +++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark.dep; + +import java.io.File; +import java.net.MalformedURLException; +import java.util.LinkedList; +import java.util.List; + +import org.apache.zeppelin.dep.Booter; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.Repository; + +import org.sonatype.aether.RepositorySystem; +import org.sonatype.aether.RepositorySystemSession; +import org.sonatype.aether.artifact.Artifact; +import org.sonatype.aether.collection.CollectRequest; +import org.sonatype.aether.graph.DependencyFilter; +import org.sonatype.aether.repository.RemoteRepository; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.resolution.ArtifactResolutionException; +import org.sonatype.aether.resolution.ArtifactResult; +import org.sonatype.aether.resolution.DependencyRequest; +import org.sonatype.aether.resolution.DependencyResolutionException; +import org.sonatype.aether.util.artifact.DefaultArtifact; +import org.sonatype.aether.util.artifact.JavaScopes; +import org.sonatype.aether.util.filter.DependencyFilterUtils; +import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; + + +/** + * + */ +public class SparkDependencyContext { + List<Dependency> dependencies = new LinkedList<Dependency>(); + List<Repository> repositories = new LinkedList<Repository>(); + + List<File> files = new LinkedList<File>(); + List<File> filesDist = new LinkedList<File>(); + private RepositorySystem system = Booter.newRepositorySystem(); + private RepositorySystemSession session; + private RemoteRepository mavenCentral = Booter.newCentralRepository(); + private RemoteRepository mavenLocal = Booter.newLocalRepository(); + private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>(); + + public SparkDependencyContext(String localRepoPath, String additionalRemoteRepository) { + session = Booter.newRepositorySystemSession(system, localRepoPath); + addRepoFromProperty(additionalRemoteRepository); + } + + public Dependency load(String lib) { + Dependency dep = new Dependency(lib); + + if (dependencies.contains(dep)) { + dependencies.remove(dep); + } + dependencies.add(dep); + return dep; + } + + public Repository addRepo(String name) { + Repository rep = new Repository(name); + repositories.add(rep); + return rep; + } + + public void reset() { + dependencies = new LinkedList<Dependency>(); + repositories = new LinkedList<Repository>(); + + files = new LinkedList<File>(); + filesDist = new LinkedList<File>(); + } + + private void addRepoFromProperty(String listOfRepo) { + if (listOfRepo != null) { + String[] repos = listOfRepo.split(";"); + for (String repo : repos) { + String[] parts = repo.split(","); + if (parts.length == 3) { + String id = parts[0].trim(); + String url = parts[1].trim(); + boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); + if (id.length() > 1 && url.length() > 1) { + RemoteRepository rr = new RemoteRepository(id, "default", url); + rr.setPolicy(isSnapshot, null); + additionalRepos.add(rr); + } + } + } + } + } + + /** + * fetch all artifacts + * @return + * @throws MalformedURLException + * @throws ArtifactResolutionException + * @throws DependencyResolutionException + */ + public List<File> fetch() throws MalformedURLException, + DependencyResolutionException, ArtifactResolutionException { + + for (Dependency dep : dependencies) { + if (!dep.isLocalFsArtifact()) { + List<ArtifactResult> artifacts = fetchArtifactWithDep(dep); + for (ArtifactResult artifact : artifacts) { + if (dep.isDist()) { + filesDist.add(artifact.getArtifact().getFile()); + } + files.add(artifact.getArtifact().getFile()); + } + } else { + if (dep.isDist()) { + filesDist.add(new File(dep.getGroupArtifactVersion())); + } + files.add(new File(dep.getGroupArtifactVersion())); + } + } + + return files; + } + + private List<ArtifactResult> fetchArtifactWithDep(Dependency dep) + throws DependencyResolutionException, ArtifactResolutionException { + Artifact artifact = new DefaultArtifact( + SparkDependencyResolver.inferScalaVersion(dep.getGroupArtifactVersion())); + + DependencyFilter classpathFlter = DependencyFilterUtils + .classpathFilter(JavaScopes.COMPILE); + PatternExclusionsDependencyFilter exclusionFilter = new PatternExclusionsDependencyFilter( + SparkDependencyResolver.inferScalaVersion(dep.getExclusions())); + + CollectRequest collectRequest = new CollectRequest(); + collectRequest.setRoot(new org.sonatype.aether.graph.Dependency(artifact, + JavaScopes.COMPILE)); + + collectRequest.addRepository(mavenCentral); + collectRequest.addRepository(mavenLocal); + for (RemoteRepository repo : additionalRepos) { + collectRequest.addRepository(repo); + } + for (Repository repo : repositories) { + RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl()); + rr.setPolicy(repo.isSnapshot(), null); + Authentication auth = repo.getAuthentication(); + if (auth != null) { + rr.setAuthentication(auth); + } + collectRequest.addRepository(rr); + } + + DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, + DependencyFilterUtils.andFilter(exclusionFilter, classpathFlter)); + + return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); + } + + public List<File> getFiles() { + return files; + } + + public List<File> getFilesDist() { + return filesDist; + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java new file mode 100644 index 0000000..e4881d3 --- /dev/null +++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark.dep; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.spark.SparkContext; +import org.apache.spark.repl.SparkIMain; +import org.apache.zeppelin.dep.AbstractDependencyResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.artifact.Artifact; +import org.sonatype.aether.collection.CollectRequest; +import org.sonatype.aether.graph.Dependency; +import org.sonatype.aether.graph.DependencyFilter; +import org.sonatype.aether.repository.RemoteRepository; +import org.sonatype.aether.resolution.ArtifactResult; +import org.sonatype.aether.resolution.DependencyRequest; +import org.sonatype.aether.util.artifact.DefaultArtifact; +import org.sonatype.aether.util.artifact.JavaScopes; +import org.sonatype.aether.util.filter.DependencyFilterUtils; +import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; + +import scala.Some; +import scala.collection.IndexedSeq; +import scala.reflect.io.AbstractFile; +import scala.tools.nsc.Global; +import scala.tools.nsc.backend.JavaPlatform; +import scala.tools.nsc.util.ClassPath; +import scala.tools.nsc.util.MergedClassPath; + +/** + * Deps resolver. + * Add new dependencies from mvn repo (at runtime) to Spark interpreter group. + */ +public class SparkDependencyResolver extends AbstractDependencyResolver { + Logger logger = LoggerFactory.getLogger(SparkDependencyResolver.class); + private Global global; + private SparkIMain intp; + private SparkContext sc; + + private final String[] exclusions = new String[] {"org.scala-lang:scala-library", + "org.scala-lang:scala-compiler", + "org.scala-lang:scala-reflect", + "org.scala-lang:scalap", + "org.apache.zeppelin:zeppelin-zengine", + "org.apache.zeppelin:zeppelin-spark", + "org.apache.zeppelin:zeppelin-server"}; + + public SparkDependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath, + String additionalRemoteRepository) { + super(localRepoPath); + this.intp = intp; + this.global = intp.global(); + this.sc = sc; + addRepoFromProperty(additionalRemoteRepository); + } + + private void addRepoFromProperty(String listOfRepo) { + if (listOfRepo != null) { + String[] repos = listOfRepo.split(";"); + for (String repo : repos) { + String[] parts = repo.split(","); + if (parts.length == 3) { + String id = parts[0].trim(); + String url = parts[1].trim(); + boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); + if (id.length() > 1 && url.length() > 1) { + addRepo(id, url, isSnapshot); + } + } + } + } + } + + private void updateCompilerClassPath(URL[] urls) throws IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + + JavaPlatform platform = (JavaPlatform) global.platform(); + MergedClassPath<AbstractFile> newClassPath = mergeUrlsIntoClassPath(platform, urls); + + Method[] methods = platform.getClass().getMethods(); + for (Method m : methods) { + if (m.getName().endsWith("currentClassPath_$eq")) { + m.invoke(platform, new Some(newClassPath)); + break; + } + } + + // NOTE: Must use reflection until this is exposed/fixed upstream in Scala + List<String> classPaths = new LinkedList<String>(); + for (URL url : urls) { + classPaths.add(url.getPath()); + } + + // Reload all jars specified into our compiler + global.invalidateClassPathEntries(scala.collection.JavaConversions.asScalaBuffer(classPaths) + .toList()); + } + + // Until spark 1.1.x + // check https://github.com/apache/spark/commit/191d7cf2a655d032f160b9fa181730364681d0e7 + private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException, + IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException { + ClassLoader cl = intp.classLoader().getParent(); + Method addURL; + addURL = cl.getClass().getDeclaredMethod("addURL", new Class[] {URL.class}); + addURL.setAccessible(true); + for (URL url : urls) { + addURL.invoke(cl, url); + } + } + + private void updateRuntimeClassPath_2_x(URL[] urls) throws SecurityException, + IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException { + ClassLoader cl = intp.classLoader().getParent(); + Method addURL; + addURL = cl.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class}); + addURL.setAccessible(true); + for (URL url : urls) { + addURL.invoke(cl, url); + } + } + + private MergedClassPath<AbstractFile> mergeUrlsIntoClassPath(JavaPlatform platform, URL[] urls) { + IndexedSeq<ClassPath<AbstractFile>> entries = + ((MergedClassPath<AbstractFile>) platform.classPath()).entries(); + List<ClassPath<AbstractFile>> cp = new LinkedList<ClassPath<AbstractFile>>(); + + for (int i = 0; i < entries.size(); i++) { + cp.add(entries.apply(i)); + } + + for (URL url : urls) { + AbstractFile file; + if ("file".equals(url.getProtocol())) { + File f = new File(url.getPath()); + if (f.isDirectory()) { + file = AbstractFile.getDirectory(scala.reflect.io.File.jfile2path(f)); + } else { + file = AbstractFile.getFile(scala.reflect.io.File.jfile2path(f)); + } + } else { + file = AbstractFile.getURL(url); + } + + ClassPath<AbstractFile> newcp = platform.classPath().context().newClassPath(file); + + // distinct + if (cp.contains(newcp) == false) { + cp.add(newcp); + } + } + + return new MergedClassPath(scala.collection.JavaConversions.asScalaBuffer(cp).toIndexedSeq(), + platform.classPath().context()); + } + + public List<String> load(String artifact, + boolean addSparkContext) throws Exception { + return load(artifact, new LinkedList<String>(), addSparkContext); + } + + public List<String> load(String artifact, Collection<String> excludes, + boolean addSparkContext) throws Exception { + if (StringUtils.isBlank(artifact)) { + // Should throw here + throw new RuntimeException("Invalid artifact to load"); + } + + // <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version> + int numSplits = artifact.split(":").length; + if (numSplits >= 3 && numSplits <= 6) { + return loadFromMvn(artifact, excludes, addSparkContext); + } else { + loadFromFs(artifact, addSparkContext); + LinkedList<String> libs = new LinkedList<String>(); + libs.add(artifact); + return libs; + } + } + + private void loadFromFs(String artifact, boolean addSparkContext) throws Exception { + File jarFile = new File(artifact); + + intp.global().new Run(); + + if (sc.version().startsWith("1.1")) { + updateRuntimeClassPath_1_x(new URL[] {jarFile.toURI().toURL()}); + } else { + updateRuntimeClassPath_2_x(new URL[] {jarFile.toURI().toURL()}); + } + + if (addSparkContext) { + sc.addJar(jarFile.getAbsolutePath()); + } + } + + private List<String> loadFromMvn(String artifact, Collection<String> excludes, + boolean addSparkContext) throws Exception { + List<String> loadedLibs = new LinkedList<String>(); + Collection<String> allExclusions = new LinkedList<String>(); + allExclusions.addAll(excludes); + allExclusions.addAll(Arrays.asList(exclusions)); + + List<ArtifactResult> listOfArtifact; + listOfArtifact = getArtifactsWithDep(artifact, allExclusions); + + Iterator<ArtifactResult> it = listOfArtifact.iterator(); + while (it.hasNext()) { + Artifact a = it.next().getArtifact(); + String gav = a.getGroupId() + ":" + a.getArtifactId() + ":" + a.getVersion(); + for (String exclude : allExclusions) { + if (gav.startsWith(exclude)) { + it.remove(); + break; + } + } + } + + List<URL> newClassPathList = new LinkedList<URL>(); + List<File> files = new LinkedList<File>(); + for (ArtifactResult artifactResult : listOfArtifact) { + logger.info("Load " + artifactResult.getArtifact().getGroupId() + ":" + + artifactResult.getArtifact().getArtifactId() + ":" + + artifactResult.getArtifact().getVersion()); + newClassPathList.add(artifactResult.getArtifact().getFile().toURI().toURL()); + files.add(artifactResult.getArtifact().getFile()); + loadedLibs.add(artifactResult.getArtifact().getGroupId() + ":" + + artifactResult.getArtifact().getArtifactId() + ":" + + artifactResult.getArtifact().getVersion()); + } + + intp.global().new Run(); + if (sc.version().startsWith("1.1")) { + updateRuntimeClassPath_1_x(newClassPathList.toArray(new URL[0])); + } else { + updateRuntimeClassPath_2_x(newClassPathList.toArray(new URL[0])); + } + updateCompilerClassPath(newClassPathList.toArray(new URL[0])); + + if (addSparkContext) { + for (File f : files) { + sc.addJar(f.getAbsolutePath()); + } + } + + return loadedLibs; + } + + /** + * @param dependency + * @param excludes list of pattern can either be of the form groupId:artifactId + * @return + * @throws Exception + */ + @Override + public List<ArtifactResult> getArtifactsWithDep(String dependency, + Collection<String> excludes) throws Exception { + Artifact artifact = new DefaultArtifact(inferScalaVersion(dependency)); + DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE); + PatternExclusionsDependencyFilter exclusionFilter = + new PatternExclusionsDependencyFilter(inferScalaVersion(excludes)); + + CollectRequest collectRequest = new CollectRequest(); + collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE)); + + synchronized (repos) { + for (RemoteRepository repo : repos) { + collectRequest.addRepository(repo); + } + } + DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, + DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter)); + return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); + } + + public static Collection<String> inferScalaVersion(Collection<String> artifact) { + List<String> list = new LinkedList<String>(); + for (String a : artifact) { + list.add(inferScalaVersion(a)); + } + return list; + } + + public static String inferScalaVersion(String artifact) { + int pos = artifact.indexOf(":"); + if (pos < 0 || pos + 2 >= artifact.length()) { + // failed to infer + return artifact; + } + + if (':' == artifact.charAt(pos + 1)) { + String restOfthem = ""; + String versionSep = ":"; + + String groupId = artifact.substring(0, pos); + int nextPos = artifact.indexOf(":", pos + 2); + if (nextPos < 0) { + if (artifact.charAt(artifact.length() - 1) == '*') { + nextPos = artifact.length() - 1; + versionSep = ""; + restOfthem = "*"; + } else { + versionSep = ""; + nextPos = artifact.length(); + } + } + + String artifactId = artifact.substring(pos + 2, nextPos); + if (nextPos < artifact.length()) { + if (!restOfthem.equals("*")) { + restOfthem = artifact.substring(nextPos + 1); + } + } + + String [] version = scala.util.Properties.versionNumberString().split("[.]"); + String scalaVersion = version[0] + "." + version[1]; + + return groupId + ":" + artifactId + "_" + scalaVersion + versionSep + restOfthem; + } else { + return artifact; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/main/java/org/apache/zeppelin/spark/dep/TransferListener.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/TransferListener.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/TransferListener.java deleted file mode 100644 index 8a4058a..0000000 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/TransferListener.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import java.io.PrintStream; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.transfer.AbstractTransferListener; -import org.sonatype.aether.transfer.TransferEvent; -import org.sonatype.aether.transfer.TransferResource; - -/** - * Simple listener that show deps downloading progress. - */ -public class TransferListener extends AbstractTransferListener { - Logger logger = LoggerFactory.getLogger(TransferListener.class); - private PrintStream out; - - private Map<TransferResource, Long> downloads = new ConcurrentHashMap<TransferResource, Long>(); - - private int lastLength; - - public TransferListener() {} - - @Override - public void transferInitiated(TransferEvent event) { - String message = - event.getRequestType() == TransferEvent.RequestType.PUT ? "Uploading" : "Downloading"; - - logger.info(message + ": " + event.getResource().getRepositoryUrl() - + event.getResource().getResourceName()); - } - - @Override - public void transferProgressed(TransferEvent event) { - TransferResource resource = event.getResource(); - downloads.put(resource, Long.valueOf(event.getTransferredBytes())); - - StringBuilder buffer = new StringBuilder(64); - - for (Map.Entry<TransferResource, Long> entry : downloads.entrySet()) { - long total = entry.getKey().getContentLength(); - long complete = entry.getValue().longValue(); - - buffer.append(getStatus(complete, total)).append(" "); - } - - int pad = lastLength - buffer.length(); - lastLength = buffer.length(); - pad(buffer, pad); - buffer.append('\r'); - - logger.info(buffer.toString()); - } - - private String getStatus(long complete, long total) { - if (total >= 1024) { - return toKB(complete) + "/" + toKB(total) + " KB "; - } else if (total >= 0) { - return complete + "/" + total + " B "; - } else if (complete >= 1024) { - return toKB(complete) + " KB "; - } else { - return complete + " B "; - } - } - - private void pad(StringBuilder buffer, int spaces) { - String block = " "; - while (spaces > 0) { - int n = Math.min(spaces, block.length()); - buffer.append(block, 0, n); - spaces -= n; - } - } - - @Override - public void transferSucceeded(TransferEvent event) { - transferCompleted(event); - - TransferResource resource = event.getResource(); - long contentLength = event.getTransferredBytes(); - if (contentLength >= 0) { - String type = - (event.getRequestType() == TransferEvent.RequestType.PUT ? "Uploaded" : "Downloaded"); - String len = contentLength >= 1024 ? toKB(contentLength) + " KB" : contentLength + " B"; - - String throughput = ""; - long duration = System.currentTimeMillis() - resource.getTransferStartTime(); - if (duration > 0) { - DecimalFormat format = new DecimalFormat("0.0", new DecimalFormatSymbols(Locale.ENGLISH)); - double kbPerSec = (contentLength / 1024.0) / (duration / 1000.0); - throughput = " at " + format.format(kbPerSec) + " KB/sec"; - } - - logger.info(type + ": " + resource.getRepositoryUrl() + resource.getResourceName() + " (" - + len + throughput + ")"); - } - } - - @Override - public void transferFailed(TransferEvent event) { - transferCompleted(event); - event.getException().printStackTrace(out); - } - - private void transferCompleted(TransferEvent event) { - downloads.remove(event.getResource()); - StringBuilder buffer = new StringBuilder(64); - pad(buffer, lastLength); - buffer.append('\r'); - logger.info(buffer.toString()); - } - - @Override - public void transferCorrupted(TransferEvent event) { - event.getException().printStackTrace(out); - } - - protected long toKB(long bytes) { - return (bytes + 1023) / 1024; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/test/java/org/apache/zeppelin/spark/dep/DependencyResolverTest.java ---------------------------------------------------------------------- diff --git a/spark/src/test/java/org/apache/zeppelin/spark/dep/DependencyResolverTest.java b/spark/src/test/java/org/apache/zeppelin/spark/dep/DependencyResolverTest.java deleted file mode 100644 index e41de60..0000000 --- a/spark/src/test/java/org/apache/zeppelin/spark/dep/DependencyResolverTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import static org.junit.Assert.assertEquals; - -import org.apache.zeppelin.spark.dep.DependencyResolver; -import org.junit.Test; - -public class DependencyResolverTest { - - @Test - public void testInferScalaVersion() { - String [] version = scala.util.Properties.versionNumberString().split("[.]"); - String scalaVersion = version[0] + "." + version[1]; - - assertEquals("groupId:artifactId:version", - DependencyResolver.inferScalaVersion("groupId:artifactId:version")); - assertEquals("groupId:artifactId_" + scalaVersion + ":version", - DependencyResolver.inferScalaVersion("groupId::artifactId:version")); - assertEquals("groupId:artifactId:version::test", - DependencyResolver.inferScalaVersion("groupId:artifactId:version::test")); - assertEquals("*", - DependencyResolver.inferScalaVersion("*")); - assertEquals("groupId:*", - DependencyResolver.inferScalaVersion("groupId:*")); - assertEquals("groupId:artifactId*", - DependencyResolver.inferScalaVersion("groupId:artifactId*")); - assertEquals("groupId:artifactId_" + scalaVersion, - DependencyResolver.inferScalaVersion("groupId::artifactId")); - assertEquals("groupId:artifactId_" + scalaVersion + "*", - DependencyResolver.inferScalaVersion("groupId::artifactId*")); - assertEquals("groupId:artifactId_" + scalaVersion + ":*", - DependencyResolver.inferScalaVersion("groupId::artifactId:*")); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java ---------------------------------------------------------------------- diff --git a/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java b/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java new file mode 100644 index 0000000..a0271f4 --- /dev/null +++ b/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark.dep; + +import static org.junit.Assert.assertEquals; + +import org.apache.zeppelin.spark.dep.SparkDependencyResolver; +import org.junit.Test; + +public class SparkDependencyResolverTest { + + @Test + public void testInferScalaVersion() { + String [] version = scala.util.Properties.versionNumberString().split("[.]"); + String scalaVersion = version[0] + "." + version[1]; + + assertEquals("groupId:artifactId:version", + SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version")); + assertEquals("groupId:artifactId_" + scalaVersion + ":version", + SparkDependencyResolver.inferScalaVersion("groupId::artifactId:version")); + assertEquals("groupId:artifactId:version::test", + SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version::test")); + assertEquals("*", + SparkDependencyResolver.inferScalaVersion("*")); + assertEquals("groupId:*", + SparkDependencyResolver.inferScalaVersion("groupId:*")); + assertEquals("groupId:artifactId*", + SparkDependencyResolver.inferScalaVersion("groupId:artifactId*")); + assertEquals("groupId:artifactId_" + scalaVersion, + SparkDependencyResolver.inferScalaVersion("groupId::artifactId")); + assertEquals("groupId:artifactId_" + scalaVersion + "*", + SparkDependencyResolver.inferScalaVersion("groupId::artifactId*")); + assertEquals("groupId:artifactId_" + scalaVersion + ":*", + SparkDependencyResolver.inferScalaVersion("groupId::artifactId:*")); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/bc715511/zeppelin-interpreter/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 59d2cd0..67b4d5f 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -97,6 +97,123 @@ <artifactId>commons-lang3</artifactId> <version>${commons-lang.version}</version> </dependency> + + <!-- Aether :: maven dependency resolution --> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>3.0</version> + <exclusions> + <exclusion> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + </exclusion> + <exclusion> + <groupId>org.sonatype.sisu</groupId> + <artifactId>sisu-inject-plexus</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.maven</groupId> + <artifactId>maven-model</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-api</artifactId> + <version>1.12</version> + </dependency> + + <dependency> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-util</artifactId> + <version>1.12</version> + </dependency> + + <dependency> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-impl</artifactId> + <version>1.12</version> + </dependency> + + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-aether-provider</artifactId> + <version>3.0.3</version> + <exclusions> + <exclusion> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-spi</artifactId> + </exclusion> + <exclusion> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-util</artifactId> + </exclusion> + <exclusion> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-impl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-connector-file</artifactId> + <version>1.12</version> + </dependency> + + <dependency> + <groupId>org.sonatype.aether</groupId> + <artifactId>aether-connector-wagon</artifactId> + <version>1.12</version> + <exclusions> + <exclusion> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-provider-api</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-provider-api</artifactId> + <version>1.0</version> + <exclusions> + <exclusion> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-http-lightweight</artifactId> + <version>1.0</version> + <exclusions> + <exclusion> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-http-shared</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-http</artifactId> + <version>1.0</version> + <exclusions> + </exclusions> + </dependency> </dependencies> <build>
