Repository: zeppelin Updated Branches: refs/heads/branch-0.8 32698dc19 -> ab704a1e2
ZEPPELIN-3531. Don't look for py4j in NewSparkInterpreter ### What is this PR for? Just remove setupConfForPySpark in NewSparkInterpreter as it is not necessary and will cause NPE when the node launch spark interpreter doesn't have spark installed in yarn cluster mode. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3531 ### How should this be tested? * CI pass & Manually tested in a 3 node cluster ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Closes #3008 from zjffdu/ZEPPELIN-3531 and squashes the following commits: e8b2969e9 [Jeff Zhang] ZEPPELIN-3531. Don't look for py4j in NewSparkInterpreter (cherry picked from commit 767a50b4164ef323ef18160485d226c013c7c00d) Signed-off-by: Jeff Zhang <zjf...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/ab704a1e Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/ab704a1e Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/ab704a1e Branch: refs/heads/branch-0.8 Commit: ab704a1e27d23a737155185ac274fc710f892f26 Parents: 32698dc Author: Jeff Zhang <zjf...@apache.org> Authored: Tue Jun 5 21:38:17 2018 +0800 Committer: Jeff Zhang <zjf...@apache.org> Committed: Thu Jun 7 16:19:26 2018 +0800 ---------------------------------------------------------------------- .../zeppelin/spark/NewSparkInterpreter.java | 43 -------------------- 1 file changed, 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ab704a1e/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java index 2fa1093..2ea79f0 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java @@ -81,7 +81,6 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { try { String scalaVersion = extractScalaVersion(); LOGGER.info("Using Scala Version: " + scalaVersion); - setupConfForPySpark(); SparkConf conf = new SparkConf(); for (Map.Entry<Object, Object> entry : getProperties().entrySet()) { if (!StringUtils.isBlank(entry.getValue().toString())) { @@ -126,48 +125,6 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter { } } - private void setupConfForPySpark() { - String sparkHome = getProperty("SPARK_HOME"); - File pysparkFolder = null; - if (sparkHome == null) { - String zeppelinHome = - new DefaultInterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../../") - .getValue().toString(); - pysparkFolder = new File(zeppelinHome, - "interpreter" + File.separator + "spark" + File.separator + "pyspark"); - } else { - pysparkFolder = new File(sparkHome, "python" + File.separator + "lib"); - } - - ArrayList<String> pysparkPackages = new ArrayList<>(); - for (File file : pysparkFolder.listFiles()) { - if (file.getName().equals("pyspark.zip")) { - pysparkPackages.add(file.getAbsolutePath()); - } - if (file.getName().startsWith("py4j-")) { - pysparkPackages.add(file.getAbsolutePath()); - } - } - - if (pysparkPackages.size() != 2) { - throw new RuntimeException("Not correct number of pyspark packages: " + - StringUtils.join(pysparkPackages, ",")); - } - // Distribute two libraries(pyspark.zip and py4j-*.zip) to workers - System.setProperty("spark.files", mergeProperty(System.getProperty("spark.files", ""), - StringUtils.join(pysparkPackages, ","))); - System.setProperty("spark.submit.pyFiles", mergeProperty( - System.getProperty("spark.submit.pyFiles", ""), StringUtils.join(pysparkPackages, ","))); - - } - - private String mergeProperty(String originalValue, String appendedValue) { - if (StringUtils.isBlank(originalValue)) { - return appendedValue; - } - return originalValue + "," + appendedValue; - } - @Override public void close() { LOGGER.info("Close SparkInterpreter");
