Repository: zeppelin Updated Branches: refs/heads/branch-0.8 90bd8cb16 -> e8291b154
ZEPPELIN-3347 Fix "PYTHONPATH" for spark.pyspark in branch-0.8 ### What is this PR for? Use system PYTHONPATH in spark.pyspark interpreter, if PYTHONPATH already exists in environment variables. ### What type of PR is it? [Bug Fix] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3347 ### How should this be tested? * Add some directories to PYTHONPATH `export PYTHONPATH="/opt/python/dir1:/opt/python/dir2:$PYTHONPATH"` * Build branch "ZEPPELIN-3347_0.8" * Run in %spark.pyspark ``` import sys print('\n'.join(sys.path)) ``` * Check that the directories are in the output list. Compare with the result in %python. ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Author: mebelousov <mebelou...@ya.ru> Closes #2907 from mebelousov/ZEPPELIN-3347_0.8 and squashes the following commits: 7200746 [Jeff Zhang] [HOTFIX] JavaDoc Fix for branch-0.8 8ada1e3 [mebelousov] ZEPPELIN-3347 Fix "PYTHONPATH" for spark.pyspark Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e8291b15 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e8291b15 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e8291b15 Branch: refs/heads/branch-0.8 Commit: e8291b1544672c1aff7e6a7c0822e0b8cc76af25 Parents: 90bd8cb Author: Jeff Zhang <zjf...@apache.org> Authored: Tue Apr 3 09:10:44 2018 +0300 Committer: Jeff Zhang <zjf...@apache.org> Committed: Wed Apr 18 10:29:42 2018 +0800 ---------------------------------------------------------------------- .../zeppelin/spark/PySparkInterpreter.java | 24 +++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e8291b15/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index a838667..58314ef 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -204,12 +204,17 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand // only set PYTHONPATH in local or yarn-client mode. // yarn-cluster will setup PYTHONPATH automatically. - SparkConf conf = getSparkConf(); + SparkConf conf = null; + try { + conf = getSparkConf(); + } catch (InterpreterException e) { + throw new IOException(e); + } if (!conf.get("spark.submit.deployMode", "client").equals("cluster")) { if (!env.containsKey("PYTHONPATH")) { env.put("PYTHONPATH", PythonUtils.sparkPythonPath()); } else { - env.put("PYTHONPATH", PythonUtils.sparkPythonPath()); + env.put("PYTHONPATH", PythonUtils.sparkPythonPath() + ":" + env.get("PYTHONPATH")); } } @@ -217,20 +222,17 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand // also, add all packages to PYTHONPATH since there might be transitive dependencies if (SparkInterpreter.useSparkSubmit() && !getSparkInterpreter().isYarnMode()) { - - String sparkSubmitJars = getSparkConf().get("spark.jars").replace(",", ":"); - - if (!"".equals(sparkSubmitJars)) { - env.put("PYTHONPATH", env.get("PYTHONPATH") + sparkSubmitJars); + String sparkSubmitJars = conf.get("spark.jars").replace(",", ":"); + if (!StringUtils.isEmpty(sparkSubmitJars)) { + env.put("PYTHONPATH", env.get("PYTHONPATH") + ":" + sparkSubmitJars); } } - LOGGER.info("PYTHONPATH: " + env.get("PYTHONPATH")); - // set PYSPARK_PYTHON - if (getSparkConf().contains("spark.pyspark.python")) { - env.put("PYSPARK_PYTHON", getSparkConf().get("spark.pyspark.python")); + if (conf.contains("spark.pyspark.python")) { + env.put("PYSPARK_PYTHON", conf.get("spark.pyspark.python")); } + LOGGER.info("PYTHONPATH: " + env.get("PYTHONPATH")); return env; }
