Repository: zeppelin Updated Branches: refs/heads/master bbf5ef511 -> b06e97c97
ZEPPELIN-3517 Remove zeppelin.pyspark.python in PySparkInterpreter ### What is this PR for? `zeppelin.pyspark.python` should be removed as it is zeppelin specific property, and only affect the driver, but not on executor. So we should use spark property instead. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3517 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Closes #2993 from zjffdu/ZEPPELIN-3517 and squashes the following commits: 24dafa199 [Jeff Zhang] ZEPPELIN-3517. Remove zeppelin.pyspark.python in PySparkInterpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/b06e97c9 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/b06e97c9 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/b06e97c9 Branch: refs/heads/master Commit: b06e97c9750c17cdbf98b21aed0c3407277c25b7 Parents: bbf5ef5 Author: Jeff Zhang <zjf...@apache.org> Authored: Thu May 31 13:24:43 2018 +0800 Committer: Jeff Zhang <zjf...@apache.org> Committed: Thu Jun 7 18:35:52 2018 +0800 ---------------------------------------------------------------------- .../apache/zeppelin/spark/PySparkInterpreter.java | 15 ++++++++++----- .../src/main/resources/interpreter-setting.json | 11 +++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b06e97c9/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index beebd42..1df6e2e 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -180,17 +180,22 @@ public class PySparkInterpreter extends PythonInterpreter { // Run python shell // Choose python in the order of - // PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python + // spark.pyspark.driver.python > spark.pyspark.python > PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON @Override protected String getPythonExec() { - String pythonExec = getProperty("zeppelin.pyspark.python", "python"); + if (!StringUtils.isBlank(getProperty("spark.pyspark.driver.python", ""))) { + return properties.getProperty("spark.pyspark.driver.python"); + } + if (!StringUtils.isBlank(getProperty("spark.pyspark.python", ""))) { + return properties.getProperty("spark.pyspark.python"); + } if (System.getenv("PYSPARK_PYTHON") != null) { - pythonExec = System.getenv("PYSPARK_PYTHON"); + return System.getenv("PYSPARK_PYTHON"); } if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) { - pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON"); + return System.getenv("PYSPARK_DRIVER_PYTHON"); } - return pythonExec; + return "python"; } @Override http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b06e97c9/spark/interpreter/src/main/resources/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/resources/interpreter-setting.json b/spark/interpreter/src/main/resources/interpreter-setting.json index 8cd82bb..8791ece 100644 --- a/spark/interpreter/src/main/resources/interpreter-setting.json +++ b/spark/interpreter/src/main/resources/interpreter-setting.json @@ -170,9 +170,16 @@ "name": "pyspark", "className": "org.apache.zeppelin.spark.PySparkInterpreter", "properties": { - "zeppelin.pyspark.python": { + "PYSPARK_PYTHON": { "envName": "PYSPARK_PYTHON", - "propertyName": null, + "propertyName": "PYSPARK_PYTHON", + "defaultValue": "python", + "description": "Python command to run pyspark with", + "type": "string" + }, + "PYSPARK_DRIVER_PYTHON": { + "envName": "PYSPARK_DRIVER_PYTHON", + "propertyName": "PYSPARK_DRIVER_PYTHON", "defaultValue": "python", "description": "Python command to run pyspark with", "type": "string"
