This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.10
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.10 by this push:
new 366b812 [ZEPPELIN-5580] Pass scala version from
SparkInterpreterLauncher instead of detect it at runtime
366b812 is described below
commit 366b812c2840ea4836fd31df95b0436ffa8a52b9
Author: huage1994 <[email protected]>
AuthorDate: Mon Nov 22 17:46:41 2021 +0800
[ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of
detect it at runtime
Currently we detect scala version via scala.util.Properties.versionString();
but it depends on the resource file library.version on classpath, sometimes
user may package this resource of scala-2.11 into his jar which cause we detect
the wrong scala version.
[Bug Fix]
* [ ] - Task
https://issues.apache.org/jira/browse/ZEPPELIN-5580
CI pass
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: huage1994 <[email protected]>
Closes #4267 from huage1994/ZEPPELIN-5580 and squashes the following
commits:
7945faa7e6 [huage1994] [ZEPPELIN-5580] Pass scala version from
SparkInterpreterLauncher instead of detect it at runtime
(cherry picked from commit d5d26583f723d9638d69020aab885afefab82c5c)
Signed-off-by: Jeff Zhang <[email protected]>
---
.../apache/zeppelin/spark/SparkInterpreter.java | 27 ++++++++++++++++------
.../launcher/SparkInterpreterLauncher.java | 9 +++++++-
2 files changed, 28 insertions(+), 8 deletions(-)
diff --git
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 7b1460a..a4c9b76 100644
---
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -74,6 +74,7 @@ public class SparkInterpreter extends AbstractInterpreter {
private Object sparkSession;
private SparkVersion sparkVersion;
+ private String scalaVersion;
private boolean enableSupportedVersionCheck;
public SparkInterpreter(Properties properties) {
@@ -151,7 +152,7 @@ public class SparkInterpreter extends AbstractInterpreter {
* @throws Exception
*/
private AbstractSparkScalaInterpreter loadSparkScalaInterpreter(SparkConf
conf) throws Exception {
- String scalaVersion = extractScalaVersion();
+ scalaVersion = extractScalaVersion(conf);
ClassLoader scalaInterpreterClassLoader =
Thread.currentThread().getContextClassLoader();
String zeppelinHome = System.getenv("ZEPPELIN_HOME");
@@ -254,14 +255,26 @@ public class SparkInterpreter extends AbstractInterpreter
{
return sparkVersion;
}
- private String extractScalaVersion() throws InterpreterException {
- String scalaVersionString = scala.util.Properties.versionString();
+ private String extractScalaVersion(SparkConf conf) throws
InterpreterException {
+ // Use the scala version if SparkLauncher pass it by name of
"zeppelin.spark.scala.version".
+
+ // If not, detect scala version by resource file library.version on
classpath.
+ // Library.version is sometimes inaccurate and it is mainly used for unit
test.
+ String scalaVersionString;
+ if (conf.contains("zeppelin.spark.scala.version")) {
+ scalaVersionString = conf.get("zeppelin.spark.scala.version");
+ } else {
+ scalaVersionString = scala.util.Properties.versionString();
+ }
LOGGER.info("Using Scala: " + scalaVersionString);
- if (scalaVersionString.contains("version 2.10")) {
+
+ if (StringUtils.isEmpty(scalaVersionString)) {
+ throw new InterpreterException("Scala Version is empty");
+ } else if (scalaVersionString.contains("2.10")) {
return "2.10";
- } else if (scalaVersionString.contains("version 2.11")) {
+ } else if (scalaVersionString.contains("2.11")) {
return "2.11";
- } else if (scalaVersionString.contains("version 2.12")) {
+ } else if (scalaVersionString.contains("2.12")) {
return "2.12";
} else {
throw new InterpreterException("Unsupported scala version: " +
scalaVersionString);
@@ -269,7 +282,7 @@ public class SparkInterpreter extends AbstractInterpreter {
}
public boolean isScala212() throws InterpreterException {
- return extractScalaVersion().equals("2.12");
+ return scalaVersion.equals("2.12");
}
public boolean isScala210() throws InterpreterException {
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 50bdc88..2aa7ccb 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -118,6 +118,14 @@ public class SparkInterpreterLauncher extends
StandardInterpreterLauncher {
}
+ String scalaVersion = null;
+ try {
+ scalaVersion = detectSparkScalaVersion(getEnv("SPARK_HOME"), env);
+ context.getProperties().put("zeppelin.spark.scala.version",
scalaVersion);
+ } catch (Exception e) {
+ throw new IOException("Fail to detect scala version, the reason is:"+
e.getMessage());
+ }
+
if (isYarnMode()
&& getDeployMode().equals("cluster")) {
try {
@@ -133,7 +141,6 @@ public class SparkInterpreterLauncher extends
StandardInterpreterLauncher {
}
}
- String scalaVersion = detectSparkScalaVersion(getEnv("SPARK_HOME"),
env);
Path scalaFolder = Paths.get(zConf.getZeppelinHome(),
"/interpreter/spark/scala-" + scalaVersion);
if (!scalaFolder.toFile().exists()) {
throw new IOException("spark scala folder " + scalaFolder.toFile() +
" doesn't exist");
