Repository: zeppelin Updated Branches: refs/heads/branch-0.8 9721d174a -> 2ca8d19ba
[ZEPPELIN-3810] Support Spark 2.4 Spark 2.4 changed it's Scala version from 2.11.8 to 2.11.12 (see SPARK-24418). There are two problems for this upgrade at Zeppelin side: 1.. Some methods that are used in private by reflection, for instance, `loopPostInit` became inaccessible. See: - https://github.com/scala/scala/blob/v2.11.8/src/repl/scala/tools/nsc/interpreter/ILoop.scala - https://github.com/scala/scala/blob/v2.11.12/src/repl/scala/tools/nsc/interpreter/ILoop.scala To work around this, I manually ported `loopPostInit` at 2.11.8 to retain the behaviour. Some functions that are commonly existing at both Scala 2.11.8 and Scala 2.11.12 are used inside of the new `loopPostInit` by reflection. 2.. Upgrade from 2.11.8 to 2.11.12 requires `jline.version` upgrade. Otherwise, we will hit: ``` Caused by: java.lang.NoSuchMethodError: jline.console.completer.CandidateListCompletionHandler.setPrintSpaceAfterFullCompletion(Z)V at scala.tools.nsc.interpreter.jline.JLineConsoleReader.initCompletion(JLineReader.scala:139) ``` To work around this, I tweaked this by upgrading jline from `2.12.1` to `2.14.3`. [Improvement] * [x] - Wait until Spark 2.4.0 is officially released. * https://issues.apache.org/jira/browse/ZEPPELIN-3810 Verified manually against Spark 2.4.0 RC3 * Does the licenses files need update? Yes * Is there breaking changes for older versions? No * Does this needs documentation? No Author: hyukjinkwon <gurwls...@apache.org> Author: Hyukjin Kwon <gurwls...@apache.org> Author: Jeff Zhang <zjf...@gmail.com> Closes #3206 from HyukjinKwon/ZEPPELIN-3810 and squashes the following commits: c2456c949 [Hyukjin Kwon] Py4J 0.10.6 to 0.10.7 573f07d2d [Jeff Zhang] add test for spark 2.4 (#1) 9ac1797a7 [hyukjinkwon] Support Spark 2.4 (cherry picked from commit 4f73272c006b3767ad40686aa5876ac5fbf57928) Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/2ca8d19b Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/2ca8d19b Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/2ca8d19b Branch: refs/heads/branch-0.8 Commit: 2ca8d19bab9ac096a0437d1f438a33041b1b9e87 Parents: 9721d17 Author: hyukjinkwon <gurwls...@apache.org> Authored: Fri Nov 9 23:18:52 2018 +0800 Committer: Jeff Zhang <zjf...@apache.org> Committed: Tue Nov 13 15:31:09 2018 +0800 ---------------------------------------------------------------------- .travis.yml | 13 ++-- spark/interpreter/pom.xml | 63 +++++++++++++++-- .../org/apache/zeppelin/spark/SparkVersion.java | 4 +- spark/pom.xml | 56 ++++++++------- .../spark/SparkScala211Interpreter.scala | 73 +++++++++++++++++++- zeppelin-distribution/src/bin_license/LICENSE | 2 +- zeppelin-interpreter/pom.xml | 2 +- .../zeppelin/rest/ZeppelinSparkClusterTest.java | 8 +-- .../interpreter/SparkIntegrationTest.java | 11 +-- 9 files changed, 181 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index e1d0871..45ce87f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,9 @@ language: java sudo: false +before_cache: + - sudo chown -R travis:travis $HOME/.m2 + cache: apt: true directories: @@ -94,15 +97,15 @@ matrix: dist: trusty env: PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test spark module for 2.1.0 with scala 2.11 - - jdk: "openjdk7" + # Test spark module for 2.4.0 with scala 2.11 + - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11 -Pr" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies,r" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.rinterpreter.*,org.apache.spark.api.r.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.4 -Phadoop2 -Pscala-2.11 -Pr" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies,r" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.rinterpreter.*,org.apache.spark.api.r.* -DfailIfNoTests=false" - # Test spark module for 2.0.2 with scala 2.11 + # Test spark module for 2.3.2 with scala 2.11 - jdk: "oraclejdk8" dist: trusty - env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11 -Pr" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies,r" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.rinterpreter.*,org.apache.spark.api.r.* -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.3 -Phadoop3 -Pscala-2.11 -Pr" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies,r" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.rinterpreter.*,org.apache.spark.api.r.* -DfailIfNoTests=false" # Test python/pyspark with python 2, livy 0.5 - sudo: required http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/spark/interpreter/pom.xml ---------------------------------------------------------------------- diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index 99b1d3f..80997e88 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -53,7 +53,7 @@ <pyspark.test.exclude>**/PySparkInterpreterMatplotlibTest.java</pyspark.test.exclude> <pyspark.test.include>**/*Test.*</pyspark.test.include> - + </properties> <dependencies> @@ -71,12 +71,6 @@ <dependency> <groupId>org.apache.zeppelin</groupId> - <artifactId>spark-scala-2.10</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.zeppelin</groupId> <artifactId>zeppelin-interpreter</artifactId> <version>${project.version}</version> </dependency> @@ -586,4 +580,59 @@ </plugins> </build> + <profiles> + + <profile> + <id>spark-2.4</id> + </profile> + + <profile> + <id>spark-2.3</id> + </profile> + + <profile> + <id>spark-2.2</id> + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>spark-scala-2.10</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </profile> + + <profile> + <id>spark-2.1</id> + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>spark-scala-2.10</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </profile> + + <profile> + <id>spark-2.0</id> + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>spark-scala-2.10</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </profile> + + <profile> + <id>spark-1.6</id> + <dependencies> + <dependency> + <groupId>org.apache.zeppelin</groupId> + <artifactId>spark-scala-2.10</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </profile> + + </profiles> </project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java index 5e412eb..df3c754 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java @@ -36,9 +36,11 @@ public class SparkVersion { public static final SparkVersion SPARK_2_0_0 = SparkVersion.fromVersionString("2.0.0"); public static final SparkVersion SPARK_2_3_1 = SparkVersion.fromVersionString("2.3.1"); public static final SparkVersion SPARK_2_4_0 = SparkVersion.fromVersionString("2.4.0"); + public static final SparkVersion SPARK_3_0_0 = SparkVersion.fromVersionString("3.0.0"); public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_1_0_0; - public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_2_4_0; + public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_3_0_0; + private int version; private String versionString; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/spark/pom.xml ---------------------------------------------------------------------- diff --git a/spark/pom.xml b/spark/pom.xml index 5505aa7..ef7fc43 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -50,17 +50,16 @@ <spark.archive>spark-${spark.version}</spark.archive> <spark.src.download.url> - http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz + https://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz </spark.src.download.url> <spark.bin.download.url> - http://d3kbcqa49mib13.cloudfront.net/${spark.archive}-bin-without-hadoop.tgz + https://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}-bin-without-hadoop.tgz </spark.bin.download.url> </properties> <modules> <module>interpreter</module> <module>spark-scala-parent</module> - <module>scala-2.10</module> <module>scala-2.11</module> <module>spark-dependencies</module> <module>spark-shims</module> @@ -194,31 +193,46 @@ <profiles> <profile> + <id>spark-2.4</id> + <properties> + <spark.version>2.4.0</spark.version> + <protobuf.version>2.5.0</protobuf.version> + <py4j.version>0.10.7</py4j.version> + </properties> + </profile> + + <profile> <id>spark-2.3</id> <properties> - <spark.version>2.3.0</spark.version> + <spark.version>2.3.2</spark.version> <protobuf.version>2.5.0</protobuf.version> - <spark.py4j.version>0.10.6</spark.py4j.version> + <py4j.version>0.10.7</py4j.version> </properties> </profile> <profile> <id>spark-2.2</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> <properties> - <spark.version>2.2.0</spark.version> + <spark.version>2.2.1</spark.version> <py4j.version>0.10.4</py4j.version> </properties> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <modules> + <module>scala-2.10</module> + </modules> </profile> <profile> <id>spark-2.1</id> <properties> - <spark.version>2.1.0</spark.version> + <spark.version>2.1.2</spark.version> <py4j.version>0.10.4</py4j.version> </properties> + <modules> + <module>scala-2.10</module> + </modules> </profile> <profile> @@ -227,6 +241,9 @@ <spark.version>2.0.2</spark.version> <py4j.version>0.10.3</py4j.version> </properties> + <modules> + <module>scala-2.10</module> + </modules> </profile> <profile> @@ -235,22 +252,9 @@ <spark.version>1.6.3</spark.version> <py4j.version>0.9</py4j.version> </properties> - </profile> - - <profile> - <id>spark-1.5</id> - <properties> - <spark.version>1.5.2</spark.version> - <py4j.version>0.8.2.1</py4j.version> - </properties> - </profile> - - <profile> - <id>spark-1.4</id> - <properties> - <spark.version>1.4.1</spark.version> - <py4j.version>0.8.2.1</py4j.version> - </properties> + <modules> + <module>scala-2.10</module> + </modules> </profile> </profiles> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala ---------------------------------------------------------------------- diff --git a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala index 757f7eb..0956e04 100644 --- a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala +++ b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala @@ -39,6 +39,8 @@ class SparkScala211Interpreter(override val conf: SparkConf, override val printReplOutput: java.lang.Boolean) extends BaseSparkScalaInterpreter(conf, depFiles, printReplOutput) { + import SparkScala211Interpreter._ + lazy override val LOGGER: Logger = LoggerFactory.getLogger(getClass) private var sparkILoop: ILoop = _ @@ -81,7 +83,7 @@ class SparkScala211Interpreter(override val conf: SparkConf, sparkILoop.in = reader sparkILoop.initializeSynchronous() - callMethod(sparkILoop, "scala$tools$nsc$interpreter$ILoop$$loopPostInit") + loopPostInit(this) this.scalaCompleter = reader.completion.completer() createSparkContext() @@ -105,3 +107,72 @@ class SparkScala211Interpreter(override val conf: SparkConf, sparkILoop.interpret(code) } + +private object SparkScala211Interpreter { + + /** + * This is a hack to call `loopPostInit` at `ILoop`. At higher version of Scala such + * as 2.11.12, `loopPostInit` became a nested function which is inaccessible. Here, + * we redefine `loopPostInit` at Scala's 2.11.8 side and ignore `loadInitFiles` being called at + * Scala 2.11.12 since here we do not have to load files. + * + * Both methods `loopPostInit` and `unleashAndSetPhase` are redefined, and `phaseCommand` and + * `asyncMessage` are being called via reflection since both exist in Scala 2.11.8 and 2.11.12. + * + * Please see the codes below: + * https://github.com/scala/scala/blob/v2.11.8/src/repl/scala/tools/nsc/interpreter/ILoop.scala + * https://github.com/scala/scala/blob/v2.11.12/src/repl/scala/tools/nsc/interpreter/ILoop.scala + * + * See also ZEPPELIN-3810. + */ + private def loopPostInit(interpreter: SparkScala211Interpreter): Unit = { + import StdReplTags._ + import scala.reflect.classTag + import scala.reflect.io + + val sparkILoop = interpreter.sparkILoop + val intp = sparkILoop.intp + val power = sparkILoop.power + val in = sparkILoop.in + + def loopPostInit() { + // Bind intp somewhere out of the regular namespace where + // we can get at it in generated code. + intp.quietBind(NamedParam[IMain]("$intp", intp)(tagOfIMain, classTag[IMain])) + // Auto-run code via some setting. + (replProps.replAutorunCode.option + flatMap (f => io.File(f).safeSlurp()) + foreach (intp quietRun _) + ) + // classloader and power mode setup + intp.setContextClassLoader() + if (isReplPower) { + replProps.power setValue true + unleashAndSetPhase() + asyncMessage(power.banner) + } + // SI-7418 Now, and only now, can we enable TAB completion. + in.postInit() + } + + def unleashAndSetPhase() = if (isReplPower) { + power.unleash() + intp beSilentDuring phaseCommand("typer") // Set the phase to "typer" + } + + def phaseCommand(name: String): Results.Result = { + interpreter.callMethod( + sparkILoop, + "scala$tools$nsc$interpreter$ILoop$$phaseCommand", + Array(classOf[String]), + Array(name)).asInstanceOf[Results.Result] + } + + def asyncMessage(msg: String): Unit = { + interpreter.callMethod( + sparkILoop, "asyncMessage", Array(classOf[String]), Array(msg)) + } + + loopPostInit() + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/zeppelin-distribution/src/bin_license/LICENSE ---------------------------------------------------------------------- diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index c927ae4..8bdea23 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -290,7 +290,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (BSD Style) JSch v0.1.53 (http://www.jcraft.com) - http://www.jcraft.com/jsch/LICENSE.txt (BSD 3 Clause) highlightjs v9.4.0 (https://highlightjs.org/) - https://github.com/isagalaev/highlight.js/blob/9.4.0/LICENSE (BSD 3 Clause) hamcrest v1.3 (http://hamcrest.org/JavaHamcrest/) - http://opensource.org/licenses/BSD-3-Clause - (BSD Style) JLine v2.12.1 (https://github.com/jline/jline2) - https://github.com/jline/jline2/blob/master/LICENSE.txt + (BSD Style) JLine v2.14.3 (https://github.com/jline/jline2) - https://github.com/jline/jline2/blob/master/LICENSE.txt (BSD New license) Google Auth Library for Java - Credentials (com.google.auth:google-auth-library-credentials:0.4.0 - https://github.com/google/google-auth-library-java/google-auth-library-credentials) (BSD New license) Google Auth Library for Java - OAuth2 HTTP (com.google.auth:google-auth-library-oauth2-http:0.4.0 - https://github.com/google/google-auth-library-java/google-auth-library-oauth2-http) (New BSD license) Protocol Buffer Java API (com.google.protobuf:protobuf-java-util:3.0.0-beta-2 - https://developers.google.com/protocol-buffers/) http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/zeppelin-interpreter/pom.xml ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index e75e1f8..7539cdf 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -43,7 +43,7 @@ <aether.version>1.12</aether.version> <maven.aeither.provider.version>3.0.3</maven.aeither.provider.version> <wagon.version>1.0</wagon.version> - <jline.version>2.12.1</jline.version> + <jline.version>2.14.3</jline.version> <!--plugin versions--> <plugin.shade.version>2.3</plugin.shade.version> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index 3d4c62b..21b861c 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -67,10 +67,10 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { @Parameterized.Parameters public static List<Object[]> data() { return Arrays.asList(new Object[][]{ - {"2.2.1"}, - {"2.1.2"}, - {"2.0.2"}, - {"1.6.3"} + {"2.4.0"}, + {"2.3.2"}, + {"2.2.1"}, + {"1.6.3"} }); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2ca8d19b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java index 313bba0..f331a0f 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest.java @@ -42,11 +42,12 @@ public class SparkIntegrationTest { @Parameterized.Parameters public static List<Object[]> data() { return Arrays.asList(new Object[][]{ - {"2.3.1"}, - {"2.2.1"}, - {"2.1.2"}, - {"2.0.2"}, - {"1.6.3"} + {"2.4.0"}, + {"2.3.2"}, + {"2.2.1"}, + {"2.1.2"}, + {"2.0.2"}, + {"1.6.3"} }); }