Repository: incubator-zeppelin Updated Branches: refs/heads/master a911b0ec4 -> 928d09987
Spark 1.4 support This PR enables Zeppelin work with upcoming Spark 1.4 to test, install spark 1.4 maven artifacts into your local .m2 repository by doing ``` sbt/sbt publish-local ``` in spark-1.4 source code, and then build Zeppelin ``` mvn clean package -Pspark-1.4 -Dspark.version=1.4.0 -DskipTests ``` I have a simple screenshot  Author: Lee moon soo <[email protected]> Closes #76 from Leemoonsoo/spark-1.4 and squashes the following commits: 77cb7b9 [Lee moon soo] Spark 1.4 support Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/928d0998 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/928d0998 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/928d0998 Branch: refs/heads/master Commit: 928d09987b0b59676d82bbfe336d1688ef01d9bc Parents: a911b0e Author: Lee moon soo <[email protected]> Authored: Wed May 20 13:34:27 2015 +0900 Committer: Lee moon soo <[email protected]> Committed: Sat May 23 14:39:11 2015 +0900 ---------------------------------------------------------------------- README.md | 4 ++ pom.xml | 60 +++++++++++++++++++- spark/pom.xml | 1 - .../zeppelin/spark/PySparkInterpreter.java | 5 +- .../apache/zeppelin/spark/SparkInterpreter.java | 10 ++++ .../zeppelin/spark/SparkSqlInterpreter.java | 2 + 6 files changed, 77 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 46e188c..44f50ee 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,10 @@ Spark 1.3.x ``` mvn clean package -Pspark-1.3 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests ``` +Spark 1.4.x +``` +mvn clean package -Pspark-1.4 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests +``` CDH 5.X ``` mvn clean package -Pspark-1.2 -Dhadoop.version=2.5.0-cdh5.3.0 -Phadoop-2.4 -DskipTests http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b4f78df..e43ccab 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,7 @@ <jets3t.version>0.7.1</jets3t.version> <commons.math3.version>3.2</commons.math3.version> <commons.httpclient.version>4.3.6</commons.httpclient.version> + <fasterxml.jackson.version>2.4.4</fasterxml.jackson.version> <codehaus.jackson.version>1.8.8</codehaus.jackson.version> <snappy.version>1.0.5</snappy.version> <io.netty.version>4.0.17.Final</io.netty.version> @@ -812,6 +813,26 @@ </dependency> <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${fasterxml.jackson.version}</version> + </dependency> + + <!-- Guava is excluded because of SPARK-6149. The Guava version referenced in this module is + 15.0, which causes runtime incompatibility issues. --> + <dependency> + <groupId>com.fasterxml.jackson.module</groupId> + <artifactId>jackson-module-scala_2.10</artifactId> + <version>${fasterxml.jackson.version}</version> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>com.thoughtworks.paranamer</groupId> <artifactId>paranamer</artifactId> <version>2.6</version> @@ -1317,8 +1338,6 @@ <profile> <id>spark-1.3</id> - <dependencies> - </dependencies> <properties> <akka.version>2.3.4-spark</akka.version> <spark.version>1.3.0</spark.version> @@ -1343,9 +1362,46 @@ <snappy.version>1.1.1.6</snappy.version> <mesos.version>0.21.0</mesos.version> </properties> + + <dependencies> + </dependencies> + </profile> <profile> + <id>spark-1.4</id> + <properties> + <akka.version>2.3.4-spark</akka.version> + <spark.version>1.4.0</spark.version> + <mesos.version>0.21.1</mesos.version> + <hbase.version>0.98.7-hadoop2</hbase.version> + <hbase.artifact>hbase</hbase.artifact> + <hive.group>org.spark-project.hive</hive.group> + <hive.version>0.13.1a</hive.version> + <derby.version>10.10.1.1</derby.version> + <orbit.version>3.0.0.v201112011016</orbit.version> + <parquet.version>1.6.0rc3</parquet.version> + <chill.version>0.5.0</chill.version> + <ivy.version>2.4.0</ivy.version> + <oro.version>2.0.8</oro.version> + <avro.mapred.classifier>hadoop2</avro.mapred.classifier> + <codahale.metrics.version>3.1.0</codahale.metrics.version> + <commons.httpclient.version>4.3.2</commons.httpclient.version> + <commons.math3.version>3.4.1</commons.math3.version> + <io.netty.version>4.0.23.Final</io.netty.version> + <codehaus.jackson.version>1.9.13</codehaus.jackson.version> + <fasterxml.jackson.version>2.4.4</fasterxml.jackson.version> + <snappy.version>1.1.1.7</snappy.version> + <mesos.version>0.21.0</mesos.version> + </properties> + + <dependencies> + </dependencies> + + </profile> + + + <profile> <id>hadoop-0.23</id> <!-- SPARK-1121: Adds an explicit dependency on Avro to work around a Hadoop 0.23.X issue --> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/pom.xml ---------------------------------------------------------------------- diff --git a/spark/pom.xml b/spark/pom.xml index 72f216d..b4c05e5 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -76,7 +76,6 @@ <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> - <version>2.3.0</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index 9fce093..95eefd8 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -48,9 +48,9 @@ import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.WrappedInterpreter; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -298,7 +298,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand SparkInterpreter sparkInterpreter = getSparkInterpreter(); if (!sparkInterpreter.getSparkContext().version().startsWith("1.2") && - !sparkInterpreter.getSparkContext().version().startsWith("1.3")) { + !sparkInterpreter.getSparkContext().version().startsWith("1.3") && + !sparkInterpreter.getSparkContext().version().startsWith("1.4")) { return new InterpreterResult(Code.ERROR, "pyspark " + sparkInterpreter.getSparkContext().version() + " is not supported"); } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index c875e85..731068e 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -415,6 +415,10 @@ public class SparkInterpreter extends Interpreter { Method loadFiles = this.interpreter.getClass().getMethod( "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class); loadFiles.invoke(this.interpreter, settings); + } else if (sc.version().startsWith("1.4")) { + Method loadFiles = this.interpreter.getClass().getMethod( + "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class); + loadFiles.invoke(this.interpreter, settings); } } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { @@ -447,6 +451,10 @@ public class SparkInterpreter extends Interpreter { intp.interpret("import sqlContext.implicits._"); intp.interpret("import sqlContext.sql"); intp.interpret("import org.apache.spark.sql.functions._"); + } else if (sc.version().startsWith("1.4")) { + intp.interpret("import sqlContext.implicits._"); + intp.interpret("import sqlContext.sql"); + intp.interpret("import org.apache.spark.sql.functions._"); } // add jar @@ -621,6 +629,8 @@ public class SparkInterpreter extends Interpreter { progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); } else if (sc.version().startsWith("1.3")) { progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); + } else if (sc.version().startsWith("1.4")) { + progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); } else { continue; } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index 618579d..3138fbc 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -171,6 +171,8 @@ public class SparkSqlInterpreter extends Interpreter { progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); } else if (sc.version().startsWith("1.3")) { progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); + } else if (sc.version().startsWith("1.4")) { + progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage()); } else { logger.warn("Spark {} getting progress information not supported" + sc.version()); continue;
