Repository: zeppelin Updated Branches: refs/heads/master ae1cb0527 -> 9bc4dce98
ZEPPELIN-1918. Fix build with Spark 2.1.0 ### What is this PR for? It's my misunderstanding of `SPARK_VER` in travis. It is only used for downloading spark distribution. We need to specify spark profile explicitly for building with specific version of spark. This PR add new profile for spark 2.1 and fix the build issue with spark 2.1.0 because `SecurityManager` changes its constructor signature in spark 2.1 ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1918 ### How should this be tested? Build with spark 2.1.0 and tested it manually as below screenshot. ### Screenshots (if appropriate) ![image](https://cloud.githubusercontent.com/assets/164491/21797414/d586aa04-d749-11e6-8c6f-3b12e9e2ae2d.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Closes #1880 from zjffdu/ZEPPELIN-1918 and squashes the following commits: 8f87459 [Jeff Zhang] update release script 34772e0 [Jeff Zhang] update build doc for spark 2.1 5efdb11 [Jeff Zhang] fix unit test and address comments def502f [Jeff Zhang] ZEPPELIN-1918. Fix build with Spark 2.1.0 Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/9bc4dce9 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/9bc4dce9 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/9bc4dce9 Branch: refs/heads/master Commit: 9bc4dce9884ec4011f677560f708286968549374 Parents: ae1cb05 Author: Jeff Zhang <zjf...@apache.org> Authored: Wed Jan 11 10:56:53 2017 +0800 Committer: Felix Cheung <felixche...@apache.org> Committed: Mon Jan 16 09:28:41 2017 -0800 ---------------------------------------------------------------------- .travis.yml | 8 ++-- dev/create_release.sh | 4 +- dev/publish_release.sh | 2 +- docs/install/build.md | 5 +++ spark-dependencies/pom.xml | 14 ++++++- spark/pom.xml | 14 ++++++- .../apache/zeppelin/spark/SparkInterpreter.java | 41 ++++++++++++++++++-- 7 files changed, 73 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 48e8aa7..5b3371d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,13 +40,13 @@ matrix: - jdk: "oraclejdk7" env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS="" - # Test all modules with spark 2.0.2 and scala 2.11 + # Test all modules with spark 2.1.0 and scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS="" + env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS="" - # Test all modules with spark 2.1.0 and scala 2.11 + # Test all modules with spark 2.0.2 and scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS="" + env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS="" # Test all modules with scala 2.10 - jdk: "oraclejdk7" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/dev/create_release.sh ---------------------------------------------------------------------- diff --git a/dev/create_release.sh b/dev/create_release.sh index 9b0b231..873bb10 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -124,8 +124,8 @@ function make_binary_release() { build_docker_base git_clone make_source_package -make_binary_release all "-Pspark-2.0 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION}" -make_binary_release netinst "-Pspark-2.0 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am" +make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION}" +make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am" # remove non release files and dirs rm -rf "${WORKING_DIR}/zeppelin" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/dev/publish_release.sh ---------------------------------------------------------------------- diff --git a/dev/publish_release.sh b/dev/publish_release.sh index 7b99f39..a2acc98 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -44,7 +44,7 @@ NC='\033[0m' # No Color RELEASE_VERSION="$1" GIT_TAG="$2" -PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.0 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr" +PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr" PROJECT_OPTIONS="-pl !zeppelin-distribution" NEXUS_STAGING="https://repository.apache.org/service/local/staging" NEXUS_PROFILE="153446d1ac37c4" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/docs/install/build.md ---------------------------------------------------------------------- diff --git a/docs/install/build.md b/docs/install/build.md index f3012a5..e452559 100644 --- a/docs/install/build.md +++ b/docs/install/build.md @@ -97,6 +97,7 @@ Set spark major version Available profiles are ``` +-Pspark-2.1 -Pspark-2.0 -Pspark-1.6 -Pspark-1.5 @@ -185,6 +186,10 @@ Bulid examples under zeppelin-examples directory Here are some examples with several options: ```bash +# build with spark-2.1, scala-2.11 +./dev/change_scala_version.sh 2.11 +mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests + # build with spark-2.0, scala-2.11 ./dev/change_scala_version.sh 2.11 mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/spark-dependencies/pom.xml ---------------------------------------------------------------------- diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index 198f2bf..887964d 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -523,13 +523,23 @@ <profile> <id>spark-2.0</id> + <properties> + <spark.version>2.0.2</spark.version> + <protobuf.version>2.5.0</protobuf.version> + <py4j.version>0.10.3</py4j.version> + <scala.version>2.11.8</scala.version> + </properties> + </profile> + + <profile> + <id>spark-2.1</id> <activation> <activeByDefault>true</activeByDefault> </activation> <properties> - <spark.version>2.0.2</spark.version> + <spark.version>2.1.0</spark.version> <protobuf.version>2.5.0</protobuf.version> - <py4j.version>0.10.3</py4j.version> + <py4j.version>0.10.4</py4j.version> <scala.version>2.11.8</scala.version> </properties> </profile> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/spark/pom.xml ---------------------------------------------------------------------- diff --git a/spark/pom.xml b/spark/pom.xml index 54690ea..4989af9 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -515,13 +515,23 @@ <profile> <id>spark-2.0</id> + <properties> + <spark.version>2.0.2</spark.version> + <protobuf.version>2.5.0</protobuf.version> + <py4j.version>0.10.3</py4j.version> + <scala.version>2.11.8</scala.version> + </properties> + </profile> + + <profile> + <id>spark-2.1</id> <activation> <activeByDefault>true</activeByDefault> </activation> <properties> - <spark.version>2.0.2</spark.version> + <spark.version>2.1.0</spark.version> <protobuf.version>2.5.0</protobuf.version> - <py4j.version>0.10.3</py4j.version> + <py4j.version>0.10.4</py4j.version> <scala.version>2.11.8</scala.version> </properties> </profile> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 7882303..0584a30 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -488,8 +488,9 @@ public class SparkInterpreter extends Interpreter { } //Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist + //TODO(zjffdu), this is not maintainable when new version is added. String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip", - "py4j-0.10.1-src.zip", "py4j-0.10.3-src.zip"}; + "py4j-0.10.1-src.zip", "py4j-0.10.3-src.zip", "py4j-0.10.4-src.zip"}; ArrayList<String> pythonLibUris = new ArrayList<>(); for (String lib : pythonLibs) { File libFile = new File(pysparkPath, lib); @@ -1452,8 +1453,10 @@ public class SparkInterpreter extends Interpreter { .getConstructor(new Class[]{ SparkConf.class, File.class, SecurityManager.class, int.class, String.class}); - return constructor.newInstance(new Object[] { - conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"}); + Object securityManager = createSecurityManager(conf); + return constructor.newInstance(new Object[]{ + conf, outputDir, securityManager, 0, "HTTP Server"}); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) { // fallback to old constructor @@ -1464,7 +1467,7 @@ public class SparkInterpreter extends Interpreter { .getConstructor(new Class[]{ File.class, SecurityManager.class, int.class, String.class}); return constructor.newInstance(new Object[] { - outputDir, new SecurityManager(conf), 0, "HTTP Server"}); + outputDir, createSecurityManager(conf), 0, "HTTP Server"}); } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e1) { logger.error(e1.getMessage(), e1); @@ -1472,4 +1475,34 @@ public class SparkInterpreter extends Interpreter { } } } + + /** + * Constructor signature of SecurityManager changes in spark 2.1.0, so we use this method to + * create SecurityManager properly for different versions of spark + * + * @param conf + * @return + * @throws ClassNotFoundException + * @throws NoSuchMethodException + * @throws IllegalAccessException + * @throws InvocationTargetException + * @throws InstantiationException + */ + private Object createSecurityManager(SparkConf conf) throws ClassNotFoundException, + NoSuchMethodException, IllegalAccessException, InvocationTargetException, + InstantiationException { + Object securityManager = null; + try { + Constructor<?> smConstructor = getClass().getClassLoader() + .loadClass("org.apache.spark.SecurityManager") + .getConstructor(new Class[]{ SparkConf.class, scala.Option.class }); + securityManager = smConstructor.newInstance(conf, null); + } catch (NoSuchMethodException e) { + Constructor<?> smConstructor = getClass().getClassLoader() + .loadClass("org.apache.spark.SecurityManager") + .getConstructor(new Class[]{ SparkConf.class }); + securityManager = smConstructor.newInstance(conf); + } + return securityManager; + } }