Repository: incubator-zeppelin Updated Branches: refs/heads/master ac5ff10b6 -> 0a82a93ce
ZEPPELIN-377 CI hanging on ./testing/startSparkCluster.sh 1.4.0 2.3 Address https://issues.apache.org/jira/browse/ZEPPELIN-377. This patch change spark package download location from apache archive to mirror, to download in 10min. Also add missing test for 1.5.1 and change test version from 1.4.0 to 1.4.1 Author: Lee moon soo <[email protected]> Closes #380 from Leemoonsoo/fix_spark_test and squashes the following commits: 142583a [Lee moon soo] Add test for 1.5.1 b8323e6 [Lee moon soo] Use mirror for 1.3.x and later version of spark Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/0a82a93c Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/0a82a93c Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/0a82a93c Branch: refs/heads/master Commit: 0a82a93ce495cf537b28104a024a8c0361fd80ac Parents: ac5ff10 Author: Lee moon soo <[email protected]> Authored: Mon Nov 2 13:46:02 2015 +0900 Committer: Lee moon soo <[email protected]> Committed: Mon Nov 2 19:39:31 2015 +0900 ---------------------------------------------------------------------- .travis.yml | 17 ++++++++++++++--- pom.xml | 1 + testing/startSparkCluster.sh | 18 +++++++++++++++--- .../apache/zeppelin/rest/AbstractTestRestApi.java | 9 +++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index a1b467d..0c74edf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,27 +28,38 @@ before_script: - script: -# spark 1.4 + # spark 1.5 - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/startSparkCluster.sh 1.4.0 2.3 + - ./testing/startSparkCluster.sh 1.5.1 2.3 + - echo "export SPARK_HOME=`pwd`/spark-1.5.1-bin-hadoop2.3" > conf/zeppelin-env.sh - mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/stopSparkCluster.sh 1.4.0 2.3 + - ./testing/stopSparkCluster.sh 1.5.1 2.3 + # spark 1.4 + - rm -rf `pwd`/interpreter/spark + - mvn package -DskipTests -Pspark-1.4 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' + - ./testing/startSparkCluster.sh 1.4.1 2.3 + - echo "export SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.3" > conf/zeppelin-env.sh + - mvn package -Pspark-1.4 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false + - ./testing/stopSparkCluster.sh 1.4.1 2.3 # spark 1.3 - rm -rf `pwd`/interpreter/spark - mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.3.1 2.3 + - echo "export SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3" > conf/zeppelin-env.sh - mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - rm -rf `pwd`/interpreter/spark - mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.2.1 2.3 + - echo "export SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3" > conf/zeppelin-env.sh - mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - rm -rf `pwd`/interpreter/spark - mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - ./testing/startSparkCluster.sh 1.1.1 2.3 + - echo "export SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3" > conf/zeppelin-env.sh - mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - ./testing/stopSparkCluster.sh 1.1.1 2.3 http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 45335fe..394f853 100755 --- a/pom.xml +++ b/pom.xml @@ -456,6 +456,7 @@ <exclude>STYLE.md</exclude> <exclude>Roadmap.md</exclude> <exclude>conf/interpreter.json</exclude> + <exclude>conf/zeppelin-env.sh</exclude> <exclude>spark-*-bin*/**</exclude> </excludes> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/testing/startSparkCluster.sh ---------------------------------------------------------------------- diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh index 1f70fe6..7333ab0 100755 --- a/testing/startSparkCluster.sh +++ b/testing/startSparkCluster.sh @@ -31,7 +31,19 @@ ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} echo "SPARK_HOME is ${SPARK_HOME} " if [ ! -d "${SPARK_HOME}" ]; then - wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz + echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null + if [ $? -eq 0 ]; then + # spark 1.1.x and spark 1.2.x can be downloaded from archive + wget http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz + else + # spark 1.3.x and later can be downloaded from mirror + # get download address from mirror + MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1";) + + PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g') + PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g') + wget "${PREFFERED}${PATHINFO}" + fi tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz fi @@ -46,8 +58,8 @@ export SPARK_MASTER_WEBUI_PORT=7072 export SPARK_WORKER_WEBUI_PORT=8082 ${SPARK_HOME}/sbin/start-master.sh -echo ${SPARK_VERSION} | grep "^1.4" > /dev/null -if [ $? -ne 0 ]; then # spark 1.3 or prior +echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null +if [ $? -eq 0 ]; then # spark 1.3 or prior ${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT} else ${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071 http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index ecf0c72..1895e16 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.lang.ref.WeakReference; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -29,6 +30,7 @@ import java.util.concurrent.Executors; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethodBase; import org.apache.commons.httpclient.methods.*; +import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterSetting; @@ -197,6 +199,13 @@ public abstract class AbstractTestRestApi { protected static void shutDown() throws Exception { if (!wasRunning) { + // restart interpreter to stop all interpreter processes + List<String> settingList = ZeppelinServer.notebook.getInterpreterFactory() + .getDefaultInterpreterSettingList(); + for (String setting : settingList) { + ZeppelinServer.notebook.getInterpreterFactory().restart(setting); + } + LOG.info("Terminating test Zeppelin..."); ZeppelinServer.jettyServer.stop(); executor.shutdown();
