Repository: incubator-zeppelin Updated Branches: refs/heads/master 98cb8e8c1 -> ff1a0c032
Get travis to retry download of Spark release ### What is this PR for? Refactor download and use travis' builtin mechanism to retry on failure - hopefully it will then hit a different apache mirror; this should mitigate the intermitted download failures that result in test failures ### What type of PR is it? Improvement ### Todos * [x] - Separate download script * [x] - Travis to retry download step * [x] - Add timeout to kill download if not complete in 5 min (need to tune this) * [x] - Add timer to log how long download takes ### Is there a relevant Jira issue? N/A ### How should this be tested? Travis ### Screenshots (if appropriate) N/A ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Felix Cheung <[email protected]> Closes #727 from felixcheung/retrydownload and squashes the following commits: 5bfebea [Felix Cheung] up timeout duration 653c5bf [Felix Cheung] fix start spark script 44ac10f [Felix Cheung] change permission 1e9e642 [Felix Cheung] fix version check e9b2272 [Felix Cheung] change timeout value baf73a3 [Felix Cheung] separate download, travis to retry download Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/ff1a0c03 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/ff1a0c03 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/ff1a0c03 Branch: refs/heads/master Commit: ff1a0c032d42f7978228c9f00d669c83c6275e30 Parents: 98cb8e8 Author: Felix Cheung <[email protected]> Authored: Fri Feb 19 12:10:26 2016 -0800 Committer: Felix Cheung <[email protected]> Committed: Tue Feb 23 16:51:12 2016 -0800 ---------------------------------------------------------------------- .gitignore | 1 + .travis.yml | 1 + testing/downloadSpark.sh | 70 +++++++++++++++++++++++++++++++++++++++ testing/startSparkCluster.sh | 16 +-------- 4 files changed, 73 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ff1a0c03/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 1cdb809..a420a30 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ zeppelin-web/bower_components **/data/ **/build/ **/testing/ +!/testing/ # OS generated files # ###################### http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ff1a0c03/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 5495398..14a0430 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,7 @@ install: - mvn $BUILD_FLAG $PROFILE -B before_script: + - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ff1a0c03/testing/downloadSpark.sh ---------------------------------------------------------------------- diff --git a/testing/downloadSpark.sh b/testing/downloadSpark.sh new file mode 100755 index 0000000..7c907fc --- /dev/null +++ b/testing/downloadSpark.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +if [ $# -ne 2 ]; then + echo "usage) $0 [spark version] [hadoop version]" + echo " eg) $0 1.3.1 2.6" + exit 1 +fi + +SPARK_VERSION="${1}" +HADOOP_VERSION="${2}" + +echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null +if [ $? -eq 0 ]; then + echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null + if [ $? -eq 0 ]; then + SPARK_VER_RANGE="<=1.2" + else + SPARK_VER_RANGE="<=1.3" + fi +else + SPARK_VER_RANGE=">1.3" +fi + +set -xe + +FWDIR=$(dirname "${BASH_SOURCE-$0}") +ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" +export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} +echo "SPARK_HOME is ${SPARK_HOME}" +if [ ! -d "${SPARK_HOME}" ]; then + if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then + # spark 1.1.x and spark 1.2.x can be downloaded from archive + STARTTIME=`date +%s` + timeout -s KILL 300 wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz + ENDTIME=`date +%s` + DOWNLOADTIME=$((ENDTIME-STARTTIME)) + else + # spark 1.3.x and later can be downloaded from mirror + # get download address from mirror + MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1";) + + PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g') + PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g') + + STARTTIME=`date +%s` + timeout -s KILL 590 wget -q "${PREFFERED}${PATHINFO}" + ENDTIME=`date +%s` + DOWNLOADTIME=$((ENDTIME-STARTTIME)) + fi + tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz +fi + +set +xe http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ff1a0c03/testing/startSparkCluster.sh ---------------------------------------------------------------------- diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh index ded1204..e47edc1 100755 --- a/testing/startSparkCluster.sh +++ b/testing/startSparkCluster.sh @@ -38,27 +38,13 @@ else SPARK_VER_RANGE=">1.3" fi + set -xe FWDIR=$(dirname "${BASH_SOURCE-$0}") ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} echo "SPARK_HOME is ${SPARK_HOME}" -if [ ! -d "${SPARK_HOME}" ]; then - if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then - # spark 1.1.x and spark 1.2.x can be downloaded from archive - wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz - else - # spark 1.3.x and later can be downloaded from mirror - # get download address from mirror - MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1";) - - PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g') - PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g') - wget -q "${PREFFERED}${PATHINFO}" - fi - tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -fi # create PID dir. test case detect pid file so they can select active spark home dir for test mkdir -p ${SPARK_HOME}/run
