Repository: incubator-zeppelin Updated Branches: refs/heads/master 5ad02a28f -> 715526824
ZEPPELIN-783: fix Spark download on CI ### What is this PR for? Fix Spark download on CI ### What type of PR is it? Hot Fix ### What is the Jira issue? [ZEPPELIN-783](https://issues.apache.org/jira/browse/ZEPPELIN-783) ### How should this be tested? CI must be green ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Alexander Bezzubov <[email protected]> Closes #818 from bzz/ZEPPELIN-783-stable-ci-part-2 and squashes the following commits: b4d66b3 [Alexander Bezzubov] ZEPPELIN-783: advanced Spark download failover procedure Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/71552682 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/71552682 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/71552682 Branch: refs/heads/master Commit: 7155268247965b9c22946d4d521838f552c6fcee Parents: 5ad02a2 Author: Alexander Bezzubov <[email protected]> Authored: Wed Apr 6 16:21:56 2016 +0900 Committer: Alexander Bezzubov <[email protected]> Committed: Fri Apr 8 12:23:25 2016 +0900 ---------------------------------------------------------------------- testing/downloadSpark.sh | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/71552682/testing/downloadSpark.sh ---------------------------------------------------------------------- diff --git a/testing/downloadSpark.sh b/testing/downloadSpark.sh index d12580f..004e1dd 100755 --- a/testing/downloadSpark.sh +++ b/testing/downloadSpark.sh @@ -40,10 +40,28 @@ fi set -xe -TIMEOUT_SEC=590 +MAX_DOWNLOAD_TIME_SEC=590 FWDIR="$(dirname "${BASH_SOURCE-$0}")" ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" +####################################### +# Downloads file from the givrn URL. +# Ties 3 times with 1s delay, 20s read and 15s connection timeouts. +# Globals: +# None +# Arguments: +# url - source URL +# Returns: +# None +####################################### +download_with_retry() { + local url="$1" + wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}" + if [[ "$?" -ne 0 ]]; then + echo "3 download attempts for ${url} failed" + fi +} + SPARK_CACHE=".spark-dist" SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}" @@ -56,11 +74,13 @@ if [[ ! -d "${SPARK_HOME}" ]]; then pwd ls -la . echo "${SPARK_CACHE} does not have ${SPARK_ARCHIVE} downloading ..." + # download archive if not cached if [[ "${SPARK_VER_RANGE}" == "<=1.2" ]]; then # spark 1.1.x and spark 1.2.x can be downloaded from archive STARTTIME=`date +%s` - timeout -s KILL "${TIMEOUT_SEC}" wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"; + #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"; + download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"; ENDTIME=`date +%s` DOWNLOADTIME="$((ENDTIME-STARTTIME))" else @@ -72,7 +92,8 @@ if [[ ! -d "${SPARK_HOME}" ]]; then PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g') STARTTIME=`date +%s` - timeout -s KILL "${TIMEOUT_SEC}" wget -q "${PREFFERED}${PATHINFO}" + #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget -q "${PREFFERED}${PATHINFO}" + download_with_retry "${PREFFERED}${PATHINFO}" ENDTIME=`date +%s` DOWNLOADTIME="$((ENDTIME-STARTTIME))" fi
