Repository: zeppelin Updated Branches: refs/heads/master c195b5609 -> 453be73ab
[HOTFIX] Change Spark download url for spark 1.6.x ### What is this PR for? Spark 1.6 was removed from CDN. We need to download it from apache archive. ### What type of PR is it? [Hot Fix] ### Todos * [x] - Change the location for downloading Spark 1.6.x ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jongyoul Lee <[email protected]> Closes #3075 from jongyoul/hotfix/change-spark-download-location and squashes the following commits: 87bfd2abf [Jongyoul Lee] Removed comments Reduce changes a51d5b93b [Jongyoul Lee] Change download url for spark 1.6.x 9f63f8358 [Jongyoul Lee] Change download url for spark 1.6.x Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/453be73a Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/453be73a Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/453be73a Branch: refs/heads/master Commit: 453be73abb19eaf2448d1a63347afb8ca5911e1a Parents: c195b56 Author: Jongyoul Lee <[email protected]> Authored: Tue Jul 17 10:31:00 2018 +0900 Committer: Jongyoul Lee <[email protected]> Committed: Tue Jul 17 11:23:37 2018 +0900 ---------------------------------------------------------------------- .../interpreter/SparkDownloadUtils.java | 34 ++++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/453be73a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java index e027bb0..0455dd8 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkDownloadUtils.java @@ -1,17 +1,16 @@ package org.apache.zeppelin.interpreter; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SparkDownloadUtils { private static Logger LOGGER = LoggerFactory.getLogger(SparkDownloadUtils.class); @@ -34,18 +33,39 @@ public class SparkDownloadUtils { return targetSparkHomeFolder.getAbsolutePath(); } // Try mirrors a few times until one succeeds + boolean downloaded = false; for (int i = 0; i < 3; i++) { try { String preferredMirror = IOUtils.toString(new URL("https://www.apache.org/dyn/closer.lua?preferred=true";)); File downloadFile = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6.tgz"); - String downloadURL = preferredMirror + "/spark/spark-" + version + "/spark-" + version + "-bin-hadoop2.6.tgz";; + String downloadURL = preferredMirror + "/spark/spark-" + version + "/spark-" + version + "-bin-hadoop2.6.tgz"; runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder}); runShellCommand(new String[]{"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder}); + downloaded = true; break; } catch (Exception e) { LOGGER.warn("Failed to download Spark", e); } } + + // fallback to use apache archive + // https://archive.apache.org/dist/spark/spark-1.6.3/spark-1.6.3-bin-hadoop2.6.tgz + if (!downloaded) { + File downloadFile = new File(downloadFolder + "/spark-" + version + "-bin-hadoop2.6.tgz"); + String downloadURL = + "https://archive.apache.org/dist/spark/spark-"; + + version + + "/spark-" + + version + + "-bin-hadoop2.6.tgz"; + try { + runShellCommand(new String[] {"wget", downloadURL, "-P", downloadFolder}); + runShellCommand( + new String[] {"tar", "-xvf", downloadFile.getAbsolutePath(), "-C", downloadFolder}); + } catch (Exception e) { + throw new RuntimeException("Fail to download spark " + version, e); + } + } return targetSparkHomeFolder.getAbsolutePath(); }
