Repository: spark Updated Branches: refs/heads/branch-2.2 a8537a5ab -> 4bc4ccd63
[SPARK-24813][TESTS][HIVE][HOTFIX][BRANCH-2.2] HiveExternalCatalogVersionsSuite still flaky; fall back to Apache archive ## What changes were proposed in this pull request? Try only unique ASF mirrors to download Spark release; fall back to Apache archive if no mirrors available or release is not mirrored ## How was this patch tested? Existing HiveExternalCatalogVersionsSuite Author: Sean Owen <sro...@gmail.com> Closes #21779 from srowen/SPARK-24813.2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4bc4ccd6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4bc4ccd6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4bc4ccd6 Branch: refs/heads/branch-2.2 Commit: 4bc4ccd63b7d56fba1814505136b0d33517ed8f0 Parents: a8537a5 Author: Sean Owen <sro...@gmail.com> Authored: Mon Jul 16 12:26:45 2018 +0800 Committer: hyukjinkwon <gurwls...@apache.org> Committed: Mon Jul 16 12:26:45 2018 +0800 ---------------------------------------------------------------------- .../hive/HiveExternalCatalogVersionsSuite.scala | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/4bc4ccd6/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 92c27e8..313059b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -53,12 +53,22 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { } private def tryDownloadSpark(version: String, path: String): Unit = { - // Try mirrors a few times until one succeeds - for (i <- 0 until 3) { - val preferredMirror = - Seq("wget", "https://www.apache.org/dyn/closer.lua?preferred=true", "-q", "-O", "-").!!.trim + // Try a few mirrors first; fall back to Apache archive + val mirrors = + (0 until 2).flatMap { _ => + try { + Some(Seq("wget", + "https://www.apache.org/dyn/closer.lua?preferred=true", "-q", "-O", "-").!!.trim) + } catch { + // If we can't get a mirror URL, skip it. No retry. + case _: Exception => None + } + } + val sites = mirrors.distinct :+ "https://archive.apache.org/dist" + logInfo(s"Trying to download Spark $version from $sites") + for (site <- sites) { val filename = s"spark-$version-bin-hadoop2.7.tgz" - val url = s"$preferredMirror/spark/spark-$version/$filename" + val url = s"$site/spark/spark-$version/$filename" logInfo(s"Downloading Spark $version from $url") if (Seq("wget", url, "-q", "-P", path).! == 0) { val downloaded = new File(sparkTestingDir, filename).getCanonicalPath --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org