spark git commit: Revert "[SPARK-15782][YARN] Set spark.jars system property in client mode"
Repository: spark
Updated Branches:
refs/heads/branch-2.0 73bf87f3c -> 9acf254ed
Revert "[SPARK-15782][YARN] Set spark.jars system property in client mode"
This reverts commit f1e9d2d92456b437803d6ebab77831e66b0eb53e.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9acf254e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9acf254e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9acf254e
Branch: refs/heads/branch-2.0
Commit: 9acf254edadcbecb0f0bee00578dc7f228b5ec1e
Parents: 73bf87f
Author: Davies Liu
Authored: Wed Jun 15 15:54:29 2016 -0700
Committer: Davies Liu
Committed: Wed Jun 15 15:54:29 2016 -0700
--
.../scala/org/apache/spark/SparkContext.scala | 2 +-
.../scala/org/apache/spark/util/Utils.scala | 20
.../apache/spark/deploy/SparkSubmitSuite.scala | 12
.../org/apache/spark/repl/SparkILoop.scala | 11 ++-
.../main/scala/org/apache/spark/repl/Main.scala | 4 +++-
5 files changed, 6 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9acf254e/core/src/main/scala/org/apache/spark/SparkContext.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index d870181..d56946e 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -391,7 +391,7 @@ class SparkContext(config: SparkConf) extends Logging with
ExecutorAllocationCli
_conf.set("spark.executor.id", SparkContext.DRIVER_IDENTIFIER)
-_jars = Utils.getUserJars(_conf)
+_jars =
_conf.getOption("spark.jars").map(_.split(",")).map(_.filter(_.nonEmpty)).toSeq.flatten
_files =
_conf.getOption("spark.files").map(_.split(",")).map(_.filter(_.nonEmpty))
.toSeq.flatten
http://git-wip-us.apache.org/repos/asf/spark/blob/9acf254e/core/src/main/scala/org/apache/spark/util/Utils.scala
--
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index aebd98b..f9d0540 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2352,26 +2352,6 @@ private[spark] object Utils extends Logging {
log.info(s"Started daemon with process name: ${Utils.getProcessName()}")
SignalUtils.registerLogger(log)
}
-
- /**
- * Unions two comma-separated lists of files and filters out empty strings.
- */
- def unionFileLists(leftList: Option[String], rightList: Option[String]):
Set[String] = {
-var allFiles = Set[String]()
-leftList.foreach { value => allFiles ++= value.split(",") }
-rightList.foreach { value => allFiles ++= value.split(",") }
-allFiles.filter { _.nonEmpty }
- }
-
- def getUserJars(conf: SparkConf): Seq[String] = {
-val sparkJars = conf.getOption("spark.jars")
-if (conf.get("spark.master") == "yarn") {
- val yarnJars = conf.getOption("spark.yarn.dist.jars")
- unionFileLists(sparkJars, yarnJars).toSeq
-} else {
- sparkJars.map(_.split(",")).map(_.filter(_.nonEmpty)).toSeq.flatten
-}
- }
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/9acf254e/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 0b02059..2718976 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -570,18 +570,6 @@ class SparkSubmitSuite
appArgs.executorMemory should be ("2.3g")
}
}
-
- test("comma separated list of files are unioned correctly") {
-val left = Option("/tmp/a.jar,/tmp/b.jar")
-val right = Option("/tmp/c.jar,/tmp/a.jar")
-val emptyString = Option("")
-Utils.unionFileLists(left, right) should be (Set("/tmp/a.jar",
"/tmp/b.jar", "/tmp/c.jar"))
-Utils.unionFileLists(emptyString, emptyString) should be (Set.empty)
-Utils.unionFileLists(Option("/tmp/a.jar"), emptyString) should be
(Set("/tmp/a.jar"))
-Utils.unionFileLists(emptyString, Option("/tmp/a.jar")) should be
(Set("/tmp/a.jar"))
-Utils.unionFileLists(None, Option("/tmp/a.jar")) should be
(Set("/tmp/a.jar"))
-Utils.unionFileLists(Option("/tmp/a.jar"), None) should be
(Set("/tmp/a.jar"))
- }
// scalastyle:on println
// NOTE: This is an expensive operation in terms of time (10 seconds+).
spark git commit: Revert "[SPARK-15782][YARN] Set spark.jars system property in client mode"
Repository: spark
Updated Branches:
refs/heads/master 1a33f2e05 -> a153e41c0
Revert "[SPARK-15782][YARN] Set spark.jars system property in client mode"
This reverts commit 4df8df5c2e68f5a5d231c401b04d762d7a648159.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a153e41c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a153e41c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a153e41c
Branch: refs/heads/master
Commit: a153e41c08b635b1b2a48b680d203c148deebc67
Parents: 1a33f2e
Author: Davies Liu
Authored: Wed Jun 15 15:55:07 2016 -0700
Committer: Davies Liu
Committed: Wed Jun 15 15:55:07 2016 -0700
--
.../scala/org/apache/spark/SparkContext.scala | 2 +-
.../scala/org/apache/spark/util/Utils.scala | 20
.../apache/spark/deploy/SparkSubmitSuite.scala | 12
.../org/apache/spark/repl/SparkILoop.scala | 11 ++-
.../main/scala/org/apache/spark/repl/Main.scala | 4 +++-
5 files changed, 6 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a153e41c/core/src/main/scala/org/apache/spark/SparkContext.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index d870181..d56946e 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -391,7 +391,7 @@ class SparkContext(config: SparkConf) extends Logging with
ExecutorAllocationCli
_conf.set("spark.executor.id", SparkContext.DRIVER_IDENTIFIER)
-_jars = Utils.getUserJars(_conf)
+_jars =
_conf.getOption("spark.jars").map(_.split(",")).map(_.filter(_.nonEmpty)).toSeq.flatten
_files =
_conf.getOption("spark.files").map(_.split(",")).map(_.filter(_.nonEmpty))
.toSeq.flatten
http://git-wip-us.apache.org/repos/asf/spark/blob/a153e41c/core/src/main/scala/org/apache/spark/util/Utils.scala
--
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index aebd98b..f9d0540 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2352,26 +2352,6 @@ private[spark] object Utils extends Logging {
log.info(s"Started daemon with process name: ${Utils.getProcessName()}")
SignalUtils.registerLogger(log)
}
-
- /**
- * Unions two comma-separated lists of files and filters out empty strings.
- */
- def unionFileLists(leftList: Option[String], rightList: Option[String]):
Set[String] = {
-var allFiles = Set[String]()
-leftList.foreach { value => allFiles ++= value.split(",") }
-rightList.foreach { value => allFiles ++= value.split(",") }
-allFiles.filter { _.nonEmpty }
- }
-
- def getUserJars(conf: SparkConf): Seq[String] = {
-val sparkJars = conf.getOption("spark.jars")
-if (conf.get("spark.master") == "yarn") {
- val yarnJars = conf.getOption("spark.yarn.dist.jars")
- unionFileLists(sparkJars, yarnJars).toSeq
-} else {
- sparkJars.map(_.split(",")).map(_.filter(_.nonEmpty)).toSeq.flatten
-}
- }
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/a153e41c/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 0b02059..2718976 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -570,18 +570,6 @@ class SparkSubmitSuite
appArgs.executorMemory should be ("2.3g")
}
}
-
- test("comma separated list of files are unioned correctly") {
-val left = Option("/tmp/a.jar,/tmp/b.jar")
-val right = Option("/tmp/c.jar,/tmp/a.jar")
-val emptyString = Option("")
-Utils.unionFileLists(left, right) should be (Set("/tmp/a.jar",
"/tmp/b.jar", "/tmp/c.jar"))
-Utils.unionFileLists(emptyString, emptyString) should be (Set.empty)
-Utils.unionFileLists(Option("/tmp/a.jar"), emptyString) should be
(Set("/tmp/a.jar"))
-Utils.unionFileLists(emptyString, Option("/tmp/a.jar")) should be
(Set("/tmp/a.jar"))
-Utils.unionFileLists(None, Option("/tmp/a.jar")) should be
(Set("/tmp/a.jar"))
-Utils.unionFileLists(Option("/tmp/a.jar"), None) should be
(Set("/tmp/a.jar"))
- }
// scalastyle:on println
// NOTE: This is an expensive operation in terms of time (10 seconds+). Use
sp
