Repository: spark Updated Branches: refs/heads/master 232d7f8d4 -> fc26f32cf
[SPARK-9708][MESOS] Spark should create local temporary directories in Mesos sandbox when launched with Mesos This is my own original work and I license this to the project under the project's open source license Author: Chris Bannister <chris.bannis...@swiftkey.com> Author: Chris Bannister <chris.bannis...@swiftkey.net> Closes #8358 from Zariel/mesos-local-dir. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc26f32c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc26f32c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc26f32c Branch: refs/heads/master Commit: fc26f32cf1bede8b9a1343dca0c0182107c9985e Parents: 232d7f8 Author: Chris Bannister <chris.bannis...@swiftkey.com> Authored: Mon Oct 19 16:24:40 2015 -0700 Committer: Andrew Or <and...@databricks.com> Committed: Mon Oct 19 16:24:40 2015 -0700 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/util/Utils.scala | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/fc26f32c/core/src/main/scala/org/apache/spark/util/Utils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 22c05a2..5595040 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -649,6 +649,7 @@ private[spark] object Utils extends Logging { * logic of locating the local directories according to deployment mode. */ def getConfiguredLocalDirs(conf: SparkConf): Array[String] = { + val shuffleServiceEnabled = conf.getBoolean("spark.shuffle.service.enabled", false) if (isRunningInYarnContainer(conf)) { // If we are in yarn mode, systems can have different disk layouts so we must set it // to what Yarn on this system said was available. Note this assumes that Yarn has @@ -657,13 +658,23 @@ private[spark] object Utils extends Logging { getYarnLocalDirs(conf).split(",") } else if (conf.getenv("SPARK_EXECUTOR_DIRS") != null) { conf.getenv("SPARK_EXECUTOR_DIRS").split(File.pathSeparator) + } else if (conf.getenv("SPARK_LOCAL_DIRS") != null) { + conf.getenv("SPARK_LOCAL_DIRS").split(",") + } else if (conf.getenv("MESOS_DIRECTORY") != null && !shuffleServiceEnabled) { + // Mesos already creates a directory per Mesos task. Spark should use that directory + // instead so all temporary files are automatically cleaned up when the Mesos task ends. + // Note that we don't want this if the shuffle service is enabled because we want to + // continue to serve shuffle files after the executors that wrote them have already exited. + Array(conf.getenv("MESOS_DIRECTORY")) } else { + if (conf.getenv("MESOS_DIRECTORY") != null && shuffleServiceEnabled) { + logInfo("MESOS_DIRECTORY available but not using provided Mesos sandbox because " + + "spark.shuffle.service.enabled is enabled.") + } // In non-Yarn mode (or for the driver in yarn-client mode), we cannot trust the user // configuration to point to a secure directory. So create a subdirectory with restricted // permissions under each listed directory. - Option(conf.getenv("SPARK_LOCAL_DIRS")) - .getOrElse(conf.get("spark.local.dir", System.getProperty("java.io.tmpdir"))) - .split(",") + conf.get("spark.local.dir", System.getProperty("java.io.tmpdir")).split(",") } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org