spark git commit: [SPARK-12345][MESOS] Filter SPARK_HOME when submitting Spark jobs with Mesos cluster mode.
Repository: spark Updated Branches: refs/heads/master 26d70bd2b -> ad8c1f0b8 [SPARK-12345][MESOS] Filter SPARK_HOME when submitting Spark jobs with Mesos cluster mode. SPARK_HOME is now causing problem with Mesos cluster mode since spark-submit script has been changed recently to take precendence when running spark-class scripts to look in SPARK_HOME if it's defined. We should skip passing SPARK_HOME from the Spark client in cluster mode with Mesos, since Mesos shouldn't use this configuration but should use spark.executor.home instead. Author: Timothy Chen Closes #10332 from tnachen/scheduler_ui. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ad8c1f0b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ad8c1f0b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ad8c1f0b Branch: refs/heads/master Commit: ad8c1f0b840284d05da737fb2cc5ebf8848f4490 Parents: 26d70bd Author: Timothy Chen Authored: Wed Dec 16 10:54:15 2015 -0800 Committer: Andrew Or Committed: Wed Dec 16 10:54:15 2015 -0800 -- .../org/apache/spark/deploy/rest/mesos/MesosRestServer.scala | 7 ++- .../spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ad8c1f0b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index 868cc35..24510db 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -94,7 +94,12 @@ private[mesos] class MesosSubmitRequestServlet( val driverMemory = sparkProperties.get("spark.driver.memory") val driverCores = sparkProperties.get("spark.driver.cores") val appArgs = request.appArgs -val environmentVariables = request.environmentVariables +// We don't want to pass down SPARK_HOME when launching Spark apps +// with Mesos cluster mode since it's populated by default on the client and it will +// cause spark-submit script to look for files in SPARK_HOME instead. +// We only need the ability to specify where to find spark-submit script +// which user can user spark.executor.home or spark.home configurations. +val environmentVariables = request.environmentVariables.filter(!_.equals("SPARK_HOME")) val name = request.sparkProperties.get("spark.app.name").getOrElse(mainClass) // Construct driver description http://git-wip-us.apache.org/repos/asf/spark/blob/ad8c1f0b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala index 721861f..573355b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala @@ -34,7 +34,7 @@ import org.apache.spark.util.Utils /** * Shared trait for implementing a Mesos Scheduler. This holds common state and helper - * methods and Mesos scheduler will use. + * methods the Mesos scheduler will use. */ private[mesos] trait MesosSchedulerUtils extends Logging { // Lock used to wait for scheduler to be registered - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-12345][MESOS] Filter SPARK_HOME when submitting Spark jobs with Mesos cluster mode.
Repository: spark Updated Branches: refs/heads/branch-1.6 f81512729 -> e5b85713d [SPARK-12345][MESOS] Filter SPARK_HOME when submitting Spark jobs with Mesos cluster mode. SPARK_HOME is now causing problem with Mesos cluster mode since spark-submit script has been changed recently to take precendence when running spark-class scripts to look in SPARK_HOME if it's defined. We should skip passing SPARK_HOME from the Spark client in cluster mode with Mesos, since Mesos shouldn't use this configuration but should use spark.executor.home instead. Author: Timothy Chen Closes #10332 from tnachen/scheduler_ui. (cherry picked from commit ad8c1f0b840284d05da737fb2cc5ebf8848f4490) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e5b85713 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e5b85713 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e5b85713 Branch: refs/heads/branch-1.6 Commit: e5b85713d8a0dbbb1a0a07481f5afa6c5098147f Parents: f815127 Author: Timothy Chen Authored: Wed Dec 16 10:54:15 2015 -0800 Committer: Andrew Or Committed: Wed Dec 16 10:55:25 2015 -0800 -- .../org/apache/spark/deploy/rest/mesos/MesosRestServer.scala | 7 ++- .../spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e5b85713/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index 868cc35..7c01ae4 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -94,7 +94,12 @@ private[mesos] class MesosSubmitRequestServlet( val driverMemory = sparkProperties.get("spark.driver.memory") val driverCores = sparkProperties.get("spark.driver.cores") val appArgs = request.appArgs -val environmentVariables = request.environmentVariables +// We don't want to pass down SPARK_HOME when launching Spark apps with Mesos cluster mode +// since it's populated by default on the client and it will cause spark-submit script to +// look for files in SPARK_HOME instead. We only need the ability to specify where to find +// spark-submit script which user can user spark.executor.home or spark.home configurations +// (SPARK-12345). +val environmentVariables = request.environmentVariables.filter(!_.equals("SPARK_HOME")) val name = request.sparkProperties.get("spark.app.name").getOrElse(mainClass) // Construct driver description http://git-wip-us.apache.org/repos/asf/spark/blob/e5b85713/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala index 721861f..573355b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala @@ -34,7 +34,7 @@ import org.apache.spark.util.Utils /** * Shared trait for implementing a Mesos Scheduler. This holds common state and helper - * methods and Mesos scheduler will use. + * methods the Mesos scheduler will use. */ private[mesos] trait MesosSchedulerUtils extends Logging { // Lock used to wait for scheduler to be registered - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org