spark git commit: [SPARK-8099] set executor cores into system in yarn-cluster mode
Repository: spark Updated Branches: refs/heads/master 4036d05ce -> 0992a0a77 [SPARK-8099] set executor cores into system in yarn-cluster mode Author: Xutingjun Author: xutingjun Closes #6643 from XuTingjun/SPARK-8099 and squashes the following commits: 80b18cd [Xutingjun] change to STANDALONE | YARN ce33148 [Xutingjun] set executor cores into system e51cc9e [Xutingjun] set executor cores into system 0600861 [xutingjun] set executor cores into system Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0992a0a7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0992a0a7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0992a0a7 Branch: refs/heads/master Commit: 0992a0a77d38081c6c206bb34333013125d85376 Parents: 4036d05 Author: Xutingjun Authored: Fri Jun 5 11:41:39 2015 -0700 Committer: Sandy Ryza Committed: Fri Jun 5 11:41:39 2015 -0700 -- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/0992a0a7/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 3aa3f94..a0eae77 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -425,7 +425,6 @@ object SparkSubmit { // Yarn client only OptionAssigner(args.queue, YARN, CLIENT, sysProp = "spark.yarn.queue"), OptionAssigner(args.numExecutors, YARN, CLIENT, sysProp = "spark.executor.instances"), - OptionAssigner(args.executorCores, YARN, CLIENT, sysProp = "spark.executor.cores"), OptionAssigner(args.files, YARN, CLIENT, sysProp = "spark.yarn.dist.files"), OptionAssigner(args.archives, YARN, CLIENT, sysProp = "spark.yarn.dist.archives"), OptionAssigner(args.principal, YARN, CLIENT, sysProp = "spark.yarn.principal"), @@ -446,7 +445,7 @@ object SparkSubmit { OptionAssigner(args.keytab, YARN, CLUSTER, clOption = "--keytab"), // Other options - OptionAssigner(args.executorCores, STANDALONE, ALL_DEPLOY_MODES, + OptionAssigner(args.executorCores, STANDALONE | YARN, ALL_DEPLOY_MODES, sysProp = "spark.executor.cores"), OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN, ALL_DEPLOY_MODES, sysProp = "spark.executor.memory"), - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-7699] [CORE] Lazy start the scheduler for dynamic allocation
Repository: spark Updated Branches: refs/heads/master 0992a0a77 -> 3f80bc841 [SPARK-7699] [CORE] Lazy start the scheduler for dynamic allocation This patch propose to lazy start the scheduler for dynamic allocation to avoid fast ramp down executor numbers is load is less. This implementation will: 1. immediately start the scheduler is `numExecutorsTarget` is 0, this is the expected behavior. 2. if `numExecutorsTarget` is not zero, start the scheduler until the number is satisfied, if the load is less, this initial started executors will last for at least 60 seconds, user will have a window to submit a job, no need to revamp the executors. 3. if `numExecutorsTarget` is not satisfied until the timeout, this means resource is not enough, the scheduler will start until this timeout, will not wait infinitely. Please help to review, thanks a lot. Author: jerryshao Closes #6430 from jerryshao/SPARK-7699 and squashes the following commits: 02cac8e [jerryshao] Address the comments 7242450 [jerryshao] Remove the useless import ecc0b00 [jerryshao] Address the comments 6f75f00 [jerryshao] Style changes 8b8decc [jerryshao] change the test name fb822ca [jerryshao] Change the solution according to comments 1cc74e5 [jerryshao] Lazy start the scheduler for dynamic allocation Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3f80bc84 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3f80bc84 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3f80bc84 Branch: refs/heads/master Commit: 3f80bc841ab155925fb0530eef5927990f4a5793 Parents: 0992a0a Author: jerryshao Authored: Fri Jun 5 12:28:37 2015 -0700 Committer: Sandy Ryza Committed: Fri Jun 5 12:28:37 2015 -0700 -- .../spark/ExecutorAllocationManager.scala | 17 +++- .../spark/ExecutorAllocationManagerSuite.scala | 90 2 files changed, 89 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3f80bc84/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala index f7323a4..9939103 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala @@ -150,6 +150,13 @@ private[spark] class ExecutorAllocationManager( // Metric source for ExecutorAllocationManager to expose internal status to MetricsSystem. val executorAllocationManagerSource = new ExecutorAllocationManagerSource + // Whether we are still waiting for the initial set of executors to be allocated. + // While this is true, we will not cancel outstanding executor requests. This is + // set to false when: + // (1) a stage is submitted, or + // (2) an executor idle timeout has elapsed. + @volatile private var initializing: Boolean = true + /** * Verify that the settings specified through the config are valid. * If not, throw an appropriate exception. @@ -240,6 +247,7 @@ private[spark] class ExecutorAllocationManager( removeTimes.retain { case (executorId, expireTime) => val expired = now >= expireTime if (expired) { +initializing = false removeExecutor(executorId) } !expired @@ -261,7 +269,11 @@ private[spark] class ExecutorAllocationManager( private def updateAndSyncNumExecutorsTarget(now: Long): Int = synchronized { val maxNeeded = maxNumExecutorsNeeded -if (maxNeeded < numExecutorsTarget) { +if (initializing) { + // Do not change our target while we are still initializing, + // Otherwise the first job may have to ramp up unnecessarily + 0 +} else if (maxNeeded < numExecutorsTarget) { // The target number exceeds the number we actually need, so stop adding new // executors and inform the cluster manager to cancel the extra pending requests val oldNumExecutorsTarget = numExecutorsTarget @@ -271,7 +283,7 @@ private[spark] class ExecutorAllocationManager( // If the new target has not changed, avoid sending a message to the cluster manager if (numExecutorsTarget < oldNumExecutorsTarget) { client.requestTotalExecutors(numExecutorsTarget) -logInfo(s"Lowering target number of executors to $numExecutorsTarget (previously " + +logDebug(s"Lowering target number of executors to $numExecutorsTarget (previously " + s"$oldNumExecutorsTarget) because not all requested executors are actually needed") } numExecutorsTarget - oldNumExecutorsTarget @@ -481,6 +493,7 @
spark git commit: [SPARK-9782] [YARN] Support YARN application tags via SparkConf
Repository: spark Updated Branches: refs/heads/master 80cb25b22 -> 9b731fad2 [SPARK-9782] [YARN] Support YARN application tags via SparkConf Add a new test case in yarn/ClientSuite which checks how the various SparkConf and ClientArguments propagate into the ApplicationSubmissionContext. Author: Dennis Huo Closes #8072 from dennishuo/dhuo-yarn-application-tags. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9b731fad Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9b731fad Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9b731fad Branch: refs/heads/master Commit: 9b731fad2b43ca18f3c5274062d4c7bc2622ab72 Parents: 80cb25b Author: Dennis Huo Authored: Tue Aug 18 14:34:20 2015 -0700 Committer: Sandy Ryza Committed: Tue Aug 18 14:34:20 2015 -0700 -- docs/running-on-yarn.md | 8 + .../org/apache/spark/deploy/yarn/Client.scala | 21 .../apache/spark/deploy/yarn/ClientSuite.scala | 36 3 files changed, 65 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9b731fad/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index ec32c41..8ac26e9 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -320,6 +320,14 @@ If you need a reference to the proper location to put log files in the YARN so t + spark.yarn.tags + (none) + + Comma-separated list of strings to pass through as YARN application tags appearing + in YARN ApplicationReports, which can be used for filtering when querying YARN apps. + + + spark.yarn.keytab (none) http://git-wip-us.apache.org/repos/asf/spark/blob/9b731fad/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala -- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 6d63dda..5c6a716 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -163,6 +163,23 @@ private[spark] class Client( appContext.setQueue(args.amQueue) appContext.setAMContainerSpec(containerContext) appContext.setApplicationType("SPARK") +sparkConf.getOption(CONF_SPARK_YARN_APPLICATION_TAGS) + .map(StringUtils.getTrimmedStringCollection(_)) + .filter(!_.isEmpty()) + .foreach { tagCollection => +try { + // The setApplicationTags method was only introduced in Hadoop 2.4+, so we need to use + // reflection to set it, printing a warning if a tag was specified but the YARN version + // doesn't support it. + val method = appContext.getClass().getMethod( +"setApplicationTags", classOf[java.util.Set[String]]) + method.invoke(appContext, new java.util.HashSet[String](tagCollection)) +} catch { + case e: NoSuchMethodException => +logWarning(s"Ignoring $CONF_SPARK_YARN_APPLICATION_TAGS because this version of " + + "YARN does not support it") +} + } sparkConf.getOption("spark.yarn.maxAppAttempts").map(_.toInt) match { case Some(v) => appContext.setMaxAppAttempts(v) case None => logDebug("spark.yarn.maxAppAttempts is not set. " + @@ -987,6 +1004,10 @@ object Client extends Logging { // of the executors val CONF_SPARK_YARN_SECONDARY_JARS = "spark.yarn.secondary.jars" + // Comma-separated list of strings to pass through as YARN application tags appearing + // in YARN ApplicationReports, which can be used for filtering when querying YARN. + val CONF_SPARK_YARN_APPLICATION_TAGS = "spark.yarn.tags" + // Staging directory is private! -> rwx val STAGING_DIR_PERMISSION: FsPermission = FsPermission.createImmutable(Integer.parseInt("700", 8).toShort) http://git-wip-us.apache.org/repos/asf/spark/blob/9b731fad/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala -- diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala index 837f8d3..0a5402c 100644 --- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala +++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala @@ -29,8 +29,11 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import or
spark git commit: [SPARK-7451] [YARN] Preemption of executors is counted as failure causing Spark job to fail
Repository: spark Updated Branches: refs/heads/master 84bf931f3 -> b6c797b08 [SPARK-7451] [YARN] Preemption of executors is counted as failure causing Spark job to fail Added a check to handle container exit status for the preemption scenario, log an INFO message in such cases and move on. andrewor14 Author: Ashwin Shankar Closes #5993 from ashwinshankar77/SPARK-7451 and squashes the following commits: 90900cf [Ashwin Shankar] Fix log info message cf8b6cf [Ashwin Shankar] Stop counting preemption of executors as failure Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b6c797b0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b6c797b0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b6c797b0 Branch: refs/heads/master Commit: b6c797b08cbd08d7aab59ad0106af0f5f41ef186 Parents: 84bf931 Author: Ashwin Shankar Authored: Fri May 8 17:51:00 2015 -0700 Committer: Sandy Ryza Committed: Fri May 8 17:51:00 2015 -0700 -- .../main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b6c797b0/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala -- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index b8f42da..88d68d5 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -373,7 +373,9 @@ private[yarn] class YarnAllocator( // Hadoop 2.2.X added a ContainerExitStatus we should switch to use // there are some exit status' we shouldn't necessarily count against us, but for // now I think its ok as none of the containers are expected to exit -if (completedContainer.getExitStatus == -103) { // vmem limit exceeded +if (completedContainer.getExitStatus == ContainerExitStatus.PREEMPTED) { + logInfo("Container preempted: " + containerId) +} else if (completedContainer.getExitStatus == -103) { // vmem limit exceeded logWarning(memLimitExceededLogMessage( completedContainer.getDiagnostics, VMEM_EXCEEDED_PATTERN)) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-7451] [YARN] Preemption of executors is counted as failure causing Spark job to fail
Repository: spark Updated Branches: refs/heads/branch-1.4 85cab3482 -> 959c7b6ca [SPARK-7451] [YARN] Preemption of executors is counted as failure causing Spark job to fail Added a check to handle container exit status for the preemption scenario, log an INFO message in such cases and move on. andrewor14 Author: Ashwin Shankar Closes #5993 from ashwinshankar77/SPARK-7451 and squashes the following commits: 90900cf [Ashwin Shankar] Fix log info message cf8b6cf [Ashwin Shankar] Stop counting preemption of executors as failure (cherry picked from commit b6c797b08cbd08d7aab59ad0106af0f5f41ef186) Signed-off-by: Sandy Ryza Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/959c7b6c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/959c7b6c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/959c7b6c Branch: refs/heads/branch-1.4 Commit: 959c7b6ca422082de9e89703e3c7b926d93d30d1 Parents: 85cab34 Author: Ashwin Shankar Authored: Fri May 8 17:51:00 2015 -0700 Committer: Sandy Ryza Committed: Fri May 8 17:51:46 2015 -0700 -- .../main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/959c7b6c/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala -- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index b8f42da..88d68d5 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -373,7 +373,9 @@ private[yarn] class YarnAllocator( // Hadoop 2.2.X added a ContainerExitStatus we should switch to use // there are some exit status' we shouldn't necessarily count against us, but for // now I think its ok as none of the containers are expected to exit -if (completedContainer.getExitStatus == -103) { // vmem limit exceeded +if (completedContainer.getExitStatus == ContainerExitStatus.PREEMPTED) { + logInfo("Container preempted: " + containerId) +} else if (completedContainer.getExitStatus == -103) { // vmem limit exceeded logWarning(memLimitExceededLogMessage( completedContainer.getDiagnostics, VMEM_EXCEEDED_PATTERN)) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6470] [YARN] Add support for YARN node labels.
Repository: spark Updated Branches: refs/heads/master 0a4844f90 -> 82fee9d9a [SPARK-6470] [YARN] Add support for YARN node labels. This is difficult to write a test for because it relies on the latest version of YARN, but I verified manually that the patch does pass along the label expression on this version and containers are successfully launched. Author: Sandy Ryza Closes #5242 from sryza/sandy-spark-6470 and squashes the following commits: 6af87b9 [Sandy Ryza] Change info to warning 6e22d99 [Sandy Ryza] [YARN] SPARK-6470. Add support for YARN node labels. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/82fee9d9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/82fee9d9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/82fee9d9 Branch: refs/heads/master Commit: 82fee9d9aad2c9ba2fb4bd658579fe99218cafac Parents: 0a4844f Author: Sandy Ryza Authored: Mon May 11 12:09:39 2015 -0700 Committer: Sandy Ryza Committed: Mon May 11 12:09:39 2015 -0700 -- docs/running-on-yarn.md | 9 ++ .../spark/deploy/yarn/YarnAllocator.scala | 31 +++- 2 files changed, 39 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/82fee9d9/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 4fb4a90..51c1339 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -220,6 +220,15 @@ Most of the configs are the same for Spark on YARN as for other deployment modes Otherwise, the client process will exit after submission. + + spark.yarn.executor.nodeLabelExpression + (none) + + A YARN node label expression that restricts the set of nodes executors will be scheduled on. + Only versions of YARN greater than or equal to 2.6 support node label expressions, so when + running against earlier versions, this property will be ignored. + + # Launching Spark on YARN http://git-wip-us.apache.org/repos/asf/spark/blob/82fee9d9/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala -- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 88d68d5..8a08f56 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -117,6 +117,24 @@ private[yarn] class YarnAllocator( // For testing private val launchContainers = sparkConf.getBoolean("spark.yarn.launchContainers", true) + private val labelExpression = sparkConf.getOption("spark.yarn.executor.nodeLabelExpression") + + // ContainerRequest constructor that can take a node label expression. We grab it through + // reflection because it's only available in later versions of YARN. + private val nodeLabelConstructor = labelExpression.flatMap { expr => +try { + Some(classOf[ContainerRequest].getConstructor(classOf[Resource], +classOf[Array[String]], classOf[Array[String]], classOf[Priority], classOf[Boolean], +classOf[String])) +} catch { + case e: NoSuchMethodException => { +logWarning(s"Node label expression $expr will be ignored because YARN version on" + + " classpath does not support it.") +None + } +} + } + def getNumExecutorsRunning: Int = numExecutorsRunning def getNumExecutorsFailed: Int = numExecutorsFailed @@ -211,7 +229,7 @@ private[yarn] class YarnAllocator( s"cores and ${resource.getMemory} MB memory including $memoryOverhead MB overhead") for (i <- 0 until missing) { -val request = new ContainerRequest(resource, null, null, RM_REQUEST_PRIORITY) +val request = createContainerRequest(resource) amClient.addContainerRequest(request) val nodes = request.getNodes val hostStr = if (nodes == null || nodes.isEmpty) "Any" else nodes.last @@ -231,6 +249,17 @@ private[yarn] class YarnAllocator( } /** + * Creates a container request, handling the reflection required to use YARN features that were + * added in recent versions. + */ + private def createContainerRequest(resource: Resource): ContainerRequest = { +nodeLabelConstructor.map { constructor => + constructor.newInstance(resource, null, null, RM_REQUEST_PRIORITY, true: java.lang.Boolean, +labelExpression.orNull) +}.getOrElse(new ContainerRequest(resource, null, null, RM_REQUEST_PRIORITY)) + } + + /** * Handle containers granted by the R
spark git commit: [SPARK-7515] [DOC] Update documentation for PySpark on YARN with cluster mode
Repository: spark Updated Branches: refs/heads/master 7ce2a33c3 -> 6e9910c21 [SPARK-7515] [DOC] Update documentation for PySpark on YARN with cluster mode Now PySpark on YARN with cluster mode is supported so let's update doc. Author: Kousuke Saruta Closes #6040 from sarutak/update-doc-for-pyspark-on-yarn and squashes the following commits: ad9f88c [Kousuke Saruta] Brushed up sentences 469fd2e [Kousuke Saruta] Merge branch 'master' of https://github.com/apache/spark into update-doc-for-pyspark-on-yarn fcfdb92 [Kousuke Saruta] Updated doc for PySpark on YARN with cluster mode Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e9910c2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e9910c2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e9910c2 Branch: refs/heads/master Commit: 6e9910c21ada19ae14122b5f14a6a30845b98229 Parents: 7ce2a33 Author: Kousuke Saruta Authored: Mon May 11 14:19:11 2015 -0700 Committer: Sandy Ryza Committed: Mon May 11 14:19:11 2015 -0700 -- docs/submitting-applications.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6e9910c2/docs/submitting-applications.md -- diff --git a/docs/submitting-applications.md b/docs/submitting-applications.md index 3ecbf23..e586452 100644 --- a/docs/submitting-applications.md +++ b/docs/submitting-applications.md @@ -59,7 +59,7 @@ for applications that involve the REPL (e.g. Spark shell). Alternatively, if your application is submitted from a machine far from the worker machines (e.g. locally on your laptop), it is common to use `cluster` mode to minimize network latency between the drivers and the executors. Note that `cluster` mode is currently not supported for -Mesos clusters or Python applications. +Mesos clusters. Currently only YARN supports cluster mode for Python applications. For Python applications, simply pass a `.py` file in the place of `` instead of a JAR, and add Python `.zip`, `.egg` or `.py` files to the search path with `--py-files`. - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-4352] [YARN] [WIP] Incorporate locality preferences in dynamic allocation requests
Repository: spark Updated Branches: refs/heads/master 2104931d7 -> ab6259566 [SPARK-4352] [YARN] [WIP] Incorporate locality preferences in dynamic allocation requests Currently there's no locality preference for container request in YARN mode, this will affect the performance if fetching data remotely, so here proposed to add locality in Yarn dynamic allocation mode. Ping sryza, please help to review, thanks a lot. Author: jerryshao Closes #6394 from jerryshao/SPARK-4352 and squashes the following commits: d45fecb [jerryshao] Add documents 6c3fe5c [jerryshao] Fix bug 8db6c0e [jerryshao] Further address the comments 2e2b2cb [jerryshao] Fix rebase compiling problem ce5f096 [jerryshao] Fix style issue 7f7df95 [jerryshao] Fix rebase issue 9ca9e07 [jerryshao] Code refactor according to comments d3e4236 [jerryshao] Further address the comments 5e7a593 [jerryshao] Fix bug introduced code rebase 9ca7783 [jerryshao] Style changes 08317f9 [jerryshao] code and comment refines 65b2423 [jerryshao] Further address the comments a27c587 [jerryshao] address the comment 27faabc [jerryshao] redundant code remove 9ce06a1 [jerryshao] refactor the code f5ba27b [jerryshao] Style fix 2c6cc8a [jerryshao] Fix bug and add unit tests 0757335 [jerryshao] Consider the distribution of existed containers to recalculate the new container requests 0ad66ff [jerryshao] Fix compile bugs 1c20381 [jerryshao] Minor fix 5ef2dc8 [jerryshao] Add docs and improve the code 3359814 [jerryshao] Fix rebase and test bugs 0398539 [jerryshao] reinitialize the new implementation 67596d6 [jerryshao] Still fix the code 654e1d2 [jerryshao] Fix some bugs 45b1c89 [jerryshao] Further polish the algorithm dea0152 [jerryshao] Enable node locality information in YarnAllocator 74bbcc6 [jerryshao] Support node locality for dynamic allocation initial commit Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ab625956 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ab625956 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ab625956 Branch: refs/heads/master Commit: ab625956616664c2b4861781a578311da75a9ae4 Parents: 2104931 Author: jerryshao Authored: Mon Jul 27 15:46:35 2015 -0700 Committer: Sandy Ryza Committed: Mon Jul 27 15:46:35 2015 -0700 -- .../apache/spark/ExecutorAllocationClient.scala | 18 +- .../spark/ExecutorAllocationManager.scala | 62 ++- .../scala/org/apache/spark/SparkContext.scala | 25 ++- .../apache/spark/scheduler/DAGScheduler.scala | 26 ++- .../org/apache/spark/scheduler/Stage.scala | 7 +- .../org/apache/spark/scheduler/StageInfo.scala | 13 +- .../cluster/CoarseGrainedClusterMessage.scala | 6 +- .../cluster/CoarseGrainedSchedulerBackend.scala | 32 +++- .../cluster/YarnSchedulerBackend.scala | 3 +- .../spark/ExecutorAllocationManagerSuite.scala | 55 +- .../apache/spark/HeartbeatReceiverSuite.scala | 7 +- .../spark/deploy/yarn/ApplicationMaster.scala | 5 +- ...ityPreferredContainerPlacementStrategy.scala | 182 +++ .../spark/deploy/yarn/YarnAllocator.scala | 47 - .../yarn/ContainerPlacementStrategySuite.scala | 125 + .../spark/deploy/yarn/YarnAllocatorSuite.scala | 14 +- 16 files changed, 578 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ab625956/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala -- diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala index 443830f..842bfdb 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala @@ -24,11 +24,23 @@ package org.apache.spark private[spark] trait ExecutorAllocationClient { /** - * Express a preference to the cluster manager for a given total number of executors. - * This can result in canceling pending requests or filing additional requests. + * Update the cluster manager on our scheduling needs. Three bits of information are included + * to help it make decisions. + * @param numExecutors The total number of executors we'd like to have. The cluster manager + * shouldn't kill any running executor to reach this number, but, + * if all existing executors were to die, this is the number of executors + * we'd want to be allocated. + * @param localityAwareTasks The number of tasks in all active stages that have a locality + * preferences. This includes running, pending, and completed tasks. + * @param hos