[1/3] spark-website git commit: Spark 2.2.2 release announcements
Repository: spark-website Updated Branches: refs/heads/asf-site 390743e8a -> 179a5897a http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-1-6-2.html -- diff --git a/site/releases/spark-release-1-6-2.html b/site/releases/spark-release-1-6-2.html index 5f459ce..d23f38d 100644 --- a/site/releases/spark-release-1-6-2.html +++ b/site/releases/spark-release-1-6-2.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-1-6-3.html -- diff --git a/site/releases/spark-release-1-6-3.html b/site/releases/spark-release-1-6-3.html index 6d9efd9..fb678c2 100644 --- a/site/releases/spark-release-1-6-3.html +++ b/site/releases/spark-release-1-6-3.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-0-0.html -- diff --git a/site/releases/spark-release-2-0-0.html b/site/releases/spark-release-2-0-0.html index 8d1ef52..40bd9f3 100644 --- a/site/releases/spark-release-2-0-0.html +++ b/site/releases/spark-release-2-0-0.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-0-1.html -- diff --git a/site/releases/spark-release-2-0-1.html b/site/releases/spark-release-2-0-1.html index 6adef9a..1860b0c 100644 --- a/site/releases/spark-release-2-0-1.html +++ b/site/releases/spark-release-2-0-1.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-0-2.html -- diff --git a/site/releases/spark-release-2-0-2.html b/site/releases/spark-release-2-0-2.html index 275bed6..9d903dd 100644 --- a/site/releases/spark-release-2-0-2.html +++ b/site/releases/spark-release-2-0-2.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-1-0.html -- diff --git a/site/releases/spark-release-2-1-0.html b/site/releases/spark-release-2-1-0.html index ddaff2f..bf6980a 100644 --- a/site/releases/spark-release-2-1-0.html +++ b/site/releases/spark-release-2-1-0.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http:/
[3/3] spark-website git commit: Spark 2.2.2 release announcements
Spark 2.2.2 release announcements 2.2.2 docs look fine, created the announcement and regenerated the docs. Author: Thomas Graves Closes #129 from tgravescs/spark222-announce. Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/179a5897 Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/179a5897 Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/179a5897 Branch: refs/heads/asf-site Commit: 179a5897a6b056ba84b3ef6fefac5cee020214ac Parents: 390743e Author: Thomas Graves Authored: Mon Jul 9 10:29:50 2018 -0500 Committer: Thomas Graves Committed: Mon Jul 9 10:29:50 2018 -0500 -- news/_posts/2018-07-02-spark-2-2-2-released.md | 14 ++ site/committers.html | 6 +++--- site/community.html | 6 +++--- site/contributing.html | 6 +++--- site/developer-tools.html| 6 +++--- site/documentation.html | 6 +++--- site/downloads.html | 6 +++--- site/examples.html | 6 +++--- site/faq.html| 6 +++--- site/graphx/index.html | 6 +++--- site/history.html| 6 +++--- site/improvement-proposals.html | 6 +++--- site/index.html | 6 +++--- site/mailing-lists.html | 6 +++--- site/mllib/index.html| 6 +++--- site/news/amp-camp-2013-registration-ope.html| 6 +++--- site/news/announcing-the-first-spark-summit.html | 6 +++--- site/news/fourth-spark-screencast-published.html | 6 +++--- site/news/index.html | 15 --- site/news/nsdi-paper.html| 6 +++--- site/news/one-month-to-spark-summit-2015.html| 6 +++--- site/news/proposals-open-for-spark-summit-east.html | 6 +++--- .../registration-open-for-spark-summit-east.html | 6 +++--- site/news/run-spark-and-shark-on-amazon-emr.html | 6 +++--- site/news/spark-0-6-1-and-0-5-2-released.html| 6 +++--- site/news/spark-0-6-2-released.html | 6 +++--- site/news/spark-0-7-0-released.html | 6 +++--- site/news/spark-0-7-2-released.html | 6 +++--- site/news/spark-0-7-3-released.html | 6 +++--- site/news/spark-0-8-0-released.html | 6 +++--- site/news/spark-0-8-1-released.html | 6 +++--- site/news/spark-0-9-0-released.html | 6 +++--- site/news/spark-0-9-1-released.html | 6 +++--- site/news/spark-0-9-2-released.html | 6 +++--- site/news/spark-1-0-0-released.html | 6 +++--- site/news/spark-1-0-1-released.html | 6 +++--- site/news/spark-1-0-2-released.html | 6 +++--- site/news/spark-1-1-0-released.html | 6 +++--- site/news/spark-1-1-1-released.html | 6 +++--- site/news/spark-1-2-0-released.html | 6 +++--- site/news/spark-1-2-1-released.html | 6 +++--- site/news/spark-1-2-2-released.html | 6 +++--- site/news/spark-1-3-0-released.html | 6 +++--- site/news/spark-1-4-0-released.html | 6 +++--- site/news/spark-1-4-1-released.html | 6 +++--- site/news/spark-1-5-0-released.html | 6 +++--- site/news/spark-1-5-1-released.html | 6 +++--- site/news/spark-1-5-2-released.html | 6 +++--- site/news/spark-1-6-0-released.html | 6 +++--- site/news/spark-1-6-1-released.html | 6 +++--- site/news/spark-1-6-2-released.html | 6 +++--- site/news/spark-1-6-3-released.html | 6 +++--- site/news/spark-2-0-0-released.html | 6 +++--- site/news/spark-2-0-1-released.html | 6 +++--- site/news/spark-2-0-2-released.html | 6 +++--- site/news/spark-2-1-0-released.html | 6 +++--- site/news/spark-2-1-1-released.html | 6 +++--- site/news/spark-2-1-2-released.html | 6 +++--- site/news/spark-2-1-3-released.html | 6 +++--- site/news/spark-2-2-0-released.html | 6 +++--- site/news/spark-2-2-1-released.html | 6 +++--- site/news/spark-2-3-0-released.html | 6 +++--- site/news/spark-2-3-1-released.html | 6 +++--- site/news/spark-2.0.0-preview.html |
[2/3] spark-website git commit: Spark 2.2.2 release announcements
http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-2-3-1-released.html -- diff --git a/site/news/spark-2-3-1-released.html b/site/news/spark-2-3-1-released.html index a6ab537..06b718a 100644 --- a/site/news/spark-2-3-1-released.html +++ b/site/news/spark-2-3-1-released.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-2.0.0-preview.html -- diff --git a/site/news/spark-2.0.0-preview.html b/site/news/spark-2.0.0-preview.html index e704bab..da54cef 100644 --- a/site/news/spark-2.0.0-preview.html +++ b/site/news/spark-2.0.0-preview.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-accepted-into-apache-incubator.html -- diff --git a/site/news/spark-accepted-into-apache-incubator.html b/site/news/spark-accepted-into-apache-incubator.html index 06ebb8d..696c6d0 100644 --- a/site/news/spark-accepted-into-apache-incubator.html +++ b/site/news/spark-accepted-into-apache-incubator.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-and-shark-in-the-news.html -- diff --git a/site/news/spark-and-shark-in-the-news.html b/site/news/spark-and-shark-in-the-news.html index f3f5f41..328d3c4 100644 --- a/site/news/spark-and-shark-in-the-news.html +++ b/site/news/spark-and-shark-in-the-news.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-becomes-tlp.html -- diff --git a/site/news/spark-becomes-tlp.html b/site/news/spark-becomes-tlp.html index 24051c2..981aaa7 100644 --- a/site/news/spark-becomes-tlp.html +++ b/site/news/spark-becomes-tlp.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-featured-in-wired.html -- diff --git a/site/news/spark-featured-in-wired.html b/site/news/spark-featured-in-wired.html index 3bd546f..709c5dc 100644 --- a/site/news/spark-featured-in-wired.html +++ b/site/news/spark-featured-in-wired.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/s
spark-website git commit: Adding missing spark 2.2.2 announcement html files
Repository: spark-website Updated Branches: refs/heads/asf-site 179a5897a -> 2b5ba2f62 Adding missing spark 2.2.2 announcement html files Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/2b5ba2f6 Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/2b5ba2f6 Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/2b5ba2f6 Branch: refs/heads/asf-site Commit: 2b5ba2f6247a97374acc3c800d4a4179a0c0a334 Parents: 179a589 Author: Thomas Graves Authored: Mon Jul 9 11:04:33 2018 -0500 Committer: Thomas Graves Committed: Mon Jul 9 11:04:33 2018 -0500 -- site/news/spark-2-2-2-released.html| 232 +++ site/releases/spark-release-2-2-2.html | 238 site/sitemap.xml | 8 +- 3 files changed, 474 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark-website/blob/2b5ba2f6/site/news/spark-2-2-2-released.html -- diff --git a/site/news/spark-2-2-2-released.html b/site/news/spark-2-2-2-released.html new file mode 100644 index 000..1c46315 --- /dev/null +++ b/site/news/spark-2-2-2-released.html @@ -0,0 +1,232 @@ + + + + + + + + + Spark 2.2.2 released | Apache Spark + + + + + + + + + + + + + + + + + var _gaq = _gaq || []; + _gaq.push(['_setAccount', 'UA-32518208-2']); + _gaq.push(['_trackPageview']); + (function() { +var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; +ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; +var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); + })(); + + + function trackOutboundLink(link, category, action) { +try { + _gaq.push(['_trackEvent', category , action]); +} catch(err){} + +setTimeout(function() { + document.location.href = link.href; +}, 100); + } + + + + + + + + +https://code.jquery.com/jquery.js";> +https://netdna.bootstrapcdn.com/bootstrap/3.0.3/js/bootstrap.min.js";> + + + + + + + + + + + Lightning-fast unified analytics engine + + + + + + + + + + Toggle navigation + + + + + + + + + + Download + + + Libraries + + + SQL and DataFrames + Spark Streaming + MLlib (machine learning) + GraphX (graph) + + Third-Party Projects + + + + + Documentation + + + Latest Release (Spark 2.3.1) + Older Versions and Other Resources + Frequently Asked Questions + + + Examples + + + Community + + + Mailing Lists & Resources + Contributing to Spark + Improvement Proposals (SPIP) + https://issues.apache.org/jira/browse/SPARK";>Issue Tracker + Powered By + Project Committers + Project History + + + + + Developers + + + Useful Developer Tools + Versioning Policy + Release Process + Security + + + + + +https://www.apache.org/"; class="dropdown-toggle" data-toggle="dropdown"> + Apache Software Foundation + + https://www.apache.org/";>Apache Homepage + https://www.apache.org/licenses/";>License + https://www.apache.org/foundation/sponsorship.html";>Sponsorship + https://www.apache.org/foundation/thanks.html";>Thanks + https://www.apache.org/security/";>Security + + + + + + + + + + + + Latest News + + + Spark 2.2.2 released + (Jul 02, 2018) + + Spark 2.1.3 released + (Jun 29, 2018) + + Spark 2.3.1 released + (Jun 08, 2018) + + Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted + (Mar 01, 2018) + + + Archive + + + https://www.apache.org/events/current-event.html";> +https://www.apache.org/events/current-event-234x60.png"/> + + + + +Download Spark + + +Built-in Libraries: + + +SQL and DataFrames +Spark Streaming +MLlib (machine learning) +
svn commit: r28003 - /dev/spark/v2.2.2-rc2-docs/
Author: tgraves Date: Mon Jul 9 16:12:21 2018 New Revision: 28003 Log: Removing RC artifacts. Removed: dev/spark/v2.2.2-rc2-docs/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24610] fix reading small files via wholeTextFiles
Repository: spark Updated Branches: refs/heads/master 9fa4a1ed3 -> 1055c94cd [SPARK-24610] fix reading small files via wholeTextFiles ## What changes were proposed in this pull request? The `WholeTextFileInputFormat` determines the `maxSplitSize` for the file/s being read using the `wholeTextFiles` method. While this works well for large files, for smaller files where the maxSplitSize is smaller than the defaults being used with configs like hive-site.xml or explicitly passed in the form of `mapreduce.input.fileinputformat.split.minsize.per.node` or `mapreduce.input.fileinputformat.split.minsize.per.rack` , it just throws up an exception. ```java java.io.IOException: Minimum split size pernode 123456 cannot be larger than maximum split size 9962 at org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.getSplits(CombineFileInputFormat.java:200) at org.apache.spark.rdd.WholeTextFileRDD.getPartitions(WholeTextFileRDD.scala:50) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:250) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2096) at org.apache.spark.rdd.RDD.count(RDD.scala:1158) ... 48 elided ` This change checks the maxSplitSize against the minSplitSizePerNode and minSplitSizePerRack and set them if `maxSplitSize < minSplitSizePerNode/Rack` ## How was this patch tested? Test manually setting the conf while launching the job and added unit test. Author: Dhruve Ashar Closes #21601 from dhruve/bug/SPARK-24610. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1055c94c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1055c94c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1055c94c Branch: refs/heads/master Commit: 1055c94cdf072bfce5e36bb6552fe9b148bb9d17 Parents: 9fa4a1e Author: Dhruve Ashar Authored: Thu Jul 12 15:36:02 2018 -0500 Committer: Thomas Graves Committed: Thu Jul 12 15:36:02 2018 -0500 -- .../spark/input/WholeTextFileInputFormat.scala | 13 +++ .../input/WholeTextFileInputFormatSuite.scala | 96 2 files changed, 109 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1055c94c/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala -- diff --git a/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala b/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala index f47cd38..04c5c4b 100644 --- a/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala +++ b/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala @@ -53,6 +53,19 @@ private[spark] class WholeTextFileInputFormat val totalLen = files.map(file => if (file.isDirectory) 0L else file.getLen).sum val maxSplitSize = Math.ceil(totalLen * 1.0 / (if (minPartitions == 0) 1 else minPartitions)).toLong + +// For small files we need to ensure the min split size per node & rack <= maxSplitSize +val config = context.getConfiguration +val minSplitSizePerNode = config.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERNODE, 0L) +val minSplitSizePerRack = config.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERRACK, 0L) + +if (maxSplitSize < minSplitSizePerNode) { + super.setMinSplitSizeNode(maxSplitSize) +} + +if (maxSplitSize < minSplitSizePerRack) { + super.setMinSplitSizeRack(maxSplitSize) +} super.setMaxSplitSize(maxSplitSize) } } http://git-wip-us.apache.org/repos/asf/spark/blob/1055c94c/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala b/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala new file mode 100644 index 000..817dc08 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You unde
spark git commit: Update for spark 2.2.2 release
Repository: spark Updated Branches: refs/heads/branch-2.2 4bc4ccd63 -> 17db57213 Update for spark 2.2.2 release Release process for Spark 2.2.2 say to update the test. Author: Thomas Graves Closes #21707 from tgravescs/spark222-release. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/17db5721 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/17db5721 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/17db5721 Branch: refs/heads/branch-2.2 Commit: 17db57213aabc13cb59d13f0f570c7539dae Parents: 4bc4ccd Author: Thomas Graves Authored: Mon Jul 16 09:29:20 2018 -0500 Committer: Thomas Graves Committed: Mon Jul 16 09:29:20 2018 -0500 -- .../apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/17db5721/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala -- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 313059b..e6a6cac 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -170,7 +170,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { object PROCESS_TABLES extends QueryTest with SQLTestUtils { // Tests the latest version of every release line. - val testingVersions = Seq("2.0.2", "2.1.2", "2.2.1") + val testingVersions = Seq("2.0.2", "2.1.3", "2.2.2") protected var spark: SparkSession = _ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
Repository: spark Updated Branches: refs/heads/master fc0c8c971 -> c8bee932c [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap ## What changes were proposed in this pull request? When speculation is enabled, TaskSetManager#markPartitionCompleted should write successful task duration to MedianHeap, not just increase tasksSuccessful. Otherwise when TaskSetManager#checkSpeculatableTasks,tasksSuccessful non-zero, but MedianHeap is empty. Then throw an exception successfulTaskDurations.median java.util.NoSuchElementException: MedianHeap is empty. Finally led to stopping SparkContext. ## How was this patch tested? TaskSetManagerSuite.scala unit test:[SPARK-24677] MedianHeap should not be empty when speculation is enabled Author: sychen Closes #21656 from cxzl25/fix_MedianHeap_empty. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c8bee932 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c8bee932 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c8bee932 Branch: refs/heads/master Commit: c8bee932cb644627c4049b5a07dd8028968572d9 Parents: fc0c8c9 Author: sychen Authored: Wed Jul 18 13:24:41 2018 -0500 Committer: Thomas Graves Committed: Wed Jul 18 13:24:41 2018 -0500 -- .../spark/scheduler/TaskSchedulerImpl.scala | 7 ++- .../apache/spark/scheduler/TaskSetManager.scala | 7 ++- .../spark/scheduler/TaskSetManagerSuite.scala | 49 3 files changed, 59 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c8bee932/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 598b62f..56c0bf6 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -697,9 +697,12 @@ private[spark] class TaskSchedulerImpl( * do not also submit those same tasks. That also means that a task completion from an earlier * attempt can lead to the entire stage getting marked as successful. */ - private[scheduler] def markPartitionCompletedInAllTaskSets(stageId: Int, partitionId: Int) = { + private[scheduler] def markPartitionCompletedInAllTaskSets( + stageId: Int, + partitionId: Int, + taskInfo: TaskInfo) = { taskSetsByStageIdAndAttempt.getOrElse(stageId, Map()).values.foreach { tsm => - tsm.markPartitionCompleted(partitionId) + tsm.markPartitionCompleted(partitionId, taskInfo) } } http://git-wip-us.apache.org/repos/asf/spark/blob/c8bee932/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index a18c665..6071605 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -758,7 +758,7 @@ private[spark] class TaskSetManager( } // There may be multiple tasksets for this stage -- we let all of them know that the partition // was completed. This may result in some of the tasksets getting completed. -sched.markPartitionCompletedInAllTaskSets(stageId, tasks(index).partitionId) +sched.markPartitionCompletedInAllTaskSets(stageId, tasks(index).partitionId, info) // This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which holds the // "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue, we should not // "deserialize" the value when holding a lock to avoid blocking other threads. So we call @@ -769,9 +769,12 @@ private[spark] class TaskSetManager( maybeFinishTaskSet() } - private[scheduler] def markPartitionCompleted(partitionId: Int): Unit = { + private[scheduler] def markPartitionCompleted(partitionId: Int, taskInfo: TaskInfo): Unit = { partitionToIndex.get(partitionId).foreach { index => if (!successful(index)) { +if (speculationEnabled && !isZombie) { + successfulTaskDurations.insert(taskInfo.duration) +} tasksSuccessful += 1 successful(index) = true if (tasksSuccessful == numTasks) { http://git-wip-us.apache.org/repos/asf/spark/blob/c8bee932/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetMa
spark git commit: [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
Repository: spark Updated Branches: refs/heads/branch-2.3 e31b4766b -> 7be70e29d [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap ## What changes were proposed in this pull request? When speculation is enabled, TaskSetManager#markPartitionCompleted should write successful task duration to MedianHeap, not just increase tasksSuccessful. Otherwise when TaskSetManager#checkSpeculatableTasks,tasksSuccessful non-zero, but MedianHeap is empty. Then throw an exception successfulTaskDurations.median java.util.NoSuchElementException: MedianHeap is empty. Finally led to stopping SparkContext. ## How was this patch tested? TaskSetManagerSuite.scala unit test:[SPARK-24677] MedianHeap should not be empty when speculation is enabled Author: sychen Closes #21656 from cxzl25/fix_MedianHeap_empty. (cherry picked from commit c8bee932cb644627c4049b5a07dd8028968572d9) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7be70e29 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7be70e29 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7be70e29 Branch: refs/heads/branch-2.3 Commit: 7be70e29dd92de36dbb30ce39623d588f48e4cac Parents: e31b476 Author: sychen Authored: Wed Jul 18 13:24:41 2018 -0500 Committer: Thomas Graves Committed: Wed Jul 18 13:24:54 2018 -0500 -- .../spark/scheduler/TaskSchedulerImpl.scala | 7 ++- .../apache/spark/scheduler/TaskSetManager.scala | 7 ++- .../spark/scheduler/TaskSetManagerSuite.scala | 49 3 files changed, 59 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/7be70e29/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 598b62f..56c0bf6 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -697,9 +697,12 @@ private[spark] class TaskSchedulerImpl( * do not also submit those same tasks. That also means that a task completion from an earlier * attempt can lead to the entire stage getting marked as successful. */ - private[scheduler] def markPartitionCompletedInAllTaskSets(stageId: Int, partitionId: Int) = { + private[scheduler] def markPartitionCompletedInAllTaskSets( + stageId: Int, + partitionId: Int, + taskInfo: TaskInfo) = { taskSetsByStageIdAndAttempt.getOrElse(stageId, Map()).values.foreach { tsm => - tsm.markPartitionCompleted(partitionId) + tsm.markPartitionCompleted(partitionId, taskInfo) } } http://git-wip-us.apache.org/repos/asf/spark/blob/7be70e29/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index b52e376..c90ae50 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -759,7 +759,7 @@ private[spark] class TaskSetManager( } // There may be multiple tasksets for this stage -- we let all of them know that the partition // was completed. This may result in some of the tasksets getting completed. -sched.markPartitionCompletedInAllTaskSets(stageId, tasks(index).partitionId) +sched.markPartitionCompletedInAllTaskSets(stageId, tasks(index).partitionId, info) // This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which holds the // "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue, we should not // "deserialize" the value when holding a lock to avoid blocking other threads. So we call @@ -770,9 +770,12 @@ private[spark] class TaskSetManager( maybeFinishTaskSet() } - private[scheduler] def markPartitionCompleted(partitionId: Int): Unit = { + private[scheduler] def markPartitionCompleted(partitionId: Int, taskInfo: TaskInfo): Unit = { partitionToIndex.get(partitionId).foreach { index => if (!successful(index)) { +if (speculationEnabled && !isZombie) { + successfulTaskDurations.insert(taskInfo.duration) +} tasksSuccessful += 1 successful(index) = true if (tasksSuccessful == numTasks) { http://git-wip-us.apache.org/repos/asf/spark/blob/7be70e29/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala -
spark git commit: [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
Repository: spark Updated Branches: refs/heads/branch-2.2 17db57213 -> 144426cff [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap ## What changes were proposed in this pull request? When speculation is enabled, TaskSetManager#markPartitionCompleted should write successful task duration to MedianHeap, not just increase tasksSuccessful. Otherwise when TaskSetManager#checkSpeculatableTasks,tasksSuccessful non-zero, but MedianHeap is empty. Then throw an exception successfulTaskDurations.median java.util.NoSuchElementException: MedianHeap is empty. Finally led to stopping SparkContext. ## How was this patch tested? TaskSetManagerSuite.scala unit test:[SPARK-24677] MedianHeap should not be empty when speculation is enabled Author: sychen Closes #21656 from cxzl25/fix_MedianHeap_empty. (cherry picked from commit c8bee932cb644627c4049b5a07dd8028968572d9) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/144426cf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/144426cf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/144426cf Branch: refs/heads/branch-2.2 Commit: 144426cffd6e4b26b676004f5489e218140f7df2 Parents: 17db572 Author: sychen Authored: Wed Jul 18 13:24:41 2018 -0500 Committer: Thomas Graves Committed: Wed Jul 18 13:26:24 2018 -0500 -- .../spark/scheduler/TaskSchedulerImpl.scala | 7 ++- .../apache/spark/scheduler/TaskSetManager.scala | 7 ++- .../spark/scheduler/TaskSetManagerSuite.scala | 49 3 files changed, 59 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/144426cf/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index df6407b..f8c62b4 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -701,9 +701,12 @@ private[spark] class TaskSchedulerImpl private[scheduler]( * do not also submit those same tasks. That also means that a task completion from an earlier * attempt can lead to the entire stage getting marked as successful. */ - private[scheduler] def markPartitionCompletedInAllTaskSets(stageId: Int, partitionId: Int) = { + private[scheduler] def markPartitionCompletedInAllTaskSets( + stageId: Int, + partitionId: Int, + taskInfo: TaskInfo) = { taskSetsByStageIdAndAttempt.getOrElse(stageId, Map()).values.foreach { tsm => - tsm.markPartitionCompleted(partitionId) + tsm.markPartitionCompleted(partitionId, taskInfo) } } http://git-wip-us.apache.org/repos/asf/spark/blob/144426cf/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index d9515fb..705b896 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -748,7 +748,7 @@ private[spark] class TaskSetManager( } // There may be multiple tasksets for this stage -- we let all of them know that the partition // was completed. This may result in some of the tasksets getting completed. -sched.markPartitionCompletedInAllTaskSets(stageId, tasks(index).partitionId) +sched.markPartitionCompletedInAllTaskSets(stageId, tasks(index).partitionId, info) // This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which holds the // "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue, we should not // "deserialize" the value when holding a lock to avoid blocking other threads. So we call @@ -759,9 +759,12 @@ private[spark] class TaskSetManager( maybeFinishTaskSet() } - private[scheduler] def markPartitionCompleted(partitionId: Int): Unit = { + private[scheduler] def markPartitionCompleted(partitionId: Int, taskInfo: TaskInfo): Unit = { partitionToIndex.get(partitionId).foreach { index => if (!successful(index)) { +if (speculationEnabled && !isZombie) { + successfulTaskDurations.insert(taskInfo.duration) +} tasksSuccessful += 1 successful(index) = true if (tasksSuccessful == numTasks) { http://git-wip-us.apache.org/repos/asf/spark/blob/144426cf/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala -
spark git commit: [SPARK-22151] PYTHONPATH not picked up from the spark.yarn.appMaste…
Repository: spark Updated Branches: refs/heads/master c8bee932c -> 1272b2034 [SPARK-22151] PYTHONPATH not picked up from the spark.yarn.appMaste⦠â¦rEnv properly Running in yarn cluster mode and trying to set pythonpath via spark.yarn.appMasterEnv.PYTHONPATH doesn't work. the yarn Client code looks at the env variables: val pythonPathStr = (sys.env.get("PYTHONPATH") ++ pythonPath) But when you set spark.yarn.appMasterEnv it puts it into the local env. So the python path set in spark.yarn.appMasterEnv isn't properly set. You can work around if you are running in cluster mode by setting it on the client like: PYTHONPATH=./addon/python/ spark-submit ## What changes were proposed in this pull request? In Client.scala, PYTHONPATH was being overridden, so changed code to append values to PYTHONPATH instead of overriding them. ## How was this patch tested? Added log statements to ApplicationMaster.scala to check for environment variable PYTHONPATH, ran a spark job in cluster mode before the change and verified the issue. Performed the same test after the change and verified the fix. Author: pgandhi Closes #21468 from pgandhi999/SPARK-22151. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1272b203 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1272b203 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1272b203 Branch: refs/heads/master Commit: 1272b2034d4eed4bfe60a49e1065871b3a3f96e0 Parents: c8bee93 Author: pgandhi Authored: Wed Jul 18 14:07:03 2018 -0500 Committer: Thomas Graves Committed: Wed Jul 18 14:07:03 2018 -0500 -- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1272b203/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala -- diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 793d012..ed9879c 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -811,10 +811,12 @@ private[spark] class Client( // Finally, update the Spark config to propagate PYTHONPATH to the AM and executors. if (pythonPath.nonEmpty) { - val pythonPathStr = (sys.env.get("PYTHONPATH") ++ pythonPath) + val pythonPathList = (sys.env.get("PYTHONPATH") ++ pythonPath) + env("PYTHONPATH") = (env.get("PYTHONPATH") ++ pythonPathList) .mkString(ApplicationConstants.CLASS_PATH_SEPARATOR) - env("PYTHONPATH") = pythonPathStr - sparkConf.setExecutorEnv("PYTHONPATH", pythonPathStr) + val pythonPathExecutorEnv = (sparkConf.getExecutorEnv.toMap.get("PYTHONPATH") ++ +pythonPathList).mkString(ApplicationConstants.CLASS_PATH_SEPARATOR) + sparkConf.setExecutorEnv("PYTHONPATH", pythonPathExecutorEnv) } if (isClusterMode) { - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24755][CORE] Executor loss can cause task to not be resubmitted
Repository: spark Updated Branches: refs/heads/master 6a9a058e0 -> 8d707b060 [SPARK-24755][CORE] Executor loss can cause task to not be resubmitted **Description** As described in [SPARK-24755](https://issues.apache.org/jira/browse/SPARK-24755), when speculation is enabled, there is scenario that executor loss can cause task to not be resubmitted. This patch changes the variable killedByOtherAttempt to keeps track of the taskId of tasks that are killed by other attempt. By doing this, we can still prevent resubmitting task killed by other attempt while resubmit successful attempt when executor lost. **How was this patch tested?** A UT is added based on the UT written by xuanyuanking with modification to simulate the scenario described in SPARK-24755. Author: Hieu Huynh <âhieu.hu...@oath.comâ> Closes #21729 from hthuynh2/SPARK_24755. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8d707b06 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8d707b06 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8d707b06 Branch: refs/heads/master Commit: 8d707b06003bc97d06630b22e6ae7c35f99b3cdd Parents: 6a9a058 Author: Hieu Huynh <âhieu.hu...@oath.comâ> Authored: Thu Jul 19 09:52:07 2018 -0500 Committer: Thomas Graves Committed: Thu Jul 19 09:52:07 2018 -0500 -- .../apache/spark/scheduler/TaskSetManager.scala | 10 +- .../spark/scheduler/TaskSetManagerSuite.scala | 112 +++ 2 files changed, 117 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/8d707b06/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 6071605..defed1e 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -84,10 +84,10 @@ private[spark] class TaskSetManager( val successful = new Array[Boolean](numTasks) private val numFailures = new Array[Int](numTasks) - // Set the coresponding index of Boolean var when the task killed by other attempt tasks, - // this happened while we set the `spark.speculation` to true. The task killed by others + // Add the tid of task into this HashSet when the task is killed by other attempt tasks. + // This happened while we set the `spark.speculation` to true. The task killed by others // should not resubmit while executor lost. - private val killedByOtherAttempt: Array[Boolean] = new Array[Boolean](numTasks) + private val killedByOtherAttempt = new HashSet[Long] val taskAttempts = Array.fill[List[TaskInfo]](numTasks)(Nil) private[scheduler] var tasksSuccessful = 0 @@ -735,7 +735,7 @@ private[spark] class TaskSetManager( logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for task ${attemptInfo.id} " + s"in stage ${taskSet.id} (TID ${attemptInfo.taskId}) on ${attemptInfo.host} " + s"as the attempt ${info.attemptNumber} succeeded on ${info.host}") - killedByOtherAttempt(index) = true + killedByOtherAttempt += attemptInfo.taskId sched.backend.killTask( attemptInfo.taskId, attemptInfo.executorId, @@ -947,7 +947,7 @@ private[spark] class TaskSetManager( && !isZombie) { for ((tid, info) <- taskInfos if info.executorId == execId) { val index = taskInfos(tid).index -if (successful(index) && !killedByOtherAttempt(index)) { +if (successful(index) && !killedByOtherAttempt.contains(tid)) { successful(index) = false copiesRunning(index) -= 1 tasksSuccessful -= 1 http://git-wip-us.apache.org/repos/asf/spark/blob/8d707b06/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index ae571e5..206b9f4 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -1414,6 +1414,118 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg taskSetManager2.checkSpeculatableTasks(0) } + + test("SPARK-24755 Executor loss can cause task to not be resubmitted") { +val conf = new SparkConf().set("spark.speculation", "true") +sc = new SparkContext("local", "test", conf) +// Set the speculation multiplier to be 0 so speculative tasks are launched imm
spark git commit: [SPARK-24755][CORE] Executor loss can cause task to not be resubmitted
Repository: spark Updated Branches: refs/heads/branch-2.3 7be70e29d -> d0280ab81 [SPARK-24755][CORE] Executor loss can cause task to not be resubmitted **Description** As described in [SPARK-24755](https://issues.apache.org/jira/browse/SPARK-24755), when speculation is enabled, there is scenario that executor loss can cause task to not be resubmitted. This patch changes the variable killedByOtherAttempt to keeps track of the taskId of tasks that are killed by other attempt. By doing this, we can still prevent resubmitting task killed by other attempt while resubmit successful attempt when executor lost. **How was this patch tested?** A UT is added based on the UT written by xuanyuanking with modification to simulate the scenario described in SPARK-24755. Author: Hieu Huynh <âhieu.hu...@oath.comâ> Closes #21729 from hthuynh2/SPARK_24755. (cherry picked from commit 8d707b06003bc97d06630b22e6ae7c35f99b3cdd) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d0280ab8 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d0280ab8 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d0280ab8 Branch: refs/heads/branch-2.3 Commit: d0280ab818391fd11662647459f1e9e683b2bc8e Parents: 7be70e2 Author: Hieu Huynh <âhieu.hu...@oath.comâ> Authored: Thu Jul 19 09:52:07 2018 -0500 Committer: Thomas Graves Committed: Thu Jul 19 09:52:30 2018 -0500 -- .../apache/spark/scheduler/TaskSetManager.scala | 10 +- .../spark/scheduler/TaskSetManagerSuite.scala | 112 +++ 2 files changed, 117 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d0280ab8/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index c90ae50..df8d914 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -85,10 +85,10 @@ private[spark] class TaskSetManager( val successful = new Array[Boolean](numTasks) private val numFailures = new Array[Int](numTasks) - // Set the coresponding index of Boolean var when the task killed by other attempt tasks, - // this happened while we set the `spark.speculation` to true. The task killed by others + // Add the tid of task into this HashSet when the task is killed by other attempt tasks. + // This happened while we set the `spark.speculation` to true. The task killed by others // should not resubmit while executor lost. - private val killedByOtherAttempt: Array[Boolean] = new Array[Boolean](numTasks) + private val killedByOtherAttempt = new HashSet[Long] val taskAttempts = Array.fill[List[TaskInfo]](numTasks)(Nil) private[scheduler] var tasksSuccessful = 0 @@ -736,7 +736,7 @@ private[spark] class TaskSetManager( logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for task ${attemptInfo.id} " + s"in stage ${taskSet.id} (TID ${attemptInfo.taskId}) on ${attemptInfo.host} " + s"as the attempt ${info.attemptNumber} succeeded on ${info.host}") - killedByOtherAttempt(index) = true + killedByOtherAttempt += attemptInfo.taskId sched.backend.killTask( attemptInfo.taskId, attemptInfo.executorId, @@ -942,7 +942,7 @@ private[spark] class TaskSetManager( && !isZombie) { for ((tid, info) <- taskInfos if info.executorId == execId) { val index = taskInfos(tid).index -if (successful(index) && !killedByOtherAttempt(index)) { +if (successful(index) && !killedByOtherAttempt.contains(tid)) { successful(index) = false copiesRunning(index) -= 1 tasksSuccessful -= 1 http://git-wip-us.apache.org/repos/asf/spark/blob/d0280ab8/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index 0f4d056..b4acccf 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -1411,6 +1411,118 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg taskSetManager2.checkSpeculatableTasks(0) } + + test("SPARK-24755 Executor loss can cause task to not be resubmitted") { +val conf = new SparkConf().set("spark.speculation", "true") +sc = new SparkContext(
spark git commit: [SPARK-13343] speculative tasks that didn't commit shouldn't be marked as success
Repository: spark Updated Branches: refs/heads/master ee5a5a092 -> 5828f41a5 [SPARK-13343] speculative tasks that didn't commit shouldn't be marked as success **Description** Currently Speculative tasks that didn't commit can show up as success (depending on timing of commit). This is a bit confusing because that task didn't really succeed in the sense it didn't write anything. I think these tasks should be marked as KILLED or something that is more obvious to the user exactly what happened. it is happened to hit the timing where it got a commit denied exception then it shows up as failed and counts against your task failures. It shouldn't count against task failures since that failure really doesn't matter. MapReduce handles these situation so perhaps we can look there for a model. https://user-images.githubusercontent.com/15680678/42013170-99db48c2-7a61-11e8-8c7b-ef94c84e36ea.png";> **How can this issue happen?** When both attempts of a task finish before the driver sends command to kill one of them, both of them send the status update FINISHED to the driver. The driver calls TaskSchedulerImpl to handle one successful task at a time. When it handles the first successful task, it sends the command to kill the other copy of the task, however, because that task is already finished, the executor will ignore the command. After finishing handling the first attempt, it processes the second one, although all actions on the result of this task are skipped, this copy of the task is still marked as SUCCESS. As a result, even though this issue does not affect the result of the job, it might cause confusing to user because both of them appear to be successful. **How does this PR fix the issue?** The simple way to fix this issue is that when taskSetManager handles successful task, it checks if any other attempt succeeded. If this is the case, it will call handleFailedTask with state==KILLED and reason==TaskKilled(âanother attempt succeededâ) to handle this task as begin killed. **How was this patch tested?** I tested this manually by running applications, that caused the issue before, a few times, and observed that the issue does not happen again. Also, I added a unit test in TaskSetManagerSuite to test that if we call handleSuccessfulTask to handle status update for 2 copies of a task, only the one that is handled first will be mark as SUCCESS Author: Hieu Huynh <âhieu.hu...@oath.comâ> Author: hthuynh2 Closes #21653 from hthuynh2/SPARK_13343. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5828f41a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5828f41a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5828f41a Branch: refs/heads/master Commit: 5828f41a52c446b774a909e96eff8d8c5831b394 Parents: ee5a5a0 Author: Hieu Huynh <âhieu.hu...@oath.comâ> Authored: Fri Jul 27 12:34:14 2018 -0500 Committer: Thomas Graves Committed: Fri Jul 27 12:34:14 2018 -0500 -- .../apache/spark/scheduler/TaskSetManager.scala | 19 +- .../spark/scheduler/TaskSetManagerSuite.scala | 70 2 files changed, 88 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/5828f41a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 0b21256..8b77641 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -29,7 +29,7 @@ import org.apache.spark._ import org.apache.spark.TaskState.TaskState import org.apache.spark.internal.{config, Logging} import org.apache.spark.scheduler.SchedulingMode._ -import org.apache.spark.util.{AccumulatorV2, Clock, SystemClock, Utils} +import org.apache.spark.util.{AccumulatorV2, Clock, LongAccumulator, SystemClock, Utils} import org.apache.spark.util.collection.MedianHeap /** @@ -728,6 +728,23 @@ private[spark] class TaskSetManager( def handleSuccessfulTask(tid: Long, result: DirectTaskResult[_]): Unit = { val info = taskInfos(tid) val index = info.index +// Check if any other attempt succeeded before this and this attempt has not been handled +if (successful(index) && killedByOtherAttempt.contains(tid)) { + // Undo the effect on calculatedTasks and totalResultSize made earlier when + // checking if can fetch more results + calculatedTasks -= 1 + val resultSizeAcc = result.accumUpdates.find(a => +a.name == Some(InternalAccumulator.RESULT_SIZE)) + if (resultSizeAcc.isDefined) { +totalResultSize -
spark git commit: [SPARK-24981][CORE] ShutdownHook timeout causes job to fail when succeeded when SparkContext stop() not called by user program
Repository: spark Updated Branches: refs/heads/master c1760da5d -> 35700bb7f [SPARK-24981][CORE] ShutdownHook timeout causes job to fail when succeeded when SparkContext stop() not called by user program **Description** The issue is described in [SPARK-24981](https://issues.apache.org/jira/browse/SPARK-24981). **How does this PR fix the issue?** This PR catch the Exception that is thrown while the Sparkcontext.stop() is running (when it is called by the ShutdownHookManager). **How was this patch tested?** I manually tested it by adding delay (60s) inside the stop(). This make the shutdownHookManger interrupt the thread that is running stop(). The Interrupted Exception was catched and the job succeed. Author: Hieu Huynh <âhieu.hu...@oath.comâ> Author: Hieu Tri Huynh Closes #21936 from hthuynh2/SPARK_24981. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/35700bb7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/35700bb7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/35700bb7 Branch: refs/heads/master Commit: 35700bb7f2e3008ff781a1b3a1da8147d26371be Parents: c1760da Author: Hieu Huynh <âhieu.hu...@oath.comâ> Authored: Mon Aug 6 09:01:51 2018 -0500 Committer: Thomas Graves Committed: Mon Aug 6 09:01:51 2018 -0500 -- core/src/main/scala/org/apache/spark/SparkContext.scala | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/35700bb7/core/src/main/scala/org/apache/spark/SparkContext.scala -- diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 03e91cd..e8bacee 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -571,7 +571,12 @@ class SparkContext(config: SparkConf) extends Logging { _shutdownHookRef = ShutdownHookManager.addShutdownHook( ShutdownHookManager.SPARK_CONTEXT_SHUTDOWN_PRIORITY) { () => logInfo("Invoking stop() from shutdown hook") - stop() + try { +stop() + } catch { +case e: Throwable => + logWarning("Ignoring Exception while stopping SparkContext from shutdown hook", e) + } } } catch { case NonFatal(e) => - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24992][CORE] spark should randomize yarn local dir selection
Repository: spark Updated Branches: refs/heads/master 1a5e46076 -> 51e2b38d9 [SPARK-24992][CORE] spark should randomize yarn local dir selection **Description: [SPARK-24992](https://issues.apache.org/jira/browse/SPARK-24992)** Utils.getLocalDir is used to get path of a temporary directory. However, it always returns the the same directory, which is the first element in the array localRootDirs. When running on YARN, this might causes the case that we always write to one disk, which makes it busy while other disks are free. We should randomize the selection to spread out the loads. **What changes were proposed in this pull request?** This PR randomized the selection of local directory inside the method Utils.getLocalDir. This change affects the Utils.fetchFile method since it based on the fact that Utils.getLocalDir always return the same directory to cache file. Therefore, a new variable cachedLocalDir is used to cache the first localDirectory that it gets from Utils.getLocalDir. Also, when getting the configured local directories (inside Utils. getConfiguredLocalDirs), in case we are in yarn mode, the array of directories are also randomized before return. Author: Hieu Huynh <âhieu.hu...@oath.comâ> Closes #21953 from hthuynh2/SPARK_24992. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/51e2b38d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/51e2b38d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/51e2b38d Branch: refs/heads/master Commit: 51e2b38d93df8cb0cc151d5e68a2190eab52644c Parents: 1a5e460 Author: Hieu Huynh <âhieu.hu...@oath.comâ> Authored: Mon Aug 6 13:58:28 2018 -0500 Committer: Thomas Graves Committed: Mon Aug 6 13:58:28 2018 -0500 -- .../scala/org/apache/spark/util/Utils.scala | 21 1 file changed, 17 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/51e2b38d/core/src/main/scala/org/apache/spark/util/Utils.scala -- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index a6fd363..7ec707d 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -83,6 +83,7 @@ private[spark] object Utils extends Logging { val random = new Random() private val sparkUncaughtExceptionHandler = new SparkUncaughtExceptionHandler + @volatile private var cachedLocalDir: String = "" /** * Define a default value for driver memory here since this value is referenced across the code @@ -462,7 +463,15 @@ private[spark] object Utils extends Logging { if (useCache && fetchCacheEnabled) { val cachedFileName = s"${url.hashCode}${timestamp}_cache" val lockFileName = s"${url.hashCode}${timestamp}_lock" - val localDir = new File(getLocalDir(conf)) + // Set the cachedLocalDir for the first time and re-use it later + if (cachedLocalDir.isEmpty) { +this.synchronized { + if (cachedLocalDir.isEmpty) { +cachedLocalDir = getLocalDir(conf) + } +} + } + val localDir = new File(cachedLocalDir) val lockFile = new File(localDir, lockFileName) val lockFileChannel = new RandomAccessFile(lockFile, "rw").getChannel() // Only one executor entry. @@ -767,13 +776,17 @@ private[spark] object Utils extends Logging { * - Otherwise, this will return java.io.tmpdir. * * Some of these configuration options might be lists of multiple paths, but this method will - * always return a single directory. + * always return a single directory. The return directory is chosen randomly from the array + * of directories it gets from getOrCreateLocalRootDirs. */ def getLocalDir(conf: SparkConf): String = { -getOrCreateLocalRootDirs(conf).headOption.getOrElse { +val localRootDirs = getOrCreateLocalRootDirs(conf) +if (localRootDirs.isEmpty) { val configuredLocalDirs = getConfiguredLocalDirs(conf) throw new IOException( s"Failed to get a temp directory under [${configuredLocalDirs.mkString(",")}].") +} else { + localRootDirs(scala.util.Random.nextInt(localRootDirs.length)) } } @@ -815,7 +828,7 @@ private[spark] object Utils extends Logging { // to what Yarn on this system said was available. Note this assumes that Yarn has // created the directories already, and that they are secured so that only the // user has access to them. - getYarnLocalDirs(conf).split(",") + randomizeInPlace(getYarnLocalDirs(conf).split(",")) } else if (conf.getenv("SPARK_EXECUTOR_DIRS") != null) { conf.getenv("SPARK_EXECUTOR
spark git commit: [SPARK-15703][SCHEDULER][CORE][WEBUI] Make ListenerBus event queue size configurable
Repository: spark Updated Branches: refs/heads/master 0869b3a5f -> 0b71d9ae0 [SPARK-15703][SCHEDULER][CORE][WEBUI] Make ListenerBus event queue size configurable ## What changes were proposed in this pull request? This change adds a new configuration entry to specify the size of the spark listener bus event queue. The value for this config ("spark.scheduler.listenerbus.eventqueue.size") is set to a default to 1. Note: I haven't currently documented the configuration entry. We can decide whether it would be appropriate to make it a public configuration or keep it as an undocumented one. Refer JIRA for more details. ## How was this patch tested? Ran existing jobs and verified the event queue size with debug logs and from the Spark WebUI Environment tab. Author: Dhruve Ashar Closes #14269 from dhruve/bug/SPARK-15703. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0b71d9ae Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0b71d9ae Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0b71d9ae Branch: refs/heads/master Commit: 0b71d9ae0804b0394e4abd02c7cebf52a9102216 Parents: 0869b3a Author: Dhruve Ashar Authored: Tue Jul 26 13:23:33 2016 -0500 Committer: Tom Graves Committed: Tue Jul 26 13:23:33 2016 -0500 -- .../scala/org/apache/spark/SparkContext.scala | 4 +-- .../apache/spark/internal/config/package.scala | 5 .../spark/scheduler/LiveListenerBus.scala | 23 +-- .../scheduler/EventLoggingListenerSuite.scala | 4 +-- .../spark/scheduler/SparkListenerSuite.scala| 30 +++- .../storage/BlockManagerReplicationSuite.scala | 9 -- .../spark/storage/BlockManagerSuite.scala | 6 ++-- .../spark/ui/storage/StorageTabSuite.scala | 11 +++ .../streaming/ReceivedBlockHandlerSuite.scala | 5 +++- 9 files changed, 60 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/0b71d9ae/core/src/main/scala/org/apache/spark/SparkContext.scala -- diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 6d7f05d..d48e2b4 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -249,7 +249,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli def isStopped: Boolean = stopped.get() // An asynchronous listener bus for Spark events - private[spark] val listenerBus = new LiveListenerBus + private[spark] val listenerBus = new LiveListenerBus(this) // This function allows components created by SparkEnv to be mocked in unit tests: private[spark] def createSparkEnv( @@ -2148,7 +2148,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli } } -listenerBus.start(this) +listenerBus.start() _listenerBusStarted = true } http://git-wip-us.apache.org/repos/asf/spark/blob/0b71d9ae/core/src/main/scala/org/apache/spark/internal/config/package.scala -- diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 05dd683..ebb21e9 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -103,4 +103,9 @@ package object config { .stringConf .checkValues(Set("hive", "in-memory")) .createWithDefault("in-memory") + + private[spark] val LISTENER_BUS_EVENT_QUEUE_SIZE = +ConfigBuilder("spark.scheduler.listenerbus.eventqueue.size") + .intConf + .createWithDefault(1) } http://git-wip-us.apache.org/repos/asf/spark/blob/0b71d9ae/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala index 1c21313..bfa3c40 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala @@ -22,7 +22,8 @@ import java.util.concurrent.atomic.AtomicBoolean import scala.util.DynamicVariable -import org.apache.spark.SparkContext +import org.apache.spark.{SparkContext, SparkException} +import org.apache.spark.internal.config._ import org.apache.spark.util.Utils /** @@ -32,18 +33,24 @@ import org.apache.spark.util.Utils * has started will events be actually propagated to all attached lis
spark git commit: [SPARK-16673][WEB UI] New Executor Page removed conditional for Logs and Thread Dump columns
Repository: spark Updated Branches: refs/heads/master 67e59d464 -> e98eb2146 [SPARK-16673][WEB UI] New Executor Page removed conditional for Logs and Thread Dump columns ## What changes were proposed in this pull request? When #13670 switched `ExecutorsPage` to use JQuery DataTables it incidentally removed the conditional for the Logs and Thread Dump columns. I reimplemented the conditional display of the Logs and Thread dump columns as it was before the switch. ## How was this patch tested? Manually tested and dev/run-tests ![both](https://cloud.githubusercontent.com/assets/13952758/17186879/da8dd1a8-53eb-11e6-8b0c-d0ff0156a9a7.png) ![dump](https://cloud.githubusercontent.com/assets/13952758/17186881/dab08a04-53eb-11e6-8b1c-50ffd0bf2ae8.png) ![logs](https://cloud.githubusercontent.com/assets/13952758/17186880/dab04d00-53eb-11e6-8754-68dd64d6d9f4.png) Author: Alex Bozarth Closes #14382 from ajbozarth/spark16673. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e98eb214 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e98eb214 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e98eb214 Branch: refs/heads/master Commit: e98eb2146f1363956bfc3e5adcc11c246182d617 Parents: 67e59d4 Author: Alex Bozarth Authored: Fri Aug 19 10:04:20 2016 -0500 Committer: Tom Graves Committed: Fri Aug 19 10:04:20 2016 -0500 -- .../org/apache/spark/ui/static/executorspage.js | 38 .../apache/spark/ui/exec/ExecutorsPage.scala| 7 ++-- 2 files changed, 34 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e98eb214/core/src/main/resources/org/apache/spark/ui/static/executorspage.js -- diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index b2b2363..1df6733 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -15,6 +15,16 @@ * limitations under the License. */ +var threadDumpEnabled = false; + +function setThreadDumpEnabled(val) { +threadDumpEnabled = val; +} + +function getThreadDumpEnabled() { +return threadDumpEnabled; +} + function formatStatus(status, type) { if (type !== 'display') return status; if (status) { @@ -116,6 +126,12 @@ function formatLogsCells(execLogs, type) { return result; } +function logsExist(execs) { +return execs.some(function(exec) { +return !($.isEmptyObject(exec["executorLogs"])); +}); +} + // Determine Color Opacity from 0.5-1 // activeTasks range from 0 to maxTasks function activeTasksAlpha(activeTasks, maxTasks) { @@ -143,18 +159,16 @@ function totalDurationAlpha(totalGCTime, totalDuration) { (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1; } +// When GCTimePercent is edited change ToolTips.TASK_TIME to match +var GCTimePercent = 0.1; + function totalDurationStyle(totalGCTime, totalDuration) { // Red if GC time over GCTimePercent of total time -// When GCTimePercent is edited change ToolTips.TASK_TIME to match -var GCTimePercent = 0.1; return (totalGCTime > GCTimePercent * totalDuration) ? ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : ""; } function totalDurationColor(totalGCTime, totalDuration) { -// Red if GC time over GCTimePercent of total time -// When GCTimePercent is edited change ToolTips.TASK_TIME to match -var GCTimePercent = 0.1; return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black"; } @@ -392,8 +406,18 @@ $(document).ready(function () { {data: 'executorLogs', render: formatLogsCells}, { data: 'id', render: function (data, type) { -return type === 'display' ? ("Thread Dump" ) : data; +return type === 'display' ? ("Thread Dump" ) : data; +} } +], +"columnDefs": [ +{ +"targets": [ 15 ], +"visible": logsExist(response) +}, +{ +"targets": [ 16 ], +"visible": getThreadDumpEnabled() } ], "order": [[0, "asc"]] @@ -458,7 +482,7 @@ $(document).ready(function () { "paging": false, "searching": false, "info": false - +
spark git commit: [SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is enabled.
Repository: spark Updated Branches: refs/heads/master e98eb2146 -> 071eaaf9d [SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is enabled. ## What changes were proposed in this pull request? If the following conditions are satisfied, executors don't load properties in `hdfs-site.xml` and UnknownHostException can be thrown. (1) NameNode HA is enabled (2) spark.eventLogging is disabled or logging path is NOT on HDFS (3) Using Standalone or Mesos for the cluster manager (4) There are no code to load `HdfsCondition` class in the driver regardless of directly or indirectly. (5) The tasks access to HDFS (There might be some more conditions...) For example, following code causes UnknownHostException when the conditions above are satisfied. ``` sc.textFile("").collect ``` ``` java.lang.IllegalArgumentException: java.net.UnknownHostException: hacluster at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:378) at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:310) at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176) at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:678) at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:619) at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:149) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:170) at org.apache.hadoop.mapred.JobConf.getWorkingDirectory(JobConf.java:656) at org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:438) at org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:411) at org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986) at org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986) at org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177) at org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177) at scala.Option.map(Option.scala:146) at org.apache.spark.rdd.HadoopRDD.getJobConf(HadoopRDD.scala:177) at org.apache.spark.rdd.HadoopRDD$$anon$1.(HadoopRDD.scala:213) at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:209) at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:102) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318) at org.apache.spark.rdd.RDD.iterator(RDD.scala:282) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318) at org.apache.spark.rdd.RDD.iterator(RDD.scala:282) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70) at org.apache.spark.scheduler.Task.run(Task.scala:85) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.UnknownHostException: hacluster ``` But following code doesn't cause the Exception because `textFile` method loads `HdfsConfiguration` indirectly. ``` sc.textFile("").collect ``` When a job includes some operations which access to HDFS, the object of `org.apache.hadoop.Configuration` is wrapped by `SerializableConfiguration`, serialized and broadcasted from driver to executors and each executor deserialize the object with `loadDefaults` false so HDFS related properties should be set before broadcasted. ## How was this patch tested? Tested manually on my standalone cluster. Author: Kousuke Saruta Closes #13738 from sarutak/SPARK-11227. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/071eaaf9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/071eaaf9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/071eaaf9 Branch: refs/heads/master Commit: 071eaaf9d2b63589f2e66e5279a16a5a484de6f5 Parents: e98eb21 Author: Kousuke Saruta Authored: Fri Aug 19 10:11:25 2016 -0500 Committer: Tom Graves Committed: Fri Aug 19 10:11:25 2016 -0500 -- .../scala/org/apache/spark/SparkContext.scala | 22 +++- 1 file changed, 21 insertions(+),
spark git commit: [SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is enabled.
Repository: spark Updated Branches: refs/heads/branch-2.0 e0c60f185 -> d0707c6ba [SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is enabled. ## What changes were proposed in this pull request? If the following conditions are satisfied, executors don't load properties in `hdfs-site.xml` and UnknownHostException can be thrown. (1) NameNode HA is enabled (2) spark.eventLogging is disabled or logging path is NOT on HDFS (3) Using Standalone or Mesos for the cluster manager (4) There are no code to load `HdfsCondition` class in the driver regardless of directly or indirectly. (5) The tasks access to HDFS (There might be some more conditions...) For example, following code causes UnknownHostException when the conditions above are satisfied. ``` sc.textFile("").collect ``` ``` java.lang.IllegalArgumentException: java.net.UnknownHostException: hacluster at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:378) at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:310) at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176) at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:678) at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:619) at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:149) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:170) at org.apache.hadoop.mapred.JobConf.getWorkingDirectory(JobConf.java:656) at org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:438) at org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:411) at org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986) at org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986) at org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177) at org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177) at scala.Option.map(Option.scala:146) at org.apache.spark.rdd.HadoopRDD.getJobConf(HadoopRDD.scala:177) at org.apache.spark.rdd.HadoopRDD$$anon$1.(HadoopRDD.scala:213) at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:209) at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:102) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318) at org.apache.spark.rdd.RDD.iterator(RDD.scala:282) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318) at org.apache.spark.rdd.RDD.iterator(RDD.scala:282) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70) at org.apache.spark.scheduler.Task.run(Task.scala:85) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.UnknownHostException: hacluster ``` But following code doesn't cause the Exception because `textFile` method loads `HdfsConfiguration` indirectly. ``` sc.textFile("").collect ``` When a job includes some operations which access to HDFS, the object of `org.apache.hadoop.Configuration` is wrapped by `SerializableConfiguration`, serialized and broadcasted from driver to executors and each executor deserialize the object with `loadDefaults` false so HDFS related properties should be set before broadcasted. ## How was this patch tested? Tested manually on my standalone cluster. Author: Kousuke Saruta Closes #13738 from sarutak/SPARK-11227. (cherry picked from commit 071eaaf9d2b63589f2e66e5279a16a5a484de6f5) Signed-off-by: Tom Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d0707c6b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d0707c6b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d0707c6b Branch: refs/heads/branch-2.0 Commit: d0707c6baeb4003735a508f98db370984354 Parents: e0c60f1 Author: Kousuke Saruta Authored: Fri Aug 19 10:11:25 2016 -0500 Committer: Tom Graves Committed: Fri Aug 19 10:11:41 2016 -0500 -- ...
spark git commit: [SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData
Repository: spark Updated Branches: refs/heads/master 40b30fcf4 -> 891ac2b91 [SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData ## What changes were proposed in this pull request? Based on #12990 by tankkyo Since the History Server currently loads all application's data it can OOM if too many applications have a significant task count. `spark.ui.trimTasks` (default: false) can be set to true to trim tasks by `spark.ui.retainedTasks` (default: 1) (This is a "quick fix" to help those running into the problem until a update of how the history server loads app data can be done) ## How was this patch tested? Manual testing and dev/run-tests ![spark-15083](https://cloud.githubusercontent.com/assets/13952758/17713694/fe82d246-63b0-11e6-9697-b87ea75ff4ef.png) Author: Alex Bozarth Closes #14673 from ajbozarth/spark15083. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/891ac2b9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/891ac2b9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/891ac2b9 Branch: refs/heads/master Commit: 891ac2b914fb6f90a62c6fbc0a3960a89d1c1d92 Parents: 40b30fc Author: Alex Bozarth Authored: Wed Aug 24 14:39:41 2016 -0500 Committer: Tom Graves Committed: Wed Aug 24 14:39:41 2016 -0500 -- .../apache/spark/internal/config/package.scala | 5 + .../spark/ui/jobs/JobProgressListener.scala | 9 +- .../org/apache/spark/ui/jobs/StagePage.scala| 12 +- .../scala/org/apache/spark/ui/jobs/UIData.scala | 4 +- .../stage_task_list_w__sortBy_expectation.json | 130 ++--- ...ortBy_short_names___runtime_expectation.json | 130 ++--- ...sortBy_short_names__runtime_expectation.json | 182 +-- .../status/api/v1/AllStagesResourceSuite.scala | 4 +- docs/configuration.md | 8 + 9 files changed, 256 insertions(+), 228 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/891ac2b9/core/src/main/scala/org/apache/spark/internal/config/package.scala -- diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index be3dac4..47174e4 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -114,4 +114,9 @@ package object config { private[spark] val PYSPARK_PYTHON = ConfigBuilder("spark.pyspark.python") .stringConf .createOptional + + // To limit memory usage, we only track information for a fixed number of tasks + private[spark] val UI_RETAINED_TASKS = ConfigBuilder("spark.ui.retainedTasks") +.intConf +.createWithDefault(10) } http://git-wip-us.apache.org/repos/asf/spark/blob/891ac2b9/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index 491f716..d3a4f9d 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -19,12 +19,13 @@ package org.apache.spark.ui.jobs import java.util.concurrent.TimeoutException -import scala.collection.mutable.{HashMap, HashSet, ListBuffer} +import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap, ListBuffer} import org.apache.spark._ import org.apache.spark.annotation.DeveloperApi import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.Logging +import org.apache.spark.internal.config._ import org.apache.spark.scheduler._ import org.apache.spark.scheduler.SchedulingMode.SchedulingMode import org.apache.spark.storage.BlockManagerId @@ -93,6 +94,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { val retainedStages = conf.getInt("spark.ui.retainedStages", SparkUI.DEFAULT_RETAINED_STAGES) val retainedJobs = conf.getInt("spark.ui.retainedJobs", SparkUI.DEFAULT_RETAINED_JOBS) + val retainedTasks = conf.get(UI_RETAINED_TASKS) // We can test for memory leaks by ensuring that collections that track non-active jobs and // stages do not grow without bound and that collections for active jobs/stages eventually become @@ -405,6 +407,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { taskData.updateTaskMetrics(taskMetrics) taskData.errorMessage = errorMessage + // If Tasks is too large, remove and garbage collect old tasks + if (stageData.taskData.size >
spark git commit: [SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData
Repository: spark Updated Branches: refs/heads/branch-2.0 356a359de -> 55db26245 [SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData ## What changes were proposed in this pull request? This is a back port of #14673 addressing merge conflicts in package.scala that prevented a cherry-pick to `branch-2.0` when it was merged to `master` Since the History Server currently loads all application's data it can OOM if too many applications have a significant task count. This trims tasks by `spark.ui.retainedTasks` (default: 10) ## How was this patch tested? Manual testing and dev/run-tests Author: Alex Bozarth Closes #14794 from ajbozarth/spark15083-branch-2.0. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/55db2624 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/55db2624 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/55db2624 Branch: refs/heads/branch-2.0 Commit: 55db26245d69bb02b7d7d5f25029b1a1cd571644 Parents: 356a359 Author: Alex Bozarth Authored: Thu Aug 25 11:54:55 2016 -0500 Committer: Tom Graves Committed: Thu Aug 25 11:54:55 2016 -0500 -- .../apache/spark/internal/config/package.scala | 5 + .../spark/ui/jobs/JobProgressListener.scala | 9 +- .../org/apache/spark/ui/jobs/StagePage.scala| 12 +- .../scala/org/apache/spark/ui/jobs/UIData.scala | 4 +- .../stage_task_list_w__sortBy_expectation.json | 130 ++--- ...ortBy_short_names___runtime_expectation.json | 130 ++--- ...sortBy_short_names__runtime_expectation.json | 182 +-- .../status/api/v1/AllStagesResourceSuite.scala | 4 +- docs/configuration.md | 8 + 9 files changed, 256 insertions(+), 228 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/55db2624/core/src/main/scala/org/apache/spark/internal/config/package.scala -- diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 05dd683..5a59aee 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -103,4 +103,9 @@ package object config { .stringConf .checkValues(Set("hive", "in-memory")) .createWithDefault("in-memory") + + // To limit memory usage, we only track information for a fixed number of tasks + private[spark] val UI_RETAINED_TASKS = ConfigBuilder("spark.ui.retainedTasks") +.intConf +.createWithDefault(10) } http://git-wip-us.apache.org/repos/asf/spark/blob/55db2624/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index 842f42b..38ad6e9 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -19,12 +19,13 @@ package org.apache.spark.ui.jobs import java.util.concurrent.TimeoutException -import scala.collection.mutable.{HashMap, HashSet, ListBuffer} +import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap, ListBuffer} import org.apache.spark._ import org.apache.spark.annotation.DeveloperApi import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.Logging +import org.apache.spark.internal.config._ import org.apache.spark.scheduler._ import org.apache.spark.scheduler.SchedulingMode.SchedulingMode import org.apache.spark.storage.BlockManagerId @@ -93,6 +94,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { val retainedStages = conf.getInt("spark.ui.retainedStages", SparkUI.DEFAULT_RETAINED_STAGES) val retainedJobs = conf.getInt("spark.ui.retainedJobs", SparkUI.DEFAULT_RETAINED_JOBS) + val retainedTasks = conf.get(UI_RETAINED_TASKS) // We can test for memory leaks by ensuring that collections that track non-active jobs and // stages do not grow without bound and that collections for active jobs/stages eventually become @@ -400,6 +402,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { taskData.updateTaskMetrics(taskMetrics) taskData.errorMessage = errorMessage + // If Tasks is too large, remove and garbage collect old tasks + if (stageData.taskData.size > retainedTasks) { +stageData.taskData = stageData.taskData.drop(stageData.taskData.size - retainedTasks) + } + for ( activeJobsDependentOnStage <- sta
spark git commit: [SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large application history
Repository: spark Updated Branches: refs/heads/master 02ac379e8 -> f7beae6da [SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large application history ## What changes were proposed in this pull request? With the new History Server the summary page loads the application list via the the REST API, this makes it very slow to impossible to load with large (10K+) application history. This pr fixes this by adding the `spark.history.ui.maxApplications` conf to limit the number of applications the History Server displays. This is accomplished using a new optional `limit` param for the `applications` api. (Note this only applies to what the summary page displays, all the Application UI's are still accessible if the user knows the App ID and goes to the Application UI directly.) I've also added a new test for the `limit` param in `HistoryServerSuite.scala` ## How was this patch tested? Manual testing and dev/run-tests Author: Alex Bozarth Closes #14835 from ajbozarth/spark17243. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f7beae6d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f7beae6d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f7beae6d Branch: refs/heads/master Commit: f7beae6da02e6b70a34c714e93136becbde7679b Parents: 02ac379 Author: Alex Bozarth Authored: Tue Aug 30 16:33:54 2016 -0500 Committer: Tom Graves Committed: Tue Aug 30 16:33:54 2016 -0500 -- .../org/apache/spark/ui/static/historypage.js | 8 ++- .../spark/deploy/history/HistoryPage.scala | 3 +- .../spark/deploy/history/HistoryServer.scala| 4 ++ .../apache/spark/internal/config/package.scala | 4 ++ .../status/api/v1/ApplicationListResource.scala | 10 ++- .../limit_app_list_json_expectation.json| 67 .../deploy/history/HistoryServerSuite.scala | 1 + docs/monitoring.md | 16 - 8 files changed, 106 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f7beae6d/core/src/main/resources/org/apache/spark/ui/static/historypage.js -- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 5b9afb5..c809400 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -15,6 +15,12 @@ * limitations under the License. */ +var appLimit = -1; + +function setAppLimit(val) { +appLimit = val; +} + function makeIdNumeric(id) { var strs = id.split("_"); if (strs.length < 3) { @@ -89,7 +95,7 @@ $(document).ready(function() { requestedIncomplete = getParameterByName("showIncomplete", searchString); requestedIncomplete = (requestedIncomplete == "true" ? true : false); -$.getJSON("api/v1/applications", function(response,status,jqXHR) { +$.getJSON("api/v1/applications?limit=" + appLimit, function(response,status,jqXHR) { var array = []; var hasMultipleAttempts = false; for (i in response) { http://git-wip-us.apache.org/repos/asf/spark/blob/f7beae6d/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala index 74f7802..b4f5a61 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala @@ -45,7 +45,8 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") ++ ++ ++ - + ++ +setAppLimit({parent.maxApplications}) } else if (requestedIncomplete) { No incomplete applications found! } else { http://git-wip-us.apache.org/repos/asf/spark/blob/f7beae6d/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index d821474..c178917 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -28,6 +28,7 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.apache.spark.{SecurityManager, SparkConf} import org
spark git commit: [SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large application history
Repository: spark Updated Branches: refs/heads/branch-2.0 bc6c0d9f9 -> 021aa28f4 [SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large application history ## What changes were proposed in this pull request? back port of #14835 addressing merge conflicts With the new History Server the summary page loads the application list via the the REST API, this makes it very slow to impossible to load with large (10K+) application history. This pr fixes this by adding the `spark.history.ui.maxApplications` conf to limit the number of applications the History Server displays. This is accomplished using a new optional `limit` param for the `applications` api. (Note this only applies to what the summary page displays, all the Application UI's are still accessible if the user knows the App ID and goes to the Application UI directly.) I've also added a new test for the `limit` param in `HistoryServerSuite.scala` ## How was this patch tested? Manual testing and dev/run-tests Author: Alex Bozarth Closes #14886 from ajbozarth/spark17243-branch-2.0. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/021aa28f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/021aa28f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/021aa28f Branch: refs/heads/branch-2.0 Commit: 021aa28f439443cda1bc7c5e3eee7c85b40c1a2d Parents: bc6c0d9 Author: Alex Bozarth Authored: Wed Aug 31 08:50:42 2016 -0500 Committer: Tom Graves Committed: Wed Aug 31 08:50:42 2016 -0500 -- .../org/apache/spark/ui/static/historypage.js | 8 ++- .../spark/deploy/history/HistoryPage.scala | 3 +- .../spark/deploy/history/HistoryServer.scala| 4 ++ .../apache/spark/internal/config/package.scala | 4 ++ .../status/api/v1/ApplicationListResource.scala | 10 ++- .../limit_app_list_json_expectation.json| 67 .../deploy/history/HistoryServerSuite.scala | 1 + docs/monitoring.md | 16 - 8 files changed, 106 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/021aa28f/core/src/main/resources/org/apache/spark/ui/static/historypage.js -- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index d216166..177120a 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -15,6 +15,12 @@ * limitations under the License. */ +var appLimit = -1; + +function setAppLimit(val) { +appLimit = val; +} + // this function works exactly the same as UIUtils.formatDuration function formatDuration(milliseconds) { if (milliseconds < 100) { @@ -111,7 +117,7 @@ $(document).ready(function() { requestedIncomplete = getParameterByName("showIncomplete", searchString); requestedIncomplete = (requestedIncomplete == "true" ? true : false); -$.getJSON("api/v1/applications", function(response,status,jqXHR) { +$.getJSON("api/v1/applications?limit=" + appLimit, function(response,status,jqXHR) { var array = []; var hasMultipleAttempts = false; for (i in response) { http://git-wip-us.apache.org/repos/asf/spark/blob/021aa28f/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala index 2fad112..a120b6c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala @@ -44,7 +44,8 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") if (allAppsSize > 0) { ++ ++ - + ++ + setAppLimit({parent.maxApplications}) } else if (requestedIncomplete) { No incomplete applications found! } else { http://git-wip-us.apache.org/repos/asf/spark/blob/021aa28f/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index d821474..c178917 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -28,6 +28,7 @@ import
spark git commit: [SPARK-16711] YarnShuffleService doesn't re-init properly on YARN rolling upgrade
Repository: spark Updated Branches: refs/heads/branch-2.0 28377da38 -> e169085cd [SPARK-16711] YarnShuffleService doesn't re-init properly on YARN rolling upgrade branch-2.0 version of this patch. The differences are in the YarnShuffleService for finding the location to put the DB. branch-2.0 does not use the yarn nm recovery path like master does. Tested in manually on 8 node yarn cluster and ran unit tests. Manually tests verified DB created properly and it found them if already existed. Verified that during rolling upgrade credentials were reloaded and running application was not affected. Author: Thomas Graves Closes #14997 from tgravescs/SPARK-16711-branch2.0. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e169085c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e169085c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e169085c Branch: refs/heads/branch-2.0 Commit: e169085cd33ff498ecd5aab180af036ca644c9e0 Parents: 28377da Author: Thomas Graves Authored: Thu Sep 8 08:16:19 2016 -0500 Committer: Tom Graves Committed: Thu Sep 8 08:16:19 2016 -0500 -- common/network-common/pom.xml | 16 ++ .../spark/network/util/LevelDBProvider.java | 152 ++ common/network-shuffle/pom.xml | 16 -- .../shuffle/ExternalShuffleBlockResolver.java | 120 +-- .../spark/network/yarn/YarnShuffleService.java | 198 +++ .../network/yarn/YarnShuffleServiceSuite.scala | 10 + 6 files changed, 345 insertions(+), 167 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e169085c/common/network-common/pom.xml -- diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index bc3b0fe..e170b9b 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -42,6 +42,22 @@ netty-all + + org.fusesource.leveldbjni + leveldbjni-all + 1.8 + + + + com.fasterxml.jackson.core + jackson-databind + + + + com.fasterxml.jackson.core + jackson-annotations + + org.slf4j http://git-wip-us.apache.org/repos/asf/spark/blob/e169085c/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java -- diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java b/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java new file mode 100644 index 000..ec900a7 --- /dev/null +++ b/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.util; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.fusesource.leveldbjni.JniDBFactory; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.Options; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * LevelDB utility class available in the network package. + */ +public class LevelDBProvider { + private static final Logger logger = LoggerFactory.getLogger(LevelDBProvider.class); + + public static DB initLevelDB(File dbFile, StoreVersion version, ObjectMapper mapper) throws + IOException { +DB tmpDb = null; +if (dbFile != null) { + Options options = new Options(); + options.createIfMissing(false); + options.logger(new LevelDBLogger()); + try { +tmpDb = JniDBFactory.factory.open(dbFile, options); + } catch (NativeDB.DBException e) { +if (e.isNotFound() || e.getMessage().contains(" does not exist ")) { +
spark git commit: [SPARK-17433] YarnShuffleService doesn't handle moving credentials levelDb
Repository: spark Updated Branches: refs/heads/master 7098a1294 -> a3981c28c [SPARK-17433] YarnShuffleService doesn't handle moving credentials levelDb The secrets leveldb isn't being moved if you run spark shuffle services without yarn nm recovery on and then turn it on. This fixes that. I unfortunately missed this when I ported the patch from our internal branch 2 to master branch due to the changes for the recovery path. Note this only applies to master since it is the only place the yarn nm recovery dir is used. Unit tests ran and tested on 8 node cluster. Fresh startup with NM recovery, fresh startup no nm recovery, switching between no nm recovery and recovery. Also tested running applications to make sure wasn't affected by rolling upgrade. Author: Thomas Graves Author: Tom Graves Closes #14999 from tgravescs/SPARK-17433. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a3981c28 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a3981c28 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a3981c28 Branch: refs/heads/master Commit: a3981c28c956a82ccf5b1c61d45b6bd252d4abed Parents: 7098a12 Author: Thomas Graves Authored: Fri Sep 9 13:43:32 2016 -0500 Committer: Tom Graves Committed: Fri Sep 9 13:43:32 2016 -0500 -- .../spark/network/yarn/YarnShuffleService.java | 56 ++-- .../network/yarn/YarnShuffleServiceSuite.scala | 12 - 2 files changed, 50 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a3981c28/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java -- diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java index df082e4..43c8df7 100644 --- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java +++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.ByteBuffer; +import java.nio.file.Files; import java.util.List; import java.util.Map; @@ -159,8 +160,7 @@ public class YarnShuffleService extends AuxiliaryService { // If we don't find one, then we choose a file to use to save the state next time. Even if // an application was stopped while the NM was down, we expect yarn to call stopApplication() // when it comes back - registeredExecutorFile = -new File(getRecoveryPath().toUri().getPath(), RECOVERY_FILE_NAME); + registeredExecutorFile = initRecoveryDb(RECOVERY_FILE_NAME); TransportConf transportConf = new TransportConf("shuffle", new HadoopConfigProvider(conf)); blockHandler = new ExternalShuffleBlockHandler(transportConf, registeredExecutorFile); @@ -196,7 +196,7 @@ public class YarnShuffleService extends AuxiliaryService { private void createSecretManager() throws IOException { secretManager = new ShuffleSecretManager(); -secretsFile = new File(getRecoveryPath().toUri().getPath(), SECRETS_RECOVERY_FILE_NAME); +secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME); // Make sure this is protected in case its not in the NM recovery dir FileSystem fs = FileSystem.getLocal(_conf); @@ -328,37 +328,59 @@ public class YarnShuffleService extends AuxiliaryService { } /** - * Get the recovery path, this will override the default one to get our own maintained - * recovery path. + * Get the path specific to this auxiliary service to use for recovery. + */ + protected Path getRecoveryPath(String fileName) { +return _recoveryPath; + } + + /** + * Figure out the recovery path and handle moving the DB if YARN NM recovery gets enabled + * when it previously was not. If YARN NM recovery is enabled it uses that path, otherwise + * it will uses a YARN local dir. */ - protected Path getRecoveryPath() { + protected File initRecoveryDb(String dbFileName) { +if (_recoveryPath != null) { +File recoveryFile = new File(_recoveryPath.toUri().getPath(), dbFileName); +if (recoveryFile.exists()) { + return recoveryFile; +} +} +// db doesn't exist in recovery path go check local dirs for it String[] localDirs = _conf.getTrimmedStrings("yarn.nodemanager.local-dirs"); for (String dir : localDirs) { - File f = new File(new Path(dir).toUri().getPath(), RECOVERY_FILE_NAME); + File f = new File(new Path(dir).toUri().getPath(), dbFileName); if (f.exists()) { if (_recoveryPath == null) {
spark git commit: [SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as Failed
Repository: spark Updated Branches: refs/heads/master 040e46979 -> ff6e4cbdc [SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as Failed ## What changes were proposed in this pull request? Due to race conditions, the ` assert(numExecutorsRunning <= targetNumExecutors)` can fail causing `AssertionError`. So removed the assertion, instead moved the conditional check before launching new container: ``` java.lang.AssertionError: assertion failed at scala.Predef$.assert(Predef.scala:156) at org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1.org$apache$spark$deploy$yarn$YarnAllocator$$anonfun$$updateInternalState$1(YarnAllocator.scala:489) at org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1$$anon$1.run(YarnAllocator.scala:519) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) ``` ## How was this patch tested? This was manually tested using a large ForkAndJoin job with Dynamic Allocation enabled to validate the failing job succeeds, without any such exception. Author: Kishor Patil Closes #15069 from kishorvpatil/SPARK-17511. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ff6e4cbd Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ff6e4cbd Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ff6e4cbd Branch: refs/heads/master Commit: ff6e4cbdc80e2ad84c5d70ee07f323fad9374e3e Parents: 040e469 Author: Kishor Patil Authored: Wed Sep 14 14:19:35 2016 -0500 Committer: Tom Graves Committed: Wed Sep 14 14:19:35 2016 -0500 -- .../spark/deploy/yarn/YarnAllocator.scala | 62 +++- .../spark/deploy/yarn/YarnAllocatorSuite.scala | 19 ++ 2 files changed, 52 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ff6e4cbd/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala -- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 2f4b498..0b66d1c 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -496,7 +496,6 @@ private[yarn] class YarnAllocator( def updateInternalState(): Unit = synchronized { numExecutorsRunning += 1 -assert(numExecutorsRunning <= targetNumExecutors) executorIdToContainer(executorId) = container containerIdToExecutorId(container.getId) = executorId @@ -506,36 +505,41 @@ private[yarn] class YarnAllocator( allocatedContainerToHostMap.put(containerId, executorHostname) } - if (launchContainers) { -launcherPool.execute(new Runnable { - override def run(): Unit = { -try { - new ExecutorRunnable( -Some(container), -conf, -sparkConf, -driverUrl, -executorId, -executorHostname, -executorMemory, -executorCores, -appAttemptId.getApplicationId.toString, -securityMgr, -localResources - ).run() - updateInternalState() -} catch { - case NonFatal(e) => -logError(s"Failed to launch executor $executorId on container $containerId", e) -// Assigned container should be released immediately to avoid unnecessary resource -// occupation. -amClient.releaseAssignedContainer(containerId) + if (numExecutorsRunning < targetNumExecutors) { +if (launchContainers) { + launcherPool.execute(new Runnable { +override def run(): Unit = { + try { +new ExecutorRunnable( + Some(container), + conf, + sparkConf, + driverUrl, + executorId, + executorHostname, + executorMemory, + executorCores, + appAttemptId.getApplicationId.toString, + securityMgr, + localResources +).run() +updateInternalState() + } catch { +case NonFatal(e) => + logError(s"Failed to launch executor $executorId on container $containerId", e) + // Assigned container should be released i
spark git commit: [SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as Failed
Repository: spark Updated Branches: refs/heads/branch-2.0 6fe5972e6 -> fab77dadf [SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as Failed Due to race conditions, the ` assert(numExecutorsRunning <= targetNumExecutors)` can fail causing `AssertionError`. So removed the assertion, instead moved the conditional check before launching new container: ``` java.lang.AssertionError: assertion failed at scala.Predef$.assert(Predef.scala:156) at org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1.org$apache$spark$deploy$yarn$YarnAllocator$$anonfun$$updateInternalState$1(YarnAllocator.scala:489) at org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1$$anon$1.run(YarnAllocator.scala:519) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) ``` This was manually tested using a large ForkAndJoin job with Dynamic Allocation enabled to validate the failing job succeeds, without any such exception. Author: Kishor Patil Closes #15069 from kishorvpatil/SPARK-17511. (cherry picked from commit ff6e4cbdc80e2ad84c5d70ee07f323fad9374e3e) Signed-off-by: Tom Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fab77dad Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fab77dad Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fab77dad Branch: refs/heads/branch-2.0 Commit: fab77dadf70d011cec8976acfe8c851816f82426 Parents: 6fe5972 Author: Kishor Patil Authored: Wed Sep 14 14:19:35 2016 -0500 Committer: Tom Graves Committed: Wed Sep 14 14:33:40 2016 -0500 -- .../spark/deploy/yarn/YarnAllocator.scala | 68 +++- .../spark/deploy/yarn/YarnAllocatorSuite.scala | 19 ++ 2 files changed, 55 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/fab77dad/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala -- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 1b80071..b321901 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -483,7 +483,6 @@ private[yarn] class YarnAllocator( def updateInternalState(): Unit = synchronized { numExecutorsRunning += 1 -assert(numExecutorsRunning <= targetNumExecutors) executorIdToContainer(executorId) = container containerIdToExecutorId(container.getId) = executorId @@ -493,39 +492,44 @@ private[yarn] class YarnAllocator( allocatedContainerToHostMap.put(containerId, executorHostname) } - if (launchContainers) { -logInfo("Launching ExecutorRunnable. driverUrl: %s, executorHostname: %s".format( - driverUrl, executorHostname)) - -launcherPool.execute(new Runnable { - override def run(): Unit = { -try { - new ExecutorRunnable( -container, -conf, -sparkConf, -driverUrl, -executorId, -executorHostname, -executorMemory, -executorCores, -appAttemptId.getApplicationId.toString, -securityMgr, -localResources - ).run() - updateInternalState() -} catch { - case NonFatal(e) => -logError(s"Failed to launch executor $executorId on container $containerId", e) -// Assigned container should be released immediately to avoid unnecessary resource -// occupation. -amClient.releaseAssignedContainer(containerId) + if (numExecutorsRunning < targetNumExecutors) { +if (launchContainers) { + logInfo("Launching ExecutorRunnable. driverUrl: %s, executorHostname: %s".format( +driverUrl, executorHostname)) + + launcherPool.execute(new Runnable { +override def run(): Unit = { + try { +new ExecutorRunnable( + container, + conf, + sparkConf, + driverUrl, + executorId, + executorHostname, + executorMemory, + executorCores, + appAttemptId.getApplicationId.toString, + securityMgr, + localResources +
spark git commit: [SPARK-16757] Set up Spark caller context to HDFS and YARN
Repository: spark Updated Branches: refs/heads/master 7f16affa2 -> 6a68c5d7b [SPARK-16757] Set up Spark caller context to HDFS and YARN ## What changes were proposed in this pull request? 1. Pass `jobId` to Task. 2. Invoke Hadoop APIs. * A new function `setCallerContext` is added in `Utils`. `setCallerContext` function invokes APIs of `org.apache.hadoop.ipc.CallerContext` to set up spark caller contexts, which will be written into `hdfs-audit.log` and Yarn RM audit log. * For HDFS: Spark sets up its caller context by invoking`org.apache.hadoop.ipc.CallerContext` in `Task` and Yarn `Client` and `ApplicationMaster`. * For Yarn: Spark sets up its caller context by invoking `org.apache.hadoop.ipc.CallerContext` in Yarn `Client`. ## How was this patch tested? Manual Tests against some Spark applications in Yarn client mode and Yarn cluster mode. Need to check if spark caller contexts are written into HDFS hdfs-audit.log and Yarn RM audit log successfully. For example, run SparkKmeans in Yarn client mode: ``` ./bin/spark-submit --verbose --executor-cores 3 --num-executors 1 --master yarn --deploy-mode client --class org.apache.spark.examples.SparkKMeans examples/target/original-spark-examples_2.11-2.1.0-SNAPSHOT.jar hdfs://localhost:9000/lr_big.txt 2 5 ``` **Before**: There will be no Spark caller context in records of `hdfs-audit.log` and Yarn RM audit log. **After**: Spark caller contexts will be written in records of `hdfs-audit.log` and Yarn RM audit log. These are records in `hdfs-audit.log`: ``` 2016-09-20 11:54:24,116 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_CLIENT_AppId_application_1474394339641_0005 2016-09-20 11:54:28,164 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0005_JobId_0_StageId_0_AttemptId_0_TaskId_2_AttemptNum_0 2016-09-20 11:54:28,164 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0005_JobId_0_StageId_0_AttemptId_0_TaskId_1_AttemptNum_0 2016-09-20 11:54:28,164 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0005_JobId_0_StageId_0_AttemptId_0_TaskId_0_AttemptNum_0 ``` ``` 2016-09-20 11:59:33,868 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=mkdirs src=/private/tmp/hadoop-wyang/nm-local-dir/usercache/wyang/appcache/application_1474394339641_0006/container_1474394339641_0006_01_01/spark-warehouse dst=nullperm=wyang:supergroup:rwxr-xr-x proto=rpc callerContext=SPARK_APPLICATION_MASTER_AppId_application_1474394339641_0006_AttemptId_1 2016-09-20 11:59:37,214 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_1_AttemptNum_0 2016-09-20 11:59:37,215 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_2_AttemptNum_0 2016-09-20 11:59:37,215 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_0_AttemptNum_0 2016-09-20 11:59:42,391 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_3_AttemptNum_0 ``` This is a record in Yarn RM log: ``` 2016-09-20 11:59:24,050 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=wyang IP=127.0.0.1OPERATION=Submit Application RequestTARGET=ClientRMService RESULT=SUCCESS APPID=application_1474394339641_0006 CALLERCONTEXT=SPARK_CLIENT_AppId_application_1474394339641_0006 ``` Author: Weiqing Yang Closes #14659 from Sherry302/callercontextSubmit. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a68c5d7 Tree: http://git-wip-us.apache.or
spark git commit: [SPARK-17710][HOTFIX] Fix ClassCircularityError in ReplSuite tests in Maven build: use 'Class.forName' instead of 'Utils.classForName'
Repository: spark Updated Branches: refs/heads/master 7d0923202 -> 7dfad4b13 [SPARK-17710][HOTFIX] Fix ClassCircularityError in ReplSuite tests in Maven build: use 'Class.forName' instead of 'Utils.classForName' ## What changes were proposed in this pull request? Fix ClassCircularityError in ReplSuite tests when Spark is built by Maven build. ## How was this patch tested? (1) ``` build/mvn -DskipTests -Phadoop-2.3 -Pyarn -Phive -Phive-thriftserver -Pkinesis-asl -Pmesos clean package ``` Then test: ``` build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.repl.ReplSuite test ``` ReplSuite tests passed (2) Manual Tests against some Spark applications in Yarn client mode and Yarn cluster mode. Need to check if spark caller contexts are written into HDFS hdfs-audit.log and Yarn RM audit log successfully. Author: Weiqing Yang Closes #15286 from Sherry302/SPARK-16757. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7dfad4b1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7dfad4b1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7dfad4b1 Branch: refs/heads/master Commit: 7dfad4b132bc46263ef788ced4a935862f5c8756 Parents: 7d09232 Author: Weiqing Yang Authored: Wed Sep 28 20:20:03 2016 -0500 Committer: Tom Graves Committed: Wed Sep 28 20:20:03 2016 -0500 -- core/src/main/scala/org/apache/spark/util/Utils.scala | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/7dfad4b1/core/src/main/scala/org/apache/spark/util/Utils.scala -- diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index caa768c..f3493bd 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2489,8 +2489,10 @@ private[spark] class CallerContext( def setCurrentContext(): Boolean = { var succeed = false try { - val callerContext = Utils.classForName("org.apache.hadoop.ipc.CallerContext") - val Builder = Utils.classForName("org.apache.hadoop.ipc.CallerContext$Builder") + // scalastyle:off classforname + val callerContext = Class.forName("org.apache.hadoop.ipc.CallerContext") + val Builder = Class.forName("org.apache.hadoop.ipc.CallerContext$Builder") + // scalastyle:on classforname val builderInst = Builder.getConstructor(classOf[String]).newInstance(context) val hdfsContext = Builder.getMethod("build").invoke(builderInst) callerContext.getMethod("setCurrent", callerContext).invoke(null, hdfsContext) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-25043] print master and appId from spark-sql on startup
Repository: spark Updated Branches: refs/heads/master 102487584 -> b81e3031f [SPARK-25043] print master and appId from spark-sql on startup ## What changes were proposed in this pull request? A small change to print the master and appId from spark-sql as with logging turned down all the way (`log4j.logger.org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver=WARN`), we may not know this information easily. This adds the following string before the `spark-sql>` prompt shows on the screen. `Spark master: yarn, Application Id: application_123456789_12345` ## How was this patch tested? I ran spark-sql locally and saw the appId displayed as expected. Please review http://spark.apache.org/contributing.html before opening a pull request. Closes #22025 from abellina/SPARK-25043_print_master_and_app_id_from_sparksql. Lead-authored-by: Alessandro Bellina Co-authored-by: Alessandro Bellina Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b81e3031 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b81e3031 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b81e3031 Branch: refs/heads/master Commit: b81e3031fd247dfb4b3e02e0a986fb4b19d00f7c Parents: 1024875 Author: Alessandro Bellina Authored: Tue Aug 14 13:15:55 2018 -0500 Committer: Thomas Graves Committed: Tue Aug 14 13:15:55 2018 -0500 -- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 8 1 file changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b81e3031/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala -- diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index d9fd3eb..bb96cea 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -258,6 +258,8 @@ private[hive] object SparkSQLCLIDriver extends Logging { def continuedPromptWithDBSpaces: String = continuedPrompt + ReflectionUtils.invokeStatic( classOf[CliDriver], "spacesForString", classOf[String] -> currentDB) +cli.printMasterAndAppId + var currentPrompt = promptWithCurrentDB var line = reader.readLine(currentPrompt + "> ") @@ -323,6 +325,12 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { hiveVariables.asScala.foreach(kv => SparkSQLEnv.sqlContext.conf.setConfString(kv._1, kv._2)) } + def printMasterAndAppId(): Unit = { +val master = SparkSQLEnv.sparkContext.master +val appId = SparkSQLEnv.sparkContext.applicationId +console.printInfo(s"Spark master: $master, Application Id: $appId") + } + override def processCmd(cmd: String): Int = { val cmd_trimmed: String = cmd.trim() val cmd_lower = cmd_trimmed.toLowerCase(Locale.ROOT) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24909][CORE] Always unregister pending partition on task completion.
Repository: spark Updated Branches: refs/heads/branch-2.3 b072717b3 -> dbf0b9340 [SPARK-24909][CORE] Always unregister pending partition on task completion. Spark scheduler can hang when fetch failures, executor lost, task running on lost executor, and multiple stage attempts. To fix this we change to always unregister the pending partition on task completion. this PR is actually reverting the change in SPARK-19263, so that it always does shuffleStage.pendingPartitions -= task.partitionId. The change in SPARK-23433, should fix the issue originally from SPARK-19263. Unit tests. The condition happens on a race which I haven't reproduced on a real customer, just see it sometimes on customers jobs in a real cluster. I am also working on adding spark scheduler integration tests. Closes #21976 from tgravescs/SPARK-24909. Authored-by: Thomas Graves Signed-off-by: Marcelo Vanzin (cherry picked from commit ec3e9986385880adce1648eae30007eccff862ba) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dbf0b934 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dbf0b934 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dbf0b934 Branch: refs/heads/branch-2.3 Commit: dbf0b934024dfa562775967e3e3114de5a163443 Parents: b072717 Author: Thomas Graves Authored: Wed Aug 29 16:32:02 2018 -0700 Committer: Thomas Graves Committed: Thu Aug 30 09:10:00 2018 -0500 -- .../apache/spark/scheduler/DAGScheduler.scala | 17 +- .../spark/scheduler/DAGSchedulerSuite.scala | 24 2 files changed, 16 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/dbf0b934/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 7029e22..0df38f1 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1252,18 +1252,10 @@ class DAGScheduler( case smt: ShuffleMapTask => val shuffleStage = stage.asInstanceOf[ShuffleMapStage] +shuffleStage.pendingPartitions -= task.partitionId val status = event.result.asInstanceOf[MapStatus] val execId = status.location.executorId logDebug("ShuffleMapTask finished on " + execId) -if (stageIdToStage(task.stageId).latestInfo.attemptNumber == task.stageAttemptId) { - // This task was for the currently running attempt of the stage. Since the task - // completed successfully from the perspective of the TaskSetManager, mark it as - // no longer pending (the TaskSetManager may consider the task complete even - // when the output needs to be ignored because the task's epoch is too small below. - // In this case, when pending partitions is empty, there will still be missing - // output locations, which will cause the DAGScheduler to resubmit the stage below.) - shuffleStage.pendingPartitions -= task.partitionId -} if (failedEpoch.contains(execId) && smt.epoch <= failedEpoch(execId)) { logInfo(s"Ignoring possibly bogus $smt completion from executor $execId") } else { @@ -1272,13 +1264,6 @@ class DAGScheduler( // available. mapOutputTracker.registerMapOutput( shuffleStage.shuffleDep.shuffleId, smt.partitionId, status) - // Remove the task's partition from pending partitions. This may have already been - // done above, but will not have been done yet in cases where the task attempt was - // from an earlier attempt of the stage (i.e., not the attempt that's currently - // running). This allows the DAGScheduler to mark the stage as complete when one - // copy of each task has finished successfully, even if the currently active stage - // still has tasks running. - shuffleStage.pendingPartitions -= task.partitionId } if (runningStages.contains(shuffleStage) && shuffleStage.pendingPartitions.isEmpty) { http://git-wip-us.apache.org/repos/asf/spark/blob/dbf0b934/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala index 8b6ec37..c0
spark git commit: [SPARK-25231] Fix synchronization of executor heartbeat receiver in TaskSchedulerImpl
Repository: spark Updated Branches: refs/heads/master 925449283 -> 559b899ac [SPARK-25231] Fix synchronization of executor heartbeat receiver in TaskSchedulerImpl Running a large Spark job with speculation turned on was causing executor heartbeats to time out on the driver end after sometime and eventually, after hitting the max number of executor failures, the job would fail. ## What changes were proposed in this pull request? The main reason for the heartbeat timeouts was that the heartbeat-receiver-event-loop-thread was blocked waiting on the TaskSchedulerImpl object which was being held by one of the dispatcher-event-loop threads executing the method dequeueSpeculativeTasks() in TaskSetManager.scala. On further analysis of the heartbeat receiver method executorHeartbeatReceived() in TaskSchedulerImpl class, we found out that instead of waiting to acquire the lock on the TaskSchedulerImpl object, we can remove that lock and make the operations to the global variables inside the code block to be atomic. The block of code in that method only uses one global HashMap taskIdToTaskSetManager. Making that map a ConcurrentHashMap, we are ensuring atomicity of operations and speeding up the heartbeat receiver thread operation. ## How was this patch tested? Screenshots of the thread dump have been attached below: **heartbeat-receiver-event-loop-thread:** https://user-images.githubusercontent.com/8190/44593413-e25df780-a788-11e8-9520-176a18401a59.png";> **dispatcher-event-loop-thread:** https://user-images.githubusercontent.com/8190/44593484-13d6c300-a789-11e8-8d88-34b1d51d4541.png";> Closes #1 from pgandhi999/SPARK-25231. Authored-by: pgandhi Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/559b899a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/559b899a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/559b899a Branch: refs/heads/master Commit: 559b899aceb160fcec3a57109c0b60a0ae40daeb Parents: 9254492 Author: pgandhi Authored: Wed Sep 5 16:10:49 2018 -0500 Committer: Thomas Graves Committed: Wed Sep 5 16:10:49 2018 -0500 -- .../org/apache/spark/scheduler/TaskSchedulerImpl.scala | 12 ++-- .../cluster/CoarseGrainedSchedulerBackend.scala | 2 +- .../spark/scheduler/SchedulerIntegrationSuite.scala | 3 ++- .../apache/spark/scheduler/TaskSchedulerImplSuite.scala | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/559b899a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 8992d7e..8b71170 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -19,7 +19,7 @@ package org.apache.spark.scheduler import java.nio.ByteBuffer import java.util.{Locale, Timer, TimerTask} -import java.util.concurrent.TimeUnit +import java.util.concurrent.{ConcurrentHashMap, TimeUnit} import java.util.concurrent.atomic.AtomicLong import scala.collection.Set @@ -91,7 +91,7 @@ private[spark] class TaskSchedulerImpl( private val taskSetsByStageIdAndAttempt = new HashMap[Int, HashMap[Int, TaskSetManager]] // Protected by `this` - private[scheduler] val taskIdToTaskSetManager = new HashMap[Long, TaskSetManager] + private[scheduler] val taskIdToTaskSetManager = new ConcurrentHashMap[Long, TaskSetManager] val taskIdToExecutorId = new HashMap[Long, String] @volatile private var hasReceivedTask = false @@ -315,7 +315,7 @@ private[spark] class TaskSchedulerImpl( for (task <- taskSet.resourceOffer(execId, host, maxLocality)) { tasks(i) += task val tid = task.taskId -taskIdToTaskSetManager(tid) = taskSet +taskIdToTaskSetManager.put(tid, taskSet) taskIdToExecutorId(tid) = execId executorIdToRunningTaskIds(execId).add(tid) availableCpus(i) -= CPUS_PER_TASK @@ -465,7 +465,7 @@ private[spark] class TaskSchedulerImpl( var reason: Option[ExecutorLossReason] = None synchronized { try { -taskIdToTaskSetManager.get(tid) match { +Option(taskIdToTaskSetManager.get(tid)) match { case Some(taskSet) => if (state == TaskState.LOST) { // TaskState.LOST is only used by the deprecated Mesos fine-grained scheduling mode, @@ -517,10 +517,10 @@ private[spark] class TaskSchedulerImpl( accumUpdates: Array[(Long, Seq[AccumulatorV2[_,
spark git commit: [SPARK-25231] Fix synchronization of executor heartbeat receiver in TaskSchedulerImpl
Repository: spark Updated Branches: refs/heads/branch-2.3 dbf0b9340 -> 31e46ec60 [SPARK-25231] Fix synchronization of executor heartbeat receiver in TaskSchedulerImpl Running a large Spark job with speculation turned on was causing executor heartbeats to time out on the driver end after sometime and eventually, after hitting the max number of executor failures, the job would fail. ## What changes were proposed in this pull request? The main reason for the heartbeat timeouts was that the heartbeat-receiver-event-loop-thread was blocked waiting on the TaskSchedulerImpl object which was being held by one of the dispatcher-event-loop threads executing the method dequeueSpeculativeTasks() in TaskSetManager.scala. On further analysis of the heartbeat receiver method executorHeartbeatReceived() in TaskSchedulerImpl class, we found out that instead of waiting to acquire the lock on the TaskSchedulerImpl object, we can remove that lock and make the operations to the global variables inside the code block to be atomic. The block of code in that method only uses one global HashMap taskIdToTaskSetManager. Making that map a ConcurrentHashMap, we are ensuring atomicity of operations and speeding up the heartbeat receiver thread operation. ## How was this patch tested? Screenshots of the thread dump have been attached below: **heartbeat-receiver-event-loop-thread:** https://user-images.githubusercontent.com/8190/44593413-e25df780-a788-11e8-9520-176a18401a59.png";> **dispatcher-event-loop-thread:** https://user-images.githubusercontent.com/8190/44593484-13d6c300-a789-11e8-8d88-34b1d51d4541.png";> Closes #1 from pgandhi999/SPARK-25231. Authored-by: pgandhi Signed-off-by: Thomas Graves (cherry picked from commit 559b899aceb160fcec3a57109c0b60a0ae40daeb) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/31e46ec6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/31e46ec6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/31e46ec6 Branch: refs/heads/branch-2.3 Commit: 31e46ec60849d315a4e83e0a332606a4405907ad Parents: dbf0b93 Author: pgandhi Authored: Wed Sep 5 16:10:49 2018 -0500 Committer: Thomas Graves Committed: Wed Sep 5 16:11:08 2018 -0500 -- .../org/apache/spark/scheduler/TaskSchedulerImpl.scala | 12 ++-- .../cluster/CoarseGrainedSchedulerBackend.scala | 2 +- .../spark/scheduler/SchedulerIntegrationSuite.scala | 3 ++- .../apache/spark/scheduler/TaskSchedulerImplSuite.scala | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/31e46ec6/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 56c0bf6..4edc6b2 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -19,7 +19,7 @@ package org.apache.spark.scheduler import java.nio.ByteBuffer import java.util.{Locale, Timer, TimerTask} -import java.util.concurrent.TimeUnit +import java.util.concurrent.{ConcurrentHashMap, TimeUnit} import java.util.concurrent.atomic.AtomicLong import scala.collection.Set @@ -90,7 +90,7 @@ private[spark] class TaskSchedulerImpl( private val taskSetsByStageIdAndAttempt = new HashMap[Int, HashMap[Int, TaskSetManager]] // Protected by `this` - private[scheduler] val taskIdToTaskSetManager = new HashMap[Long, TaskSetManager] + private[scheduler] val taskIdToTaskSetManager = new ConcurrentHashMap[Long, TaskSetManager] val taskIdToExecutorId = new HashMap[Long, String] @volatile private var hasReceivedTask = false @@ -286,7 +286,7 @@ private[spark] class TaskSchedulerImpl( for (task <- taskSet.resourceOffer(execId, host, maxLocality)) { tasks(i) += task val tid = task.taskId -taskIdToTaskSetManager(tid) = taskSet +taskIdToTaskSetManager.put(tid, taskSet) taskIdToExecutorId(tid) = execId executorIdToRunningTaskIds(execId).add(tid) availableCpus(i) -= CPUS_PER_TASK @@ -392,7 +392,7 @@ private[spark] class TaskSchedulerImpl( var reason: Option[ExecutorLossReason] = None synchronized { try { -taskIdToTaskSetManager.get(tid) match { +Option(taskIdToTaskSetManager.get(tid)) match { case Some(taskSet) => if (state == TaskState.LOST) { // TaskState.LOST is only used by the deprecated Mesos fine-grained scheduling mode, @@ -444,10
spark git commit: [SPARK-24415][CORE] Fixed the aggregated stage metrics by retaining stage objects in liveStages until all tasks are complete
Repository: spark Updated Branches: refs/heads/branch-2.3 84922e506 -> 5b8b6b4e9 [SPARK-24415][CORE] Fixed the aggregated stage metrics by retaining stage objects in liveStages until all tasks are complete The problem occurs because stage object is removed from liveStages in AppStatusListener onStageCompletion. Because of this any onTaskEnd event received after onStageCompletion event do not update stage metrics. The fix is to retain stage objects in liveStages until all tasks are complete. 1. Fixed the reproducible example posted in the JIRA 2. Added unit test Closes #22209 from ankuriitg/ankurgupta/SPARK-24415. Authored-by: ankurgupta Signed-off-by: Marcelo Vanzin (cherry picked from commit 39a02d8f75def7191c66d388729ba1721c92188d) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5b8b6b4e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5b8b6b4e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5b8b6b4e Branch: refs/heads/branch-2.3 Commit: 5b8b6b4e9e36228e993a15cab19c80e7fad43786 Parents: 84922e5 Author: ankurgupta Authored: Wed Sep 5 09:41:05 2018 -0700 Committer: Thomas Graves Committed: Fri Sep 7 08:48:39 2018 -0500 -- .../apache/spark/status/AppStatusListener.scala | 61 +++- .../spark/status/AppStatusListenerSuite.scala | 55 ++ .../spark/streaming/UISeleniumSuite.scala | 9 ++- 3 files changed, 108 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/5b8b6b4e/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala -- diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala index 496165c..9375ae9 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala @@ -310,11 +310,20 @@ private[spark] class AppStatusListener( val e = it.next() if (job.stageIds.contains(e.getKey()._1)) { val stage = e.getValue() - stage.status = v1.StageStatus.SKIPPED - job.skippedStages += stage.info.stageId - job.skippedTasks += stage.info.numTasks - it.remove() - update(stage, now) + if (v1.StageStatus.PENDING.equals(stage.status)) { +stage.status = v1.StageStatus.SKIPPED +job.skippedStages += stage.info.stageId +job.skippedTasks += stage.info.numTasks +job.activeStages -= 1 + +pools.get(stage.schedulingPool).foreach { pool => + pool.stageIds = pool.stageIds - stage.info.stageId + update(pool, now) +} + +it.remove() +update(stage, now, last = true) + } } } @@ -466,7 +475,16 @@ private[spark] class AppStatusListener( if (killedDelta > 0) { stage.killedSummary = killedTasksSummary(event.reason, stage.killedSummary) } - maybeUpdate(stage, now) + // [SPARK-24415] Wait for all tasks to finish before removing stage from live list + val removeStage = +stage.activeTasks == 0 && + (v1.StageStatus.COMPLETE.equals(stage.status) || +v1.StageStatus.FAILED.equals(stage.status)) + if (removeStage) { +update(stage, now, last = true) + } else { +maybeUpdate(stage, now) + } // Store both stage ID and task index in a single long variable for tracking at job level. val taskIndex = (event.stageId.toLong << Integer.SIZE) | event.taskInfo.index @@ -481,7 +499,7 @@ private[spark] class AppStatusListener( if (killedDelta > 0) { job.killedSummary = killedTasksSummary(event.reason, job.killedSummary) } -maybeUpdate(job, now) +conditionalLiveUpdate(job, now, removeStage) } val esummary = stage.executorSummary(event.taskInfo.executorId) @@ -492,7 +510,7 @@ private[spark] class AppStatusListener( if (metricsDelta != null) { esummary.metrics = LiveEntityHelpers.addMetrics(esummary.metrics, metricsDelta) } - maybeUpdate(esummary, now) + conditionalLiveUpdate(esummary, now, removeStage) if (!stage.cleaning && stage.savedTasks.get() > maxTasksPerStage) { stage.cleaning = true @@ -500,6 +518,9 @@ private[spark] class AppStatusListener( cleanupTasks(stage) } } + if (removeStage) { +liveStages.remove((event.stageId, event.stageAttemptId)) + } } liveExecutors.get(event.taskInfo.executorId).foreach { exec => @@ -524,17 +545,13 @@ private[spark] clas
spark git commit: [SPARK-24355] Spark external shuffle server improvement to better handle block fetch requests.
Repository: spark Updated Branches: refs/heads/master 2c9d8f56c -> ff601cf71 [SPARK-24355] Spark external shuffle server improvement to better handle block fetch requests. ## What changes were proposed in this pull request? Description: Right now, the default server side netty handler threads is 2 * # cores, and can be further configured with parameter spark.shuffle.io.serverThreads. In order to process a client request, it would require one available server netty handler thread. However, when the server netty handler threads start to process ChunkFetchRequests, they will be blocked on disk I/O, mostly due to disk contentions from the random read operations initiated by all the ChunkFetchRequests received from clients. As a result, when the shuffle server is serving many concurrent ChunkFetchRequests, the server side netty handler threads could all be blocked on reading shuffle files, thus leaving no handler thread available to process other types of requests which should all be very quick to process. This issue could potentially be fixed by limiting the number of netty handler threads that could get blocked when processing ChunkFetchRequest. We have a patch to do this by using a separate EventLoopGroup with a dedicated ChannelHandler to process ChunkFetchRequest. This enables shuffle server to reserve netty handler threads for non-ChunkFetchRequest, thus enabling consistent processing time for these requests which are fast to process. After deploying the patch in our infrastructure, we no longer see timeout issues with either executor registration with local shuffle server or shuffle client establishing connection with remote shuffle server. (Please fill in changes proposed in this fix) For Original PR please refer here https://github.com/apache/spark/pull/21402 ## How was this patch tested? Unit tests and stress testing. (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) Please review http://spark.apache.org/contributing.html before opening a pull request. Closes #22173 from redsanket/SPARK-24335. Authored-by: Sanket Chintapalli Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ff601cf7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ff601cf7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ff601cf7 Branch: refs/heads/master Commit: ff601cf71d226082e156c4ff9a8f5593aa7a2085 Parents: 2c9d8f5 Author: Sanket Chintapalli Authored: Fri Sep 21 09:05:56 2018 -0500 Committer: Thomas Graves Committed: Fri Sep 21 09:05:56 2018 -0500 -- .../apache/spark/network/TransportContext.java | 66 - .../server/ChunkFetchRequestHandler.java| 135 +++ .../network/server/TransportChannelHandler.java | 21 ++- .../network/server/TransportRequestHandler.java | 35 + .../spark/network/util/TransportConf.java | 28 .../network/ChunkFetchRequestHandlerSuite.java | 102 ++ .../spark/network/ExtendedChannelPromise.java | 69 ++ .../network/TransportRequestHandlerSuite.java | 55 +--- .../network/shuffle/ExternalShuffleClient.java | 2 +- 9 files changed, 425 insertions(+), 88 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ff601cf7/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java -- diff --git a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java index ae91bc9..480b526 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java +++ b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; import io.netty.channel.Channel; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.handler.timeout.IdleStateHandler; import org.slf4j.Logger; @@ -32,11 +34,13 @@ import org.apache.spark.network.client.TransportClientFactory; import org.apache.spark.network.client.TransportResponseHandler; import org.apache.spark.network.protocol.MessageDecoder; import org.apache.spark.network.protocol.MessageEncoder; +import org.apache.spark.network.server.ChunkFetchRequestHandler; import org.apache.spark.network.server.RpcHandler; import org.apache.spark.network.server.TransportChannelHandler; import org.apache.spark.network.server.TransportRequestHandler; import org.apache.spark.network.serve
spark git commit: [SPARK-18364][YARN] Expose metrics for YarnShuffleService
Repository: spark Updated Branches: refs/heads/master b96fd44f0 -> a802c69b1 [SPARK-18364][YARN] Expose metrics for YarnShuffleService ## What changes were proposed in this pull request? This PR is follow-up of closed https://github.com/apache/spark/pull/17401 which only ended due to of inactivity, but its still nice feature to have. Given review by jerryshao taken in consideration and edited: - VisibleForTesting deleted because of dependency conflicts - removed unnecessary reflection for `MetricsSystemImpl` - added more available types for gauge ## How was this patch tested? Manual deploy of new yarn-shuffle jar into a Node Manager and verifying that the metrics appear in the Node Manager-standard location. This is JMX with an query endpoint running on `hostname:port` Resulting metrics look like this: ``` curl -sk -XGET hostname:port | grep -v '#' | grep 'shuffleService' hadoop_nodemanager_openblockrequestlatencymillis_rate15{name="shuffleService",} 0.31428910657834713 hadoop_nodemanager_blocktransferratebytes_rate15{name="shuffleService",} 566144.9983653595 hadoop_nodemanager_blocktransferratebytes_ratemean{name="shuffleService",} 2464409.9678099006 hadoop_nodemanager_openblockrequestlatencymillis_rate1{name="shuffleService",} 1.2893844732240272 hadoop_nodemanager_registeredexecutorssize{name="shuffleService",} 2.0 hadoop_nodemanager_openblockrequestlatencymillis_ratemean{name="shuffleService",} 1.255574678369966 hadoop_nodemanager_openblockrequestlatencymillis_count{name="shuffleService",} 315.0 hadoop_nodemanager_openblockrequestlatencymillis_rate5{name="shuffleService",} 0.7661929192569739 hadoop_nodemanager_registerexecutorrequestlatencymillis_ratemean{name="shuffleService",} 0.0 hadoop_nodemanager_registerexecutorrequestlatencymillis_count{name="shuffleService",} 0.0 hadoop_nodemanager_registerexecutorrequestlatencymillis_rate1{name="shuffleService",} 0.0 hadoop_nodemanager_registerexecutorrequestlatencymillis_rate5{name="shuffleService",} 0.0 hadoop_nodemanager_blocktransferratebytes_count{name="shuffleService",} 6.18271213E8 hadoop_nodemanager_registerexecutorrequestlatencymillis_rate15{name="shuffleService",} 0.0 hadoop_nodemanager_blocktransferratebytes_rate5{name="shuffleService",} 1154114.4881816586 hadoop_nodemanager_blocktransferratebytes_rate1{name="shuffleService",} 574745.0749848988 ``` Closes #22485 from mareksimunek/SPARK-18364. Lead-authored-by: marek.simunek Co-authored-by: Andrew Ash Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a802c69b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a802c69b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a802c69b Branch: refs/heads/master Commit: a802c69b130b69a35b372ffe1b01289577f6fafb Parents: b96fd44 Author: marek.simunek Authored: Mon Oct 1 11:04:37 2018 -0500 Committer: Thomas Graves Committed: Mon Oct 1 11:04:37 2018 -0500 -- .../spark/network/yarn/YarnShuffleService.java | 11 ++ .../network/yarn/YarnShuffleServiceMetrics.java | 137 +++ .../yarn/YarnShuffleServiceMetricsSuite.scala | 73 ++ 3 files changed, 221 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a802c69b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java -- diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java index d8b2ed6..72ae1a1 100644 --- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java +++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java @@ -35,6 +35,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.api.*; import org.apache.spark.network.util.LevelDBProvider; @@ -168,6 +170,15 @@ public class YarnShuffleService extends AuxiliaryService { TransportConf transportConf = new TransportConf("shuffle", new HadoopConfigProvider(conf)); blockHandler = new ExternalShuffleBlockHandler(transportConf, registeredExecutorFile); + // register metrics on the block handler into the Node Manager's metrics system. + YarnShuffleServiceMetrics serviceMetrics = +new YarnShuffleServiceMetrics(blockHandler.getAllMetrics()); + + MetricsSystemImpl metricsSy
spark git commit: [SPARK-25641] Change the spark.shuffle.server.chunkFetchHandlerThreadsPercent default to 100
Repository: spark Updated Branches: refs/heads/master 1a2862535 -> 6353425af [SPARK-25641] Change the spark.shuffle.server.chunkFetchHandlerThreadsPercent default to 100 ## What changes were proposed in this pull request? We want to change the default percentage to 100 for spark.shuffle.server.chunkFetchHandlerThreadsPercent. The reason being currently this is set to 0. Which means currently if server.ioThreads > 0, the default number of threads would be 2 * #cores instead of server.io.Threads. We want the default to server.io.Threads in case this is not set at all. Also here a default of 0 would also mean 2 * #cores ## How was this patch tested? Manual (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) Please review http://spark.apache.org/contributing.html before opening a pull request. Closes #22628 from redsanket/SPARK-25641. Lead-authored-by: Sanket Chintapalli Co-authored-by: Sanket Chintapalli Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6353425a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6353425a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6353425a Branch: refs/heads/master Commit: 6353425af76f9cc9de7ee4094f41df7a7390d898 Parents: 1a28625 Author: Sanket Chintapalli Authored: Mon Oct 8 13:19:34 2018 -0500 Committer: Thomas Graves Committed: Mon Oct 8 13:19:34 2018 -0500 -- .../apache/spark/network/util/TransportConf.java| 16 ++-- 1 file changed, 10 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6353425a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java -- diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 6d5cccd..43a6bc7 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -296,17 +296,21 @@ public class TransportConf { * and could take long time to process due to disk contentions. By configuring a slightly * higher number of shuffler server threads, we are able to reserve some threads for * handling other RPC messages, thus making the Client less likely to experience timeout - * when sending RPC messages to the shuffle server. Default to 0, which is 2*#cores - * or io.serverThreads. 90 would mean 90% of 2*#cores or 90% of io.serverThreads - * which equals 0.9 * 2*#cores or 0.9 * io.serverThreads. + * when sending RPC messages to the shuffle server. The number of threads used for handling + * chunked fetch requests are percentage of io.serverThreads (if defined) else it is a percentage + * of 2 * #cores. However, a percentage of 0 means netty default number of threads which + * is 2 * #cores ignoring io.serverThreads. The percentage here is configured via + * spark.shuffle.server.chunkFetchHandlerThreadsPercent. The returned value is rounded off to + * ceiling of the nearest integer. */ public int chunkFetchHandlerThreads() { if (!this.getModuleName().equalsIgnoreCase("shuffle")) { return 0; } int chunkFetchHandlerThreadsPercent = - conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 0); -return this.serverThreads() > 0 ? (this.serverThreads() * chunkFetchHandlerThreadsPercent)/100: - (2 * NettyRuntime.availableProcessors() * chunkFetchHandlerThreadsPercent)/100; + conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 100); +return (int)Math.ceil( + (this.serverThreads() > 0 ? this.serverThreads() : 2 * NettyRuntime.availableProcessors()) * + chunkFetchHandlerThreadsPercent/(double)100); } } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24851][UI] Map a Stage ID to it's Associated Job ID
Repository: spark Updated Branches: refs/heads/master e3133f4ab -> deb9588b2 [SPARK-24851][UI] Map a Stage ID to it's Associated Job ID It would be nice to have a field in Stage Page UI which would show mapping of the current stage id to the job id's to which that stage belongs to. ## What changes were proposed in this pull request? Added a field in Stage UI to display the corresponding job id for that particular stage. ## How was this patch tested? https://user-images.githubusercontent.com/8190/43220447-a8e94f80-900f-11e8-8a20-a235bbd5a369.png";> Closes #21809 from pgandhi999/SPARK-24851. Authored-by: pgandhi Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/deb9588b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/deb9588b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/deb9588b Branch: refs/heads/master Commit: deb9588b2ab6596b30ab17f56c59951cabf57162 Parents: e3133f4 Author: pgandhi Authored: Tue Oct 9 08:59:21 2018 -0500 Committer: Thomas Graves Committed: Tue Oct 9 08:59:21 2018 -0500 -- .../scala/org/apache/spark/status/AppStatusStore.scala | 8 +--- .../apache/spark/status/api/v1/StagesResource.scala| 2 +- .../scala/org/apache/spark/ui/jobs/StagePage.scala | 13 +++-- 3 files changed, 17 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/deb9588b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala -- diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala index e237281..9839cbb 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala @@ -112,10 +112,12 @@ private[spark] class AppStatusStore( } } - def stageAttempt(stageId: Int, stageAttemptId: Int, details: Boolean = false): v1.StageData = { + def stageAttempt(stageId: Int, stageAttemptId: Int, + details: Boolean = false): (v1.StageData, Seq[Int]) = { val stageKey = Array(stageId, stageAttemptId) -val stage = store.read(classOf[StageDataWrapper], stageKey).info -if (details) stageWithDetails(stage) else stage +val stageDataWrapper = store.read(classOf[StageDataWrapper], stageKey) +val stage = if (details) stageWithDetails(stageDataWrapper.info) else stageDataWrapper.info +(stage, stageDataWrapper.jobIds.toSeq) } def taskCount(stageId: Int, stageAttemptId: Int): Long = { http://git-wip-us.apache.org/repos/asf/spark/blob/deb9588b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala -- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala index 96249e4..30d52b9 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala @@ -56,7 +56,7 @@ private[v1] class StagesResource extends BaseAppResource { @PathParam("stageAttemptId") stageAttemptId: Int, @QueryParam("details") @DefaultValue("true") details: Boolean): StageData = withUI { ui => try { - ui.store.stageAttempt(stageId, stageAttemptId, details = details) + ui.store.stageAttempt(stageId, stageAttemptId, details = details)._1 } catch { case _: NoSuchElementException => // Change the message depending on whether there are any attempts for the requested stage. http://git-wip-us.apache.org/repos/asf/spark/blob/deb9588b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 7428bbe..0f74b07 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -105,7 +105,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We val stageAttemptId = parameterAttempt.toInt val stageHeader = s"Details for Stage $stageId (Attempt $stageAttemptId)" -val stageData = parent.store +val (stageData, stageJobIds) = parent.store .asOption(parent.store.stageAttempt(stageId, stageAttemptId, details = false)) .getOrElse { val content = @@ -183,6 +183,15 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We {Utils.bytesToString(stageDat
spark git commit: [SPARK-25753][CORE] fix reading small files via BinaryFileRDD
Repository: spark Updated Branches: refs/heads/master fc64e83f9 -> 81a305dd0 [SPARK-25753][CORE] fix reading small files via BinaryFileRDD ## What changes were proposed in this pull request? This is a follow up of #21601, `StreamFileInputFormat` and `WholeTextFileInputFormat` have the same problem. `Minimum split size pernode 5123456 cannot be larger than maximum split size 4194304 java.io.IOException: Minimum split size pernode 5123456 cannot be larger than maximum split size 4194304 at org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.getSplits(CombineFileInputFormat.java: 201) at org.apache.spark.rdd.BinaryFileRDD.getPartitions(BinaryFileRDD.scala:52) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:254) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:252) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2138)` ## How was this patch tested? Added a unit test Closes #22725 from 10110346/maxSplitSize_node_rack. Authored-by: liuxian Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/81a305dd Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/81a305dd Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/81a305dd Branch: refs/heads/master Commit: 81a305dd0418f6e0136b4e38ffe91e0b76c8806e Parents: fc64e83 Author: liuxian Authored: Mon Oct 22 08:53:18 2018 -0500 Committer: Thomas Graves Committed: Mon Oct 22 08:53:18 2018 -0500 -- .../org/apache/spark/input/PortableDataStream.scala| 12 core/src/test/scala/org/apache/spark/FileSuite.scala | 13 + 2 files changed, 25 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/81a305dd/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala -- diff --git a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala index ab020aa..5b33c11 100644 --- a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala +++ b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala @@ -52,6 +52,18 @@ private[spark] abstract class StreamFileInputFormat[T] val totalBytes = files.filterNot(_.isDirectory).map(_.getLen + openCostInBytes).sum val bytesPerCore = totalBytes / defaultParallelism val maxSplitSize = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore)) + +// For small files we need to ensure the min split size per node & rack <= maxSplitSize +val jobConfig = context.getConfiguration +val minSplitSizePerNode = jobConfig.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERNODE, 0L) +val minSplitSizePerRack = jobConfig.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERRACK, 0L) + +if (maxSplitSize < minSplitSizePerNode) { + super.setMinSplitSizeNode(maxSplitSize) +} +if (maxSplitSize < minSplitSizePerRack) { + super.setMinSplitSizeRack(maxSplitSize) +} super.setMaxSplitSize(maxSplitSize) } http://git-wip-us.apache.org/repos/asf/spark/blob/81a305dd/core/src/test/scala/org/apache/spark/FileSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/FileSuite.scala b/core/src/test/scala/org/apache/spark/FileSuite.scala index 81b18c7..34efcdf 100644 --- a/core/src/test/scala/org/apache/spark/FileSuite.scala +++ b/core/src/test/scala/org/apache/spark/FileSuite.scala @@ -320,6 +320,19 @@ class FileSuite extends SparkFunSuite with LocalSparkContext { } } + test("minimum split size per node and per rack should be less than or equal to maxSplitSize") { +sc = new SparkContext("local", "test") +val testOutput = Array[Byte](1, 2, 3, 4, 5) +val outFile = writeBinaryData(testOutput, 1) +sc.hadoopConfiguration.setLong( + "mapreduce.input.fileinputformat.split.minsize.per.node", 5123456) +sc.hadoopConfiguration.setLong( + "mapreduce.input.fileinputformat.split.minsize.per.rack", 5123456) + +val (_, data) = sc.binaryFiles(outFile.getAbsolutePath).collect().head +assert(data.toArray === testOutput) + } + test("fixed record length binary file as byte array") { sc = new SparkContext("local", "test") val testOutput = Array[Byte](1, 2, 3, 4, 5, 6) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-21501] Change CacheLoader to limit entries based on memory footprint
Repository: spark Updated Branches: refs/heads/master d6b30edd4 -> 1662e9311 [SPARK-21501] Change CacheLoader to limit entries based on memory footprint Right now the spark shuffle service has a cache for index files. It is based on a # of files cached (spark.shuffle.service.index.cache.entries). This can cause issues if people have a lot of reducers because the size of each entry can fluctuate based on the # of reducers. We saw an issues with a job that had 17 reducers and it caused NM with spark shuffle service to use 700-800MB or memory in NM by itself. We should change this cache to be memory based and only allow a certain memory size used. When I say memory based I mean the cache should have a limit of say 100MB. https://issues.apache.org/jira/browse/SPARK-21501 Manual Testing with 17 reducers has been performed with cache loaded up to max 100MB default limit, with each shuffle index file of size 1.3MB. Eviction takes place as soon as the total cache size reaches the 100MB limit and the objects will be ready for garbage collection there by avoiding NM to crash. No notable difference in runtime has been observed. Author: Sanket Chintapalli Closes #18940 from redsanket/SPARK-21501. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1662e931 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1662e931 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1662e931 Branch: refs/heads/master Commit: 1662e93119d68498942386906de309d35f4a135f Parents: d6b30ed Author: Sanket Chintapalli Authored: Wed Aug 23 11:51:11 2017 -0500 Committer: Tom Graves Committed: Wed Aug 23 11:51:11 2017 -0500 -- .../org/apache/spark/network/util/TransportConf.java | 4 .../network/shuffle/ExternalShuffleBlockResolver.java| 11 +-- .../spark/network/shuffle/ShuffleIndexInformation.java | 11 ++- core/src/main/scala/org/apache/spark/SparkConf.scala | 4 +++- docs/configuration.md| 6 +++--- 5 files changed, 29 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1662e931/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java -- diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 88256b8..fa2ff42 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -67,6 +67,10 @@ public class TransportConf { return conf.getInt(name, defaultValue); } + public String get(String name, String defaultValue) { +return conf.get(name, defaultValue); + } + private String getConfKey(String suffix) { return "spark." + module + "." + suffix; } http://git-wip-us.apache.org/repos/asf/spark/blob/1662e931/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java -- diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java index d7ec0e2..e639989 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java @@ -33,6 +33,7 @@ import com.google.common.base.Objects; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.google.common.cache.Weigher; import com.google.common.collect.Maps; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBIterator; @@ -104,7 +105,7 @@ public class ExternalShuffleBlockResolver { Executor directoryCleaner) throws IOException { this.conf = conf; this.registeredExecutorFile = registeredExecutorFile; -int indexCacheEntries = conf.getInt("spark.shuffle.service.index.cache.entries", 1024); +String indexCacheSize = conf.get("spark.shuffle.service.index.cache.size", "100m"); CacheLoader indexCacheLoader = new CacheLoader() { public ShuffleIndexInformation load(File file) throws IOException { @@ -112,7 +113,13 @@ public class ExternalShuffleBlockResolver { } }; shuffleIndexCache = CacheBuilder.newBuilder() - .maximumSize(indexCacheEntries).build(indexCac
spark git commit: [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server
Repository: spark Updated Branches: refs/heads/master 0456b4050 -> 24e6c187f [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server History Server Launch uses SparkClassCommandBuilder for launching the server. It is observed that SPARK_CLASSPATH has been removed and deprecated. For spark-submit this takes a different route and spark.driver.extraClasspath takes care of specifying additional jars in the classpath that were previously specified in the SPARK_CLASSPATH. Right now the only way specify the additional jars for launching daemons such as history server is using SPARK_DIST_CLASSPATH (https://spark.apache.org/docs/latest/hadoop-provided.html) but this I presume is a distribution classpath. It would be nice to have a similar config like spark.driver.extraClasspath for launching daemons similar to history server. Added new environment variable SPARK_DAEMON_CLASSPATH to set classpath for launching daemons. Tested and verified for History Server and Standalone Mode. ## How was this patch tested? Initially, history server start script would fail for the reason being that it could not find the required jars for launching the server in the java classpath. Same was true for running Master and Worker in standalone mode. By adding the environment variable SPARK_DAEMON_CLASSPATH to the java classpath, both the daemons(History Server, Standalone daemons) are starting up and running. Author: pgandhi Author: pgandhi999 Closes #19047 from pgandhi999/master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/24e6c187 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/24e6c187 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/24e6c187 Branch: refs/heads/master Commit: 24e6c187fbaa6874eedbdda6b3b5dc6ff9e1de36 Parents: 0456b40 Author: pgandhi Authored: Mon Aug 28 08:51:22 2017 -0500 Committer: Tom Graves Committed: Mon Aug 28 08:51:22 2017 -0500 -- conf/spark-env.sh.template | 1 + docs/monitoring.md | 4 docs/running-on-mesos.md| 2 ++ docs/spark-standalone.md| 4 .../org/apache/spark/launcher/SparkClassCommandBuilder.java | 5 + 5 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/24e6c187/conf/spark-env.sh.template -- diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index 1663019..f8c895f 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -52,6 +52,7 @@ # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") +# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers # Generic options for the daemons used in the standalone deploy mode http://git-wip-us.apache.org/repos/asf/spark/blob/24e6c187/docs/monitoring.md -- diff --git a/docs/monitoring.md b/docs/monitoring.md index 3e577c5..d22cd94 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -62,6 +62,10 @@ The history server can be configured as follows: JVM options for the history server (default: none). +SPARK_DAEMON_CLASSPATH +Classpath for the history server (default: none). + + SPARK_PUBLIC_DNS The public address for the history server. If this is not set, links to application history http://git-wip-us.apache.org/repos/asf/spark/blob/24e6c187/docs/running-on-mesos.md -- diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 0e5a20c..c12b858 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -160,6 +160,8 @@ If you like to run the `MesosClusterDispatcher` with Marathon, you need to run t The `MesosClusterDispatcher` also supports writing recovery state into Zookeeper. This will allow the `MesosClusterDispatcher` to be able to recover all submitted and running containers on relaunch. In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spark-env by configuring `spark.deploy.recoveryMode` and related spark.deploy.zookeeper.* configurations. For more information about these configurations please refer to the configurations [doc](configurations.html#deploy). +You can also specify any
spark git commit: [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server
Repository: spark Updated Branches: refs/heads/branch-2.2 0d4ef2f69 -> 59bb7ebfb [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server History Server Launch uses SparkClassCommandBuilder for launching the server. It is observed that SPARK_CLASSPATH has been removed and deprecated. For spark-submit this takes a different route and spark.driver.extraClasspath takes care of specifying additional jars in the classpath that were previously specified in the SPARK_CLASSPATH. Right now the only way specify the additional jars for launching daemons such as history server is using SPARK_DIST_CLASSPATH (https://spark.apache.org/docs/latest/hadoop-provided.html) but this I presume is a distribution classpath. It would be nice to have a similar config like spark.driver.extraClasspath for launching daemons similar to history server. Added new environment variable SPARK_DAEMON_CLASSPATH to set classpath for launching daemons. Tested and verified for History Server and Standalone Mode. ## How was this patch tested? Initially, history server start script would fail for the reason being that it could not find the required jars for launching the server in the java classpath. Same was true for running Master and Worker in standalone mode. By adding the environment variable SPARK_DAEMON_CLASSPATH to the java classpath, both the daemons(History Server, Standalone daemons) are starting up and running. Author: pgandhi Author: pgandhi999 Closes #19047 from pgandhi999/master. (cherry picked from commit 24e6c187fbaa6874eedbdda6b3b5dc6ff9e1de36) Signed-off-by: Tom Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/59bb7ebf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/59bb7ebf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/59bb7ebf Branch: refs/heads/branch-2.2 Commit: 59bb7ebfb83c292cea853d6cd6fdf9748baa6ce2 Parents: 0d4ef2f Author: pgandhi Authored: Mon Aug 28 08:51:22 2017 -0500 Committer: Tom Graves Committed: Mon Aug 28 08:51:49 2017 -0500 -- conf/spark-env.sh.template | 1 + docs/monitoring.md | 4 docs/running-on-mesos.md| 2 ++ docs/spark-standalone.md| 4 .../org/apache/spark/launcher/SparkClassCommandBuilder.java | 5 + 5 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/59bb7ebf/conf/spark-env.sh.template -- diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index b7c985a..0f9150b 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -51,6 +51,7 @@ # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") +# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers # Generic options for the daemons used in the standalone deploy mode http://git-wip-us.apache.org/repos/asf/spark/blob/59bb7ebf/docs/monitoring.md -- diff --git a/docs/monitoring.md b/docs/monitoring.md index 3e577c5..d22cd94 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -62,6 +62,10 @@ The history server can be configured as follows: JVM options for the history server (default: none). +SPARK_DAEMON_CLASSPATH +Classpath for the history server (default: none). + + SPARK_PUBLIC_DNS The public address for the history server. If this is not set, links to application history http://git-wip-us.apache.org/repos/asf/spark/blob/59bb7ebf/docs/running-on-mesos.md -- diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 847a659..6b69bfc 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -158,6 +158,8 @@ If you like to run the `MesosClusterDispatcher` with Marathon, you need to run t The `MesosClusterDispatcher` also supports writing recovery state into Zookeeper. This will allow the `MesosClusterDispatcher` to be able to recover all submitted and running containers on relaunch. In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spark-env by configuring `spark.deploy.recoveryMode` and related spark.deploy.zookeeper.* configurations. For more information about these configur
spark git commit: [SPARK-21890] Credentials not being passed to add the tokens
Repository: spark Updated Branches: refs/heads/master eea2b877c -> b9ab791a9 [SPARK-21890] Credentials not being passed to add the tokens I observed this while running a oozie job trying to connect to hbase via spark. It look like the creds are not being passed in thehttps://github.com/apache/spark/blob/branch-2.2/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/HadoopFSCredentialProvider.scala#L53 for 2.2 release. More Info as to why it fails on secure grid: Oozie client gets the necessary tokens the application needs before launching. It passes those tokens along to the oozie launcher job (MR job) which will then actually call the Spark client to launch the spark app and pass the tokens along. The oozie launcher job cannot get anymore tokens because all it has is tokens ( you can't get tokens with tokens, you need tgt or keytab). The error here is because the launcher job runs the Spark Client to submit the spark job but the spark client doesn't see that it already has the hdfs tokens so it tries to get more, which ends with the exception. There was a change with SPARK-19021 to generalize the hdfs credentials provider that changed it so we don't pass the existing credentials into the call to get tokens so it doesn't realize it already has the necessary tokens. https://issues.apache.org/jira/browse/SPARK-21890 Modified to pass creds to get delegation tokens Author: Sanket Chintapalli Closes #19140 from redsanket/SPARK-21890-master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b9ab791a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b9ab791a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b9ab791a Branch: refs/heads/master Commit: b9ab791a9efb0dc165ba283c91acf831fa6be5d8 Parents: eea2b87 Author: Sanket Chintapalli Authored: Thu Sep 7 11:25:24 2017 -0500 Committer: Tom Graves Committed: Thu Sep 7 11:25:24 2017 -0500 -- .../security/HBaseDelegationTokenProvider.scala | 2 + .../security/HadoopDelegationTokenManager.scala | 2 +- .../HadoopDelegationTokenProvider.scala | 3 ++ .../HadoopFSDelegationTokenProvider.scala | 50 ++-- .../security/HiveDelegationTokenProvider.scala | 2 + .../HadoopDelegationTokenManagerSuite.scala | 4 +- 6 files changed, 35 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b9ab791a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala index 35621da..78b0e6b 100644 --- a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.security.Credentials import org.apache.hadoop.security.token.{Token, TokenIdentifier} +import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.util.Utils @@ -34,6 +35,7 @@ private[security] class HBaseDelegationTokenProvider override def obtainDelegationTokens( hadoopConf: Configuration, + sparkConf: SparkConf, creds: Credentials): Option[Long] = { try { val mirror = universe.runtimeMirror(Utils.getContextOrSparkClassLoader) http://git-wip-us.apache.org/repos/asf/spark/blob/b9ab791a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala index c317c4f..c134b7e 100644 --- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala +++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala @@ -116,7 +116,7 @@ private[spark] class HadoopDelegationTokenManager( creds: Credentials): Long = { delegationTokenProviders.values.flatMap { provider => if (provider.delegationTokensRequired(hadoopConf)) { -provider.obtainDelegationTokens(hadoopConf, creds) +provider.obtainDelegationTokens(hadoopConf, sparkConf, creds) } else { logDebug(s"Service ${provider.serviceName} does not require a token." + s" Check your configuration to see if security is disabled or not.") http://git-wip-us.apache.org/repos/asf/spark/blob/b9
spark git commit: [SPARK-22683][CORE] Add a executorAllocationRatio parameter to throttle the parallelism of the dynamic allocation
Repository: spark Updated Branches: refs/heads/master 4926a7c2f -> 55c4ca88a [SPARK-22683][CORE] Add a executorAllocationRatio parameter to throttle the parallelism of the dynamic allocation ## What changes were proposed in this pull request? By default, the dynamic allocation will request enough executors to maximize the parallelism according to the number of tasks to process. While this minimizes the latency of the job, with small tasks this setting can waste a lot of resources due to executor allocation overhead, as some executor might not even do any work. This setting allows to set a ratio that will be used to reduce the number of target executors w.r.t. full parallelism. The number of executors computed with this setting is still fenced by `spark.dynamicAllocation.maxExecutors` and `spark.dynamicAllocation.minExecutors` ## How was this patch tested? Units tests and runs on various actual workloads on a Yarn Cluster Author: Julien Cuquemelle Closes #19881 from jcuquemelle/AddTaskPerExecutorSlot. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/55c4ca88 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/55c4ca88 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/55c4ca88 Branch: refs/heads/master Commit: 55c4ca88a3b093ee197a8689631be8d1fac1f10f Parents: 4926a7c Author: Julien Cuquemelle Authored: Tue Apr 24 10:56:55 2018 -0500 Committer: Thomas Graves Committed: Tue Apr 24 10:56:55 2018 -0500 -- .../spark/ExecutorAllocationManager.scala | 24 +++--- .../apache/spark/internal/config/package.scala | 4 +++ .../spark/ExecutorAllocationManagerSuite.scala | 33 docs/configuration.md | 18 +++ 4 files changed, 74 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/55c4ca88/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala index 189d913..aa363ee 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala @@ -26,7 +26,7 @@ import scala.util.control.{ControlThrowable, NonFatal} import com.codahale.metrics.{Gauge, MetricRegistry} import org.apache.spark.internal.Logging -import org.apache.spark.internal.config.{DYN_ALLOCATION_MAX_EXECUTORS, DYN_ALLOCATION_MIN_EXECUTORS} +import org.apache.spark.internal.config._ import org.apache.spark.metrics.source.Source import org.apache.spark.scheduler._ import org.apache.spark.storage.BlockManagerMaster @@ -69,6 +69,10 @@ import org.apache.spark.util.{Clock, SystemClock, ThreadUtils, Utils} * spark.dynamicAllocation.maxExecutors - Upper bound on the number of executors * spark.dynamicAllocation.initialExecutors - Number of executors to start with * + * spark.dynamicAllocation.executorAllocationRatio - + * This is used to reduce the parallelism of the dynamic allocation that can waste + * resources when tasks are small + * * spark.dynamicAllocation.schedulerBacklogTimeout (M) - * If there are backlogged tasks for this duration, add new executors * @@ -116,9 +120,12 @@ private[spark] class ExecutorAllocationManager( // TODO: The default value of 1 for spark.executor.cores works right now because dynamic // allocation is only supported for YARN and the default number of cores per executor in YARN is // 1, but it might need to be attained differently for different cluster managers - private val tasksPerExecutor = + private val tasksPerExecutorForFullParallelism = conf.getInt("spark.executor.cores", 1) / conf.getInt("spark.task.cpus", 1) + private val executorAllocationRatio = +conf.get(DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO) + validateSettings() // Number of executors to add in the next round @@ -209,8 +216,13 @@ private[spark] class ExecutorAllocationManager( throw new SparkException("Dynamic allocation of executors requires the external " + "shuffle service. You may enable this through spark.shuffle.service.enabled.") } -if (tasksPerExecutor == 0) { - throw new SparkException("spark.executor.cores must not be less than spark.task.cpus.") +if (tasksPerExecutorForFullParallelism == 0) { + throw new SparkException("spark.executor.cores must not be < spark.task.cpus.") +} + +if (executorAllocationRatio > 1.0 || executorAllocationRatio <= 0.0) { + throw new SparkException( +"spark.dynamicAllocation.executorAllocationRatio must be > 0 and <= 1.0") } } @@ -2
git commit: SPARK-1291: Link the spark UI to RM ui in yarn-client mode
Repository: spark Updated Branches: refs/heads/master 9dd635eb5 -> 72ea56da8 SPARK-1291: Link the spark UI to RM ui in yarn-client mode Author: witgo Closes #1112 from witgo/SPARK-1291 and squashes the following commits: 6022bcd [witgo] review commit 1fbb925 [witgo] add addAmIpFilter to yarn alpha 210299c [witgo] review commit 1b92a07 [witgo] review commit 6896586 [witgo] Add comments to addWebUIFilter 3e9630b [witgo] review commit 142ee29 [witgo] review commit 1fe7710 [witgo] Link the spark UI to RM ui in yarn-client mode Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/72ea56da Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/72ea56da Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/72ea56da Branch: refs/heads/master Commit: 72ea56da8e383c61c6f18eeefef03b9af00f5158 Parents: 9dd635e Author: witgo Authored: Tue Jul 15 13:52:56 2014 -0500 Committer: Thomas Graves Committed: Tue Jul 15 13:52:56 2014 -0500 -- .../cluster/CoarseGrainedClusterMessage.scala | 3 +++ .../cluster/CoarseGrainedSchedulerBackend.scala | 18 +++ .../scala/org/apache/spark/ui/UIUtils.scala | 11 +- .../spark/deploy/yarn/ExecutorLauncher.scala| 22 --- .../cluster/YarnClientSchedulerBackend.scala| 1 + .../spark/deploy/yarn/ExecutorLauncher.scala| 23 +--- 6 files changed, 71 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/72ea56da/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala index 318e165..6abf6d9 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala @@ -66,4 +66,7 @@ private[spark] object CoarseGrainedClusterMessages { case class RemoveExecutor(executorId: String, reason: String) extends CoarseGrainedClusterMessage + case class AddWebUIFilter(filterName:String, filterParams: String, proxyBase :String) +extends CoarseGrainedClusterMessage + } http://git-wip-us.apache.org/repos/asf/spark/blob/72ea56da/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 0f5545e..9f085ee 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -31,6 +31,7 @@ import org.apache.spark.{SparkEnv, Logging, SparkException, TaskState} import org.apache.spark.scheduler.{SchedulerBackend, SlaveLost, TaskDescription, TaskSchedulerImpl, WorkerOffer} import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._ import org.apache.spark.util.{SerializableBuffer, AkkaUtils, Utils} +import org.apache.spark.ui.JettyUtils /** * A scheduler backend that waits for coarse grained executors to connect to it through Akka. @@ -136,6 +137,9 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, actorSystem: A removeExecutor(executorId, reason) sender ! true + case AddWebUIFilter(filterName, filterParams, proxyBase) => +addWebUIFilter(filterName, filterParams, proxyBase) +sender ! true case DisassociatedEvent(_, address, _) => addressToExecutorId.get(address).foreach(removeExecutor(_, "remote Akka client disassociated")) @@ -276,6 +280,20 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, actorSystem: A } false } + + // Add filters to the SparkUI + def addWebUIFilter(filterName: String, filterParams: String, proxyBase: String) { +if (proxyBase != null && proxyBase.nonEmpty) { + System.setProperty("spark.ui.proxyBase", proxyBase) +} + +if (Seq(filterName, filterParams).forall(t => t != null && t.nonEmpty)) { + logInfo(s"Add WebUI Filter. $filterName, $filterParams, $proxyBase") + conf.set("spark.ui.filters", filterName) + conf.set(s"spark.$filterName.params", filterParams) + JettyUtils.addFilters(scheduler.sc.ui.getHandlers, conf) +} + } } private[spark] object CoarseGrainedSchedulerBackend { http://git-wip-us.apache.org/repos/asf/spark/blob/
git commit: SPARK-1707. Remove unnecessary 3 second sleep in YarnClusterScheduler
Repository: spark Updated Branches: refs/heads/master cd273a238 -> f89cf65d7 SPARK-1707. Remove unnecessary 3 second sleep in YarnClusterScheduler Author: Sandy Ryza Closes #634 from sryza/sandy-spark-1707 and squashes the following commits: 2f6e358 [Sandy Ryza] Default min registered executors ratio to .8 for YARN 354c630 [Sandy Ryza] Remove outdated comments c744ef3 [Sandy Ryza] Take out waitForInitialAllocations 2a4329b [Sandy Ryza] SPARK-1707. Remove unnecessary 3 second sleep in YarnClusterScheduler Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f89cf65d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f89cf65d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f89cf65d Branch: refs/heads/master Commit: f89cf65d7aced0bb387c05586f9f51cb29865022 Parents: cd273a2 Author: Sandy Ryza Authored: Mon Jul 21 13:15:46 2014 -0500 Committer: Thomas Graves Committed: Mon Jul 21 13:15:46 2014 -0500 -- .../spark/deploy/yarn/ApplicationMaster.scala | 39 -- .../cluster/YarnClientClusterScheduler.scala| 10 - .../cluster/YarnClientSchedulerBackend.scala| 5 +++ .../cluster/YarnClusterScheduler.scala | 8 +--- .../cluster/YarnClusterSchedulerBackend.scala | 5 +++ .../spark/deploy/yarn/ApplicationMaster.scala | 43 6 files changed, 11 insertions(+), 99 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f89cf65d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 062f946..3ec3648 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -255,10 +255,6 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, sparkContext.getConf) } } -} finally { - // in case of exceptions, etc - ensure that count is atleast ALLOCATOR_LOOP_WAIT_COUNT : - // so that the loop (in ApplicationMaster.sparkContextInitialized) breaks - ApplicationMaster.incrementAllocatorLoop(ApplicationMaster.ALLOCATOR_LOOP_WAIT_COUNT) } } @@ -277,13 +273,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, } yarnAllocator.allocateContainers( math.max(args.numExecutors - yarnAllocator.getNumExecutorsRunning, 0)) -ApplicationMaster.incrementAllocatorLoop(1) Thread.sleep(ApplicationMaster.ALLOCATE_HEARTBEAT_INTERVAL) } -} finally { - // In case of exceptions, etc - ensure that count is at least ALLOCATOR_LOOP_WAIT_COUNT, - // so that the loop in ApplicationMaster#sparkContextInitialized() breaks. - ApplicationMaster.incrementAllocatorLoop(ApplicationMaster.ALLOCATOR_LOOP_WAIT_COUNT) } logInfo("All executors have launched.") @@ -411,24 +402,10 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, } object ApplicationMaster extends Logging { - // Number of times to wait for the allocator loop to complete. - // Each loop iteration waits for 100ms, so maximum of 3 seconds. - // This is to ensure that we have reasonable number of containers before we start // TODO: Currently, task to container is computed once (TaskSetManager) - which need not be // optimal as more containers are available. Might need to handle this better. - private val ALLOCATOR_LOOP_WAIT_COUNT = 30 private val ALLOCATE_HEARTBEAT_INTERVAL = 100 - def incrementAllocatorLoop(by: Int) { -val count = yarnAllocatorLoop.getAndAdd(by) -if (count >= ALLOCATOR_LOOP_WAIT_COUNT) { - yarnAllocatorLoop.synchronized { -// to wake threads off wait ... -yarnAllocatorLoop.notifyAll() - } -} - } - private val applicationMasters = new CopyOnWriteArrayList[ApplicationMaster]() def register(master: ApplicationMaster) { @@ -437,7 +414,6 @@ object ApplicationMaster extends Logging { val sparkContextRef: AtomicReference[SparkContext] = new AtomicReference[SparkContext](null /* initialValue */) - val yarnAllocatorLoop: AtomicInteger = new AtomicInteger(0) def sparkContextInitialized(sc: SparkContext): Boolean = { var modified = false @@ -472,21 +448,6 @@ object ApplicationMaster extends Logging { modified } - - /** - * Returns when we've either - * 1) received all the requested executors, - * 2) waited ALLOCATOR_LOOP_WAIT_COUNT * ALLOCATE_HEARTBEAT_INTERVAL ms, - * 3) hi
git commit: [YARN][SPARK-2606]:In some cases, the spark UI pages display incorrect
Repository: spark Updated Branches: refs/heads/master 5f7b99168 -> ddadf1b00 [YARN][SPARK-2606]:In some cases,the spark UI pages display incorrect The issue is caused by #1112 . Author: GuoQiang Li Closes #1501 from witgo/webui_style and squashes the following commits: 4b34998 [GuoQiang Li] In some cases, pages display incorrect in WebUI Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ddadf1b0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ddadf1b0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ddadf1b0 Branch: refs/heads/master Commit: ddadf1b00470b9d7bf7386dacf198d41407a0a2b Parents: 5f7b991 Author: GuoQiang Li Authored: Tue Jul 22 20:34:40 2014 -0500 Committer: Thomas Graves Committed: Tue Jul 22 20:34:40 2014 -0500 -- core/src/main/scala/org/apache/spark/ui/UIUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ddadf1b0/core/src/main/scala/org/apache/spark/ui/UIUtils.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index e07aa2e..715cc2f 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -149,7 +149,7 @@ private[spark] object UIUtils extends Logging { def prependBaseUri(basePath: String = "", resource: String = "") = uiRoot + basePath + resource - val commonHeaderNodes = { + def commonHeaderNodes = {
git commit: [YARN] SPARK-2577: File upload to viewfs is broken due to mount point re...
Repository: spark Updated Branches: refs/heads/master ddadf1b00 -> 02e457294 [YARN] SPARK-2577: File upload to viewfs is broken due to mount point re... Opting to the option 2 defined in SPARK-2577, i.e., retrieve and pass the correct file system object to addResource. Author: Gera Shegalov Closes #1483 from gerashegalov/master and squashes the following commits: 90c9087 [Gera Shegalov] [YARN] SPARK-2577: File upload to viewfs is broken due to mount point resolution Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/02e45729 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/02e45729 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/02e45729 Branch: refs/heads/master Commit: 02e45729472a22a31629cdd17dc3836ba6810189 Parents: ddadf1b Author: Gera Shegalov Authored: Tue Jul 22 21:05:12 2014 -0500 Committer: Thomas Graves Committed: Tue Jul 22 21:05:12 2014 -0500 -- .../src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/02e45729/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 556f493..a1298e8 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -232,7 +232,8 @@ trait ClientBase extends Logging { if (!ClientBase.LOCAL_SCHEME.equals(localURI.getScheme())) { val setPermissions = if (destName.equals(ClientBase.APP_JAR)) true else false val destPath = copyRemoteFile(dst, qualifyForLocal(localURI), replication, setPermissions) - distCacheMgr.addResource(fs, conf, destPath, localResources, LocalResourceType.FILE, + val destFs = FileSystem.get(destPath.toUri(), conf) + distCacheMgr.addResource(destFs, conf, destPath, localResources, LocalResourceType.FILE, destName, statCache) } else if (confKey != null) { sparkConf.set(confKey, localPath)
git commit: SPARK-2150: Provide direct link to finished application UI in yarn resou...
Repository: spark Updated Branches: refs/heads/master 42dfab7d3 -> 46e224aaa SPARK-2150: Provide direct link to finished application UI in yarn resou... ...rce manager UI Use the event logger directory to provide a direct link to finished application UI in yarn resourcemanager UI. Author: Rahul Singhal Closes #1094 from rahulsinghaliitd/SPARK-2150 and squashes the following commits: 95f230c [Rahul Singhal] SPARK-2150: Provide direct link to finished application UI in yarn resource manager UI Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/46e224aa Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/46e224aa Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/46e224aa Branch: refs/heads/master Commit: 46e224aaa26df4b232c5176e98472a902862b76c Parents: 42dfab7 Author: Rahul Singhal Authored: Thu Jul 24 09:31:04 2014 -0500 Committer: Thomas Graves Committed: Thu Jul 24 09:31:04 2014 -0500 -- .../spark/deploy/history/FsHistoryProvider.scala| 3 ++- .../apache/spark/deploy/history/HistoryPage.scala | 2 +- .../apache/spark/deploy/history/HistoryServer.scala | 4 +++- .../org/apache/spark/deploy/master/Master.scala | 11 +++ .../spark/scheduler/EventLoggingListener.scala | 7 +++ .../spark/deploy/yarn/ApplicationMaster.scala | 4 +++- .../apache/spark/deploy/yarn/ExecutorLauncher.scala | 2 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 16 .../cluster/YarnClientSchedulerBackend.scala| 3 ++- .../spark/deploy/yarn/ApplicationMaster.scala | 5 +++-- .../apache/spark/deploy/yarn/ExecutorLauncher.scala | 2 +- 11 files changed, 46 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/46e224aa/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index a8c9ac0..01e7065 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -169,7 +169,8 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis val ui: SparkUI = if (renderUI) { val conf = this.conf.clone() val appSecManager = new SecurityManager(conf) -new SparkUI(conf, appSecManager, replayBus, appId, "/history/" + appId) +new SparkUI(conf, appSecManager, replayBus, appId, + HistoryServer.UI_PATH_PREFIX + s"/$appId") // Do not call ui.bind() to avoid creating a new server for each application } else { null http://git-wip-us.apache.org/repos/asf/spark/blob/46e224aa/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala index a958c83..d7a3e3f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala @@ -75,7 +75,7 @@ private[spark] class HistoryPage(parent: HistoryServer) extends WebUIPage("") { "Last Updated") private def appRow(info: ApplicationHistoryInfo): Seq[Node] = { -val uiAddress = "/history/" + info.id +val uiAddress = HistoryServer.UI_PATH_PREFIX + s"/${info.id}" val startTime = UIUtils.formatDate(info.startTime) val endTime = UIUtils.formatDate(info.endTime) val duration = UIUtils.formatDuration(info.endTime - info.startTime) http://git-wip-us.apache.org/repos/asf/spark/blob/46e224aa/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index 56b38dd..cacb9da 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -114,7 +114,7 @@ class HistoryServer( attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static")) val contextHandler = new ServletContextHandler -contextHandler.setContextPath("/history") +contextHandler.setContextPath(HistoryServer.UI_PATH_PREFIX) contextHandler.addServlet(new ServletHolder(loaderServlet), "/*") attachHandler(contextHandler) } @@ -172,6 +17
git commit: [SPARK-2037]: yarn client mode doesn't support spark.yarn.max.executor.failures
Repository: spark Updated Branches: refs/heads/master c960b5051 -> 323a83c52 [SPARK-2037]: yarn client mode doesn't support spark.yarn.max.executor.failures Author: GuoQiang Li Closes #1180 from witgo/SPARK-2037 and squashes the following commits: 3d52411 [GuoQiang Li] review commit 7058f4d [GuoQiang Li] Correctly stop SparkContext 6d0561f [GuoQiang Li] Fix: yarn client mode doesn't support spark.yarn.max.executor.failures Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/323a83c5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/323a83c5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/323a83c5 Branch: refs/heads/master Commit: 323a83c5235f9289cd9526491d62365df96a429b Parents: c960b50 Author: GuoQiang Li Authored: Thu Jul 24 14:46:10 2014 -0500 Committer: Thomas Graves Committed: Thu Jul 24 14:46:10 2014 -0500 -- .../spark/deploy/yarn/ExecutorLauncher.scala| 80 +--- .../cluster/YarnClientSchedulerBackend.scala| 28 +++ .../spark/deploy/yarn/ExecutorLauncher.scala| 45 --- 3 files changed, 115 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/323a83c5/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala index d232c18..184e2ad 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{ConverterUtils, Records} import akka.actor._ import akka.remote._ -import akka.actor.Terminated import org.apache.spark.{Logging, SecurityManager, SparkConf} import org.apache.spark.util.{Utils, AkkaUtils} import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend @@ -57,10 +56,17 @@ class ExecutorLauncher(args: ApplicationMasterArguments, conf: Configuration, sp private val yarnConf: YarnConfiguration = new YarnConfiguration(conf) private var yarnAllocator: YarnAllocationHandler = _ - private var driverClosed:Boolean = false + + private var driverClosed: Boolean = false + private var isFinished: Boolean = false + private var registered: Boolean = false + + // Default to numExecutors * 2, with minimum of 3 + private val maxNumExecutorFailures = sparkConf.getInt("spark.yarn.max.executor.failures", +sparkConf.getInt("spark.yarn.max.worker.failures", math.max(args.numExecutors * 2, 3))) val securityManager = new SecurityManager(sparkConf) - val actorSystem : ActorSystem = AkkaUtils.createActorSystem("sparkYarnAM", Utils.localHostName, 0, + val actorSystem: ActorSystem = AkkaUtils.createActorSystem("sparkYarnAM", Utils.localHostName, 0, conf = sparkConf, securityManager = securityManager)._1 var actor: ActorRef = _ @@ -97,23 +103,26 @@ class ExecutorLauncher(args: ApplicationMasterArguments, conf: Configuration, sp appAttemptId = getApplicationAttemptId() resourceManager = registerWithResourceManager() -val appMasterResponse: RegisterApplicationMasterResponse = registerApplicationMaster() - -// Compute number of threads for akka -val minimumMemory = appMasterResponse.getMinimumResourceCapability().getMemory() - -if (minimumMemory > 0) { - val mem = args.executorMemory + sparkConf.getInt("spark.yarn.executor.memoryOverhead", -YarnAllocationHandler.MEMORY_OVERHEAD) - val numCore = (mem / minimumMemory) + (if (0 != (mem % minimumMemory)) 1 else 0) - - if (numCore > 0) { -// do not override - hits https://issues.apache.org/jira/browse/HADOOP-8406 -// TODO: Uncomment when hadoop is on a version which has this fixed. -// args.workerCores = numCore +synchronized { + if (!isFinished) { +val appMasterResponse: RegisterApplicationMasterResponse = registerApplicationMaster() +// Compute number of threads for akka +val minimumMemory = appMasterResponse.getMinimumResourceCapability().getMemory() + +if (minimumMemory > 0) { + val mem = args.executorMemory + sparkConf.getInt("spark.yarn.executor.memoryOverhead", +YarnAllocationHandler.MEMORY_OVERHEAD) + val numCore = (mem / minimumMemory) + (if (0 != (mem % minimumMemory)) 1 else 0) + + if (numCore > 0) { +// do not override - hits https://issues.apache.org/jira/browse/HADOOP-8406 +// TODO: Uncomment when hadoop is on a version which has this fixed. +// args
git commit: SPARK-1528 - spark on yarn, add support for accessing remote HDFS
Repository: spark Updated Branches: refs/heads/master e87075df9 -> 2c0f705e2 SPARK-1528 - spark on yarn, add support for accessing remote HDFS Add a config (spark.yarn.access.namenodes) to allow applications running on yarn to access other secure HDFS cluster. User just specifies the namenodes of the other clusters and we get Tokens for those and ship them with the spark application. Author: Thomas Graves Closes #1159 from tgravescs/spark-1528 and squashes the following commits: ddbcd16 [Thomas Graves] review comments 0ac8501 [Thomas Graves] SPARK-1528 - add support for accessing remote HDFS Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2c0f705e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2c0f705e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2c0f705e Branch: refs/heads/master Commit: 2c0f705e26ca3dfc43a1e9a0722c0e57f67c970a Parents: e87075d Author: Thomas Graves Authored: Tue Aug 5 12:48:26 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 12:48:26 2014 -0500 -- docs/running-on-yarn.md | 7 +++ .../apache/spark/deploy/yarn/ClientBase.scala | 56 ++-- .../spark/deploy/yarn/ClientBaseSuite.scala | 55 ++- 3 files changed, 101 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/2c0f705e/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 0362f5a..573930d 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -106,6 +106,13 @@ Most of the configs are the same for Spark on YARN as for other deployment modes set this configuration to "hdfs:///some/path". + + spark.yarn.access.namenodes + (none) + +A list of secure HDFS namenodes your Spark application is going to access. For example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The Spark application must have acess to the namenodes listed and Kerberos must be properly configured to be able to access them (either in the same realm or in a trusted realm). Spark acquires security tokens for each of the namenodes so that the Spark application can access those remote HDFS clusters. + + # Launching Spark on YARN http://git-wip-us.apache.org/repos/asf/spark/blob/2c0f705e/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index b7e8636..ed8f56a 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -29,7 +29,7 @@ import org.apache.hadoop.fs._ import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.mapred.Master import org.apache.hadoop.mapreduce.MRJobConfig -import org.apache.hadoop.security.UserGroupInformation +import org.apache.hadoop.security.{Credentials, UserGroupInformation} import org.apache.hadoop.util.StringUtils import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment @@ -191,23 +191,11 @@ trait ClientBase extends Logging { // Upload Spark and the application JAR to the remote file system if necessary. Add them as // local resources to the application master. val fs = FileSystem.get(conf) - -val delegTokenRenewer = Master.getMasterPrincipal(conf) -if (UserGroupInformation.isSecurityEnabled()) { - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { -val errorMessage = "Can't get Master Kerberos principal for use as renewer" -logError(errorMessage) -throw new SparkException(errorMessage) - } -} val dst = new Path(fs.getHomeDirectory(), appStagingDir) -val replication = sparkConf.getInt("spark.yarn.submit.file.replication", 3).toShort - -if (UserGroupInformation.isSecurityEnabled()) { - val dstFs = dst.getFileSystem(conf) - dstFs.addDelegationTokens(delegTokenRenewer, credentials) -} +val nns = ClientBase.getNameNodesToAccess(sparkConf) + dst +ClientBase.obtainTokensForNamenodes(nns, conf, credentials) +val replication = sparkConf.getInt("spark.yarn.submit.file.replication", 3).toShort val localResources = HashMap[String, LocalResource]() FileSystem.mkdirs(fs, dst, new FsPermission(STAGING_DIR_PERMISSION)) @@ -614,4 +602,40 @@ object ClientBase extends Logging { YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, path, File.pathS
git commit: SPARK-1528 - spark on yarn, add support for accessing remote HDFS
Repository: spark Updated Branches: refs/heads/branch-1.1 b92a45058 -> 6c0c65fc8 SPARK-1528 - spark on yarn, add support for accessing remote HDFS Add a config (spark.yarn.access.namenodes) to allow applications running on yarn to access other secure HDFS cluster. User just specifies the namenodes of the other clusters and we get Tokens for those and ship them with the spark application. Author: Thomas Graves Closes #1159 from tgravescs/spark-1528 and squashes the following commits: ddbcd16 [Thomas Graves] review comments 0ac8501 [Thomas Graves] SPARK-1528 - add support for accessing remote HDFS (cherry picked from commit 2c0f705e26ca3dfc43a1e9a0722c0e57f67c970a) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c0c65fc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c0c65fc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c0c65fc Branch: refs/heads/branch-1.1 Commit: 6c0c65fc85677ab2cae819a546ea50ed660994c3 Parents: b92a450 Author: Thomas Graves Authored: Tue Aug 5 12:48:26 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 12:48:51 2014 -0500 -- docs/running-on-yarn.md | 7 +++ .../apache/spark/deploy/yarn/ClientBase.scala | 56 ++-- .../spark/deploy/yarn/ClientBaseSuite.scala | 55 ++- 3 files changed, 101 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6c0c65fc/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 0362f5a..573930d 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -106,6 +106,13 @@ Most of the configs are the same for Spark on YARN as for other deployment modes set this configuration to "hdfs:///some/path". + + spark.yarn.access.namenodes + (none) + +A list of secure HDFS namenodes your Spark application is going to access. For example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The Spark application must have acess to the namenodes listed and Kerberos must be properly configured to be able to access them (either in the same realm or in a trusted realm). Spark acquires security tokens for each of the namenodes so that the Spark application can access those remote HDFS clusters. + + # Launching Spark on YARN http://git-wip-us.apache.org/repos/asf/spark/blob/6c0c65fc/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index b7e8636..ed8f56a 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -29,7 +29,7 @@ import org.apache.hadoop.fs._ import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.mapred.Master import org.apache.hadoop.mapreduce.MRJobConfig -import org.apache.hadoop.security.UserGroupInformation +import org.apache.hadoop.security.{Credentials, UserGroupInformation} import org.apache.hadoop.util.StringUtils import org.apache.hadoop.yarn.api._ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment @@ -191,23 +191,11 @@ trait ClientBase extends Logging { // Upload Spark and the application JAR to the remote file system if necessary. Add them as // local resources to the application master. val fs = FileSystem.get(conf) - -val delegTokenRenewer = Master.getMasterPrincipal(conf) -if (UserGroupInformation.isSecurityEnabled()) { - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { -val errorMessage = "Can't get Master Kerberos principal for use as renewer" -logError(errorMessage) -throw new SparkException(errorMessage) - } -} val dst = new Path(fs.getHomeDirectory(), appStagingDir) -val replication = sparkConf.getInt("spark.yarn.submit.file.replication", 3).toShort - -if (UserGroupInformation.isSecurityEnabled()) { - val dstFs = dst.getFileSystem(conf) - dstFs.addDelegationTokens(delegTokenRenewer, credentials) -} +val nns = ClientBase.getNameNodesToAccess(sparkConf) + dst +ClientBase.obtainTokensForNamenodes(nns, conf, credentials) +val replication = sparkConf.getInt("spark.yarn.submit.file.replication", 3).toShort val localResources = HashMap[String, LocalResource]() FileSystem.mkdirs(fs, dst, new FsPermission(STAGING_DIR_PERMISSION)) @@ -614,4 +602,40 @@ object ClientBase extends Loggin
git commit: SPARK-1890 and SPARK-1891- add admin and modify acls
Repository: spark Updated Branches: refs/heads/master 2c0f705e2 -> 1ca23 SPARK-1890 and SPARK-1891- add admin and modify acls It was easier to combine these 2 jira since they touch many of the same places. This pr adds the following: - adds modify acls - adds admin acls (list of admins/users that get added to both view and modify acls) - modify Kill button on UI to take modify acls into account - changes config name of spark.ui.acls.enable to spark.acls.enable since I choose poorly in original name. We keep backwards compatibility so people can still use spark.ui.acls.enable. The acls should apply to any web ui as well as any CLI interfaces. - send view and modify acls information on to YARN so that YARN interfaces can use (yarn cli for killing applications for example). Author: Thomas Graves Closes #1196 from tgravescs/SPARK-1890 and squashes the following commits: 8292eb1 [Thomas Graves] review comments b92ec89 [Thomas Graves] remove unneeded variable from applistener 4c765f4 [Thomas Graves] Add in admin acls 72eb0ac [Thomas Graves] Add modify acls Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1ca2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1ca2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1ca2 Branch: refs/heads/master Commit: 1ca23d3aa40423d658cfbf2c956ad415a6b1 Parents: 2c0f705 Author: Thomas Graves Authored: Tue Aug 5 12:52:52 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 12:52:52 2014 -0500 -- .../org/apache/spark/SecurityManager.scala | 107 --- .../deploy/history/FsHistoryProvider.scala | 4 +- .../scheduler/ApplicationEventListener.scala| 4 +- .../apache/spark/ui/jobs/JobProgressTab.scala | 2 +- .../org/apache/spark/SecurityManagerSuite.scala | 83 -- docs/configuration.md | 27 - docs/security.md| 7 +- .../apache/spark/deploy/yarn/ClientBase.scala | 9 +- 8 files changed, 206 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1ca2/core/src/main/scala/org/apache/spark/SecurityManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala index 74aa441..25c2c9f 100644 --- a/core/src/main/scala/org/apache/spark/SecurityManager.scala +++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala @@ -41,10 +41,19 @@ import org.apache.spark.deploy.SparkHadoopUtil * secure the UI if it has data that other users should not be allowed to see. The javax * servlet filter specified by the user can authenticate the user and then once the user * is logged in, Spark can compare that user versus the view acls to make sure they are - * authorized to view the UI. The configs 'spark.ui.acls.enable' and 'spark.ui.view.acls' + * authorized to view the UI. The configs 'spark.acls.enable' and 'spark.ui.view.acls' * control the behavior of the acls. Note that the person who started the application * always has view access to the UI. * + * Spark has a set of modify acls (`spark.modify.acls`) that controls which users have permission + * to modify a single application. This would include things like killing the application. By + * default the person who started the application has modify access. For modify access through + * the UI, you must have a filter that does authentication in place for the modify acls to work + * properly. + * + * Spark also has a set of admin acls (`spark.admin.acls`) which is a set of users/administrators + * who always have permission to view or modify the Spark application. + * * Spark does not currently support encryption after authentication. * * At this point spark has multiple communication protocols that need to be secured and @@ -137,18 +146,32 @@ private[spark] class SecurityManager(sparkConf: SparkConf) extends Logging { private val sparkSecretLookupKey = "sparkCookie" private val authOn = sparkConf.getBoolean("spark.authenticate", false) - private var uiAclsOn = sparkConf.getBoolean("spark.ui.acls.enable", false) + // keep spark.ui.acls.enable for backwards compatibility with 1.0 + private var aclsOn = sparkConf.getOption("spark.acls.enable").getOrElse( +sparkConf.get("spark.ui.acls.enable", "false")).toBoolean + + // admin acls should be set before view or modify acls + private var adminAcls: Set[String] = +stringToSet(sparkConf.get("spark.admin.acls", "")) private var viewAcls: Set[String] = _ + + // list of users who have permission to modify the application. This should + // apply to both UI and CLI for things like killing th
git commit: SPARK-1890 and SPARK-1891- add admin and modify acls
Repository: spark Updated Branches: refs/heads/branch-1.1 6c0c65fc8 -> e3fe6571d SPARK-1890 and SPARK-1891- add admin and modify acls It was easier to combine these 2 jira since they touch many of the same places. This pr adds the following: - adds modify acls - adds admin acls (list of admins/users that get added to both view and modify acls) - modify Kill button on UI to take modify acls into account - changes config name of spark.ui.acls.enable to spark.acls.enable since I choose poorly in original name. We keep backwards compatibility so people can still use spark.ui.acls.enable. The acls should apply to any web ui as well as any CLI interfaces. - send view and modify acls information on to YARN so that YARN interfaces can use (yarn cli for killing applications for example). Author: Thomas Graves Closes #1196 from tgravescs/SPARK-1890 and squashes the following commits: 8292eb1 [Thomas Graves] review comments b92ec89 [Thomas Graves] remove unneeded variable from applistener 4c765f4 [Thomas Graves] Add in admin acls 72eb0ac [Thomas Graves] Add modify acls (cherry picked from commit 1ca23d3aa40423d658cfbf2c956ad415a6b1) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e3fe6571 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e3fe6571 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e3fe6571 Branch: refs/heads/branch-1.1 Commit: e3fe6571decfdc406ec6d505fd92f9f2b85a618c Parents: 6c0c65f Author: Thomas Graves Authored: Tue Aug 5 12:52:52 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 12:53:05 2014 -0500 -- .../org/apache/spark/SecurityManager.scala | 107 --- .../deploy/history/FsHistoryProvider.scala | 4 +- .../scheduler/ApplicationEventListener.scala| 4 +- .../apache/spark/ui/jobs/JobProgressTab.scala | 2 +- .../org/apache/spark/SecurityManagerSuite.scala | 83 -- docs/configuration.md | 27 - docs/security.md| 7 +- .../apache/spark/deploy/yarn/ClientBase.scala | 9 +- 8 files changed, 206 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e3fe6571/core/src/main/scala/org/apache/spark/SecurityManager.scala -- diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala index 74aa441..25c2c9f 100644 --- a/core/src/main/scala/org/apache/spark/SecurityManager.scala +++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala @@ -41,10 +41,19 @@ import org.apache.spark.deploy.SparkHadoopUtil * secure the UI if it has data that other users should not be allowed to see. The javax * servlet filter specified by the user can authenticate the user and then once the user * is logged in, Spark can compare that user versus the view acls to make sure they are - * authorized to view the UI. The configs 'spark.ui.acls.enable' and 'spark.ui.view.acls' + * authorized to view the UI. The configs 'spark.acls.enable' and 'spark.ui.view.acls' * control the behavior of the acls. Note that the person who started the application * always has view access to the UI. * + * Spark has a set of modify acls (`spark.modify.acls`) that controls which users have permission + * to modify a single application. This would include things like killing the application. By + * default the person who started the application has modify access. For modify access through + * the UI, you must have a filter that does authentication in place for the modify acls to work + * properly. + * + * Spark also has a set of admin acls (`spark.admin.acls`) which is a set of users/administrators + * who always have permission to view or modify the Spark application. + * * Spark does not currently support encryption after authentication. * * At this point spark has multiple communication protocols that need to be secured and @@ -137,18 +146,32 @@ private[spark] class SecurityManager(sparkConf: SparkConf) extends Logging { private val sparkSecretLookupKey = "sparkCookie" private val authOn = sparkConf.getBoolean("spark.authenticate", false) - private var uiAclsOn = sparkConf.getBoolean("spark.ui.acls.enable", false) + // keep spark.ui.acls.enable for backwards compatibility with 1.0 + private var aclsOn = sparkConf.getOption("spark.acls.enable").getOrElse( +sparkConf.get("spark.ui.acls.enable", "false")).toBoolean + + // admin acls should be set before view or modify acls + private var adminAcls: Set[String] = +stringToSet(sparkConf.get("spark.admin.acls", "")) private var viewAcls: Set[String] = _ + + // list of users who have
git commit: SPARK-1680: use configs for specifying environment variables on YARN
Repository: spark Updated Branches: refs/heads/master 74f82c71b -> 41e0a21b2 SPARK-1680: use configs for specifying environment variables on YARN Note that this also documents spark.executorEnv.* which to me means its public. If we don't want that please speak up. Author: Thomas Graves Closes #1512 from tgravescs/SPARK-1680 and squashes the following commits: 11525df [Thomas Graves] more doc changes 553bad0 [Thomas Graves] fix documentation 152bf7c [Thomas Graves] fix docs 5382326 [Thomas Graves] try fix docs 32f86a4 [Thomas Graves] use configs for specifying environment variables on YARN Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/41e0a21b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/41e0a21b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/41e0a21b Branch: refs/heads/master Commit: 41e0a21b22ccd2788dc079790788e505b0d4e37d Parents: 74f82c7 Author: Thomas Graves Authored: Tue Aug 5 15:57:32 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 15:57:32 2014 -0500 -- docs/configuration.md | 8 +++ docs/running-on-yarn.md | 22 +++- .../apache/spark/deploy/yarn/ClientBase.scala | 13 .../deploy/yarn/ExecutorRunnableUtil.scala | 6 +- 4 files changed, 43 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/41e0a21b/docs/configuration.md -- diff --git a/docs/configuration.md b/docs/configuration.md index 25adea2..5e7556c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -206,6 +206,14 @@ Apart from these, the following properties are also available, and may be useful used during aggregation goes above this amount, it will spill the data into disks. + + spark.executorEnv.[EnvironmentVariableName] + (none) + +Add the environment variable specified by EnvironmentVariableName to the Executor +process. The user can specify multiple of these and to set multiple environment variables. + + Shuffle Behavior http://git-wip-us.apache.org/repos/asf/spark/blob/41e0a21b/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 573930d..9bc20db 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -17,10 +17,6 @@ To build Spark yourself, refer to the [building with Maven guide](building-with- Most of the configs are the same for Spark on YARN as for other deployment modes. See the [configuration page](configuration.html) for more information on those. These are configs that are specific to Spark on YARN. - Environment Variables - -* `SPARK_YARN_USER_ENV`, to add environment variables to the Spark processes launched on YARN. This can be a comma separated list of environment variables, e.g. `SPARK_YARN_USER_ENV="JAVA_HOME=/jdk64,FOO=bar"`. - Spark Properties @@ -110,7 +106,23 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.access.namenodes (none) -A list of secure HDFS namenodes your Spark application is going to access. For example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The Spark application must have acess to the namenodes listed and Kerberos must be properly configured to be able to access them (either in the same realm or in a trusted realm). Spark acquires security tokens for each of the namenodes so that the Spark application can access those remote HDFS clusters. +A list of secure HDFS namenodes your Spark application is going to access. For +example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. +The Spark application must have acess to the namenodes listed and Kerberos must +be properly configured to be able to access them (either in the same realm or in +a trusted realm). Spark acquires security tokens for each of the namenodes so that +the Spark application can access those remote HDFS clusters. + + + + spark.yarn.appMasterEnv.[EnvironmentVariableName] + (none) + + Add the environment variable specified by EnvironmentVariableName to the + Application Master process launched on YARN. The user can specify multiple of + these and to set multiple environment variables. In yarn-cluster mode this controls + the environment of the SPARK driver and in yarn-client mode it only controls + the environment of the executor launcher. http://git-wip-us.apache.org/repos/asf/spark/blob/41e0a21b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala ---
git commit: SPARK-1680: use configs for specifying environment variables on YARN
Repository: spark Updated Branches: refs/heads/branch-1.1 46b698307 -> 7b798e10e SPARK-1680: use configs for specifying environment variables on YARN Note that this also documents spark.executorEnv.* which to me means its public. If we don't want that please speak up. Author: Thomas Graves Closes #1512 from tgravescs/SPARK-1680 and squashes the following commits: 11525df [Thomas Graves] more doc changes 553bad0 [Thomas Graves] fix documentation 152bf7c [Thomas Graves] fix docs 5382326 [Thomas Graves] try fix docs 32f86a4 [Thomas Graves] use configs for specifying environment variables on YARN (cherry picked from commit 41e0a21b22ccd2788dc079790788e505b0d4e37d) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7b798e10 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7b798e10 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7b798e10 Branch: refs/heads/branch-1.1 Commit: 7b798e10e214cd407d3399e2cab9e3789f9a929e Parents: 46b6983 Author: Thomas Graves Authored: Tue Aug 5 15:57:32 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 15:57:42 2014 -0500 -- docs/configuration.md | 8 +++ docs/running-on-yarn.md | 22 +++- .../apache/spark/deploy/yarn/ClientBase.scala | 13 .../deploy/yarn/ExecutorRunnableUtil.scala | 6 +- 4 files changed, 43 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/7b798e10/docs/configuration.md -- diff --git a/docs/configuration.md b/docs/configuration.md index 1333465..6ae453d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -206,6 +206,14 @@ Apart from these, the following properties are also available, and may be useful used during aggregation goes above this amount, it will spill the data into disks. + + spark.executorEnv.[EnvironmentVariableName] + (none) + +Add the environment variable specified by EnvironmentVariableName to the Executor +process. The user can specify multiple of these and to set multiple environment variables. + + Shuffle Behavior http://git-wip-us.apache.org/repos/asf/spark/blob/7b798e10/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 573930d..9bc20db 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -17,10 +17,6 @@ To build Spark yourself, refer to the [building with Maven guide](building-with- Most of the configs are the same for Spark on YARN as for other deployment modes. See the [configuration page](configuration.html) for more information on those. These are configs that are specific to Spark on YARN. - Environment Variables - -* `SPARK_YARN_USER_ENV`, to add environment variables to the Spark processes launched on YARN. This can be a comma separated list of environment variables, e.g. `SPARK_YARN_USER_ENV="JAVA_HOME=/jdk64,FOO=bar"`. - Spark Properties @@ -110,7 +106,23 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.access.namenodes (none) -A list of secure HDFS namenodes your Spark application is going to access. For example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The Spark application must have acess to the namenodes listed and Kerberos must be properly configured to be able to access them (either in the same realm or in a trusted realm). Spark acquires security tokens for each of the namenodes so that the Spark application can access those remote HDFS clusters. +A list of secure HDFS namenodes your Spark application is going to access. For +example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. +The Spark application must have acess to the namenodes listed and Kerberos must +be properly configured to be able to access them (either in the same realm or in +a trusted realm). Spark acquires security tokens for each of the namenodes so that +the Spark application can access those remote HDFS clusters. + + + + spark.yarn.appMasterEnv.[EnvironmentVariableName] + (none) + + Add the environment variable specified by EnvironmentVariableName to the + Application Master process launched on YARN. The user can specify multiple of + these and to set multiple environment variables. In yarn-cluster mode this controls + the environment of the SPARK driver and in yarn-client mode it only controls + the environment of the executor launcher. http://git-wip-us.apache.org/repos/asf/spark/blob/7
git commit: [SPARK-3072] YARN - Exit when reach max number failed executors
Repository: spark Updated Branches: refs/heads/master cd0720ca7 -> 7eb9cbc27 [SPARK-3072] YARN - Exit when reach max number failed executors In some cases on hadoop 2.x the spark application master doesn't properly exit and hangs around for 10 minutes after its really done. We should make sure it exits properly and stops the driver. Author: Thomas Graves Closes #2022 from tgravescs/SPARK-3072 and squashes the following commits: 665701d [Thomas Graves] Exit when reach max number failed executors Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7eb9cbc2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7eb9cbc2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7eb9cbc2 Branch: refs/heads/master Commit: 7eb9cbc273d758522e787fcb2ef68ef65911475f Parents: cd0720c Author: Thomas Graves Authored: Tue Aug 19 09:40:31 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 19 09:40:31 2014 -0500 -- .../spark/deploy/yarn/ApplicationMaster.scala | 33 +--- .../spark/deploy/yarn/ExecutorLauncher.scala| 5 +-- .../spark/deploy/yarn/ApplicationMaster.scala | 16 +++--- .../spark/deploy/yarn/ExecutorLauncher.scala| 5 +-- 4 files changed, 40 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/7eb9cbc2/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 62b5c3b..46a01f5 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -267,12 +267,10 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, // TODO: This is a bit ugly. Can we make it nicer? // TODO: Handle container failure - // Exists the loop if the user thread exits. - while (yarnAllocator.getNumExecutorsRunning < args.numExecutors && userThread.isAlive) { -if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) { - finishApplicationMaster(FinalApplicationStatus.FAILED, -"max number of executor failures reached") -} + // Exits the loop if the user thread exits. + while (yarnAllocator.getNumExecutorsRunning < args.numExecutors && userThread.isAlive + && !isFinished) { +checkNumExecutorsFailed() yarnAllocator.allocateContainers( math.max(args.numExecutors - yarnAllocator.getNumExecutorsRunning, 0)) Thread.sleep(ApplicationMaster.ALLOCATE_HEARTBEAT_INTERVAL) @@ -303,11 +301,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, val t = new Thread { override def run() { -while (userThread.isAlive) { - if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) { -finishApplicationMaster(FinalApplicationStatus.FAILED, - "max number of executor failures reached") - } +while (userThread.isAlive && !isFinished) { + checkNumExecutorsFailed() val missingExecutorCount = args.numExecutors - yarnAllocator.getNumExecutorsRunning if (missingExecutorCount > 0) { logInfo("Allocating %d containers to make up for (potentially) lost containers". @@ -327,6 +322,22 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, t } + private def checkNumExecutorsFailed() { +if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) { + logInfo("max number of executor failures reached") + finishApplicationMaster(FinalApplicationStatus.FAILED, +"max number of executor failures reached") + // make sure to stop the user thread + val sparkContext = ApplicationMaster.sparkContextRef.get() + if (sparkContext != null) { +logInfo("Invoking sc stop from checkNumExecutorsFailed") +sparkContext.stop() + } else { +logError("sparkContext is null when should shutdown") + } +} + } + private def sendProgress() { logDebug("Sending progress") // Simulated with an allocate request with no nodes requested ... http://git-wip-us.apache.org/repos/asf/spark/blob/7eb9cbc2/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLaunche
git commit: [SPARK-3072] YARN - Exit when reach max number failed executors
Repository: spark Updated Branches: refs/heads/branch-1.1 f3b0f34b4 -> 1418893da [SPARK-3072] YARN - Exit when reach max number failed executors In some cases on hadoop 2.x the spark application master doesn't properly exit and hangs around for 10 minutes after its really done. We should make sure it exits properly and stops the driver. Author: Thomas Graves Closes #2022 from tgravescs/SPARK-3072 and squashes the following commits: 665701d [Thomas Graves] Exit when reach max number failed executors (cherry picked from commit 7eb9cbc273d758522e787fcb2ef68ef65911475f) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1418893d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1418893d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1418893d Branch: refs/heads/branch-1.1 Commit: 1418893da557892b86fc47f1e41e91880d4f8eda Parents: f3b0f34 Author: Thomas Graves Authored: Tue Aug 19 09:40:31 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 19 09:40:55 2014 -0500 -- .../spark/deploy/yarn/ApplicationMaster.scala | 33 +--- .../spark/deploy/yarn/ExecutorLauncher.scala| 5 +-- .../spark/deploy/yarn/ApplicationMaster.scala | 16 +++--- .../spark/deploy/yarn/ExecutorLauncher.scala| 5 +-- 4 files changed, 40 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1418893d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 62b5c3b..46a01f5 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -267,12 +267,10 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, // TODO: This is a bit ugly. Can we make it nicer? // TODO: Handle container failure - // Exists the loop if the user thread exits. - while (yarnAllocator.getNumExecutorsRunning < args.numExecutors && userThread.isAlive) { -if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) { - finishApplicationMaster(FinalApplicationStatus.FAILED, -"max number of executor failures reached") -} + // Exits the loop if the user thread exits. + while (yarnAllocator.getNumExecutorsRunning < args.numExecutors && userThread.isAlive + && !isFinished) { +checkNumExecutorsFailed() yarnAllocator.allocateContainers( math.max(args.numExecutors - yarnAllocator.getNumExecutorsRunning, 0)) Thread.sleep(ApplicationMaster.ALLOCATE_HEARTBEAT_INTERVAL) @@ -303,11 +301,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, val t = new Thread { override def run() { -while (userThread.isAlive) { - if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) { -finishApplicationMaster(FinalApplicationStatus.FAILED, - "max number of executor failures reached") - } +while (userThread.isAlive && !isFinished) { + checkNumExecutorsFailed() val missingExecutorCount = args.numExecutors - yarnAllocator.getNumExecutorsRunning if (missingExecutorCount > 0) { logInfo("Allocating %d containers to make up for (potentially) lost containers". @@ -327,6 +322,22 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, t } + private def checkNumExecutorsFailed() { +if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) { + logInfo("max number of executor failures reached") + finishApplicationMaster(FinalApplicationStatus.FAILED, +"max number of executor failures reached") + // make sure to stop the user thread + val sparkContext = ApplicationMaster.sparkContextRef.get() + if (sparkContext != null) { +logInfo("Invoking sc stop from checkNumExecutorsFailed") +sparkContext.stop() + } else { +logError("sparkContext is null when should shutdown") + } +} + } + private def sendProgress() { logDebug("Sending progress") // Simulated with an allocate request with no nodes requested ... http://git-wip-us.apache.org/repos/asf/spark/blob/1418893d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/
git commit: [SPARK-2742][yarn] delete useless variables
Repository: spark Updated Branches: refs/heads/master 050f8d01e -> 220c2d769 [SPARK-2742][yarn] delete useless variables Author: XuTingjun <1039320...@qq.com> Closes #1614 from XuTingjun/yarn-bug and squashes the following commits: f07096e [XuTingjun] Update ClientArguments.scala Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/220c2d76 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/220c2d76 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/220c2d76 Branch: refs/heads/master Commit: 220c2d7699a522c50d26cd6fdd94f4635823e408 Parents: 050f8d0 Author: XuTingjun <1039320...@qq.com> Authored: Fri Aug 22 10:45:00 2014 -0500 Committer: Thomas Graves Committed: Fri Aug 22 10:45:00 2014 -0500 -- .../scala/org/apache/spark/deploy/yarn/ClientArguments.scala| 5 + 1 file changed, 1 insertion(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/220c2d76/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala index 62f9b3cf..afa4fd4 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala @@ -39,7 +39,6 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) { var amMemory: Int = 512 // MB var amClass: String = "org.apache.spark.deploy.yarn.ApplicationMaster" var appName: String = "Spark" - var inputFormatInfo: List[InputFormatInfo] = null var priority = 0 parseArgs(args.toList) @@ -58,8 +57,7 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) { private def parseArgs(inputArgs: List[String]): Unit = { val userArgsBuffer: ArrayBuffer[String] = new ArrayBuffer[String]() -val inputFormatMap: HashMap[String, InputFormatInfo] = new HashMap[String, InputFormatInfo]() - + var args = inputArgs while (!args.isEmpty) { @@ -145,7 +143,6 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) { } userArgs = userArgsBuffer.readOnly -inputFormatInfo = inputFormatMap.values.toList } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[1/2] [SPARK-2933] [yarn] Refactor and cleanup Yarn AM code.
Repository: spark Updated Branches: refs/heads/master 6f671d04f -> b92d823ad http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala index 3474112..d162b4c 100644 --- a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala +++ b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala @@ -19,22 +19,21 @@ package org.apache.spark.scheduler.cluster import org.apache.spark._ import org.apache.hadoop.conf.Configuration -import org.apache.spark.deploy.yarn.YarnAllocationHandler +import org.apache.spark.deploy.yarn.YarnSparkHadoopUtil import org.apache.spark.scheduler.TaskSchedulerImpl import org.apache.spark.util.Utils /** - * - * This scheduler launches executors through Yarn - by calling into Client to launch ExecutorLauncher as AM. + * This scheduler launches executors through Yarn - by calling into Client to launch the Spark AM. */ -private[spark] class YarnClientClusterScheduler(sc: SparkContext, conf: Configuration) extends TaskSchedulerImpl(sc) { +private[spark] class YarnClientClusterScheduler(sc: SparkContext, conf: Configuration) + extends TaskSchedulerImpl(sc) { def this(sc: SparkContext) = this(sc, new Configuration()) // By default, rack is unknown override def getRackForHost(hostPort: String): Option[String] = { val host = Utils.parseHostPort(hostPort)._1 -val retval = YarnAllocationHandler.lookupRack(conf, host) -if (retval != null) Some(retval) else None +Option(YarnSparkHadoopUtil.lookupRack(conf, host)) } } http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala index 833e249..a5f537d 100644 --- a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala +++ b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala @@ -19,7 +19,7 @@ package org.apache.spark.scheduler.cluster import org.apache.hadoop.yarn.api.records.{ApplicationId, YarnApplicationState} import org.apache.spark.{SparkException, Logging, SparkContext} -import org.apache.spark.deploy.yarn.{Client, ClientArguments, ExecutorLauncher, YarnSparkHadoopUtil} +import org.apache.spark.deploy.yarn.{Client, ClientArguments, YarnSparkHadoopUtil} import org.apache.spark.scheduler.TaskSchedulerImpl import scala.collection.mutable.ArrayBuffer @@ -60,10 +60,7 @@ private[spark] class YarnClientSchedulerBackend( val argsArrayBuf = new ArrayBuffer[String]() argsArrayBuf += ( - "--class", "notused", - "--jar", null, // The primary jar will be added dynamically in SparkContext. - "--args", hostport, - "--am-class", classOf[ExecutorLauncher].getName + "--args", hostport ) // process any optional arguments, given either as environment variables http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala index 9aeca4a..69f4022 100644 --- a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala +++ b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala @@ -18,16 +18,17 @@ package org.apache.spark.scheduler.cluster import org.apache.spark._ -import org.apache.spark.deploy.yarn.{ApplicationMaster, YarnAllocationHandler} +import org.apache.spark.deploy.yarn.{ApplicationMaster, YarnSparkHadoopUtil} import org.apache.spark.scheduler.TaskSchedulerImpl import org.apache.spark.util.Utils import org.apache.hadoop.conf.Configuration /** - * - * This is a simple extension to ClusterScheduler - to ensure that appropriate initialization of ApplicationMaster, etc is done + * This is a simple extension to ClusterScheduler - to ensure that appropriate initialization of + * ApplicationMaster, etc is done */ -private[spark] class YarnClusterScheduler(sc: SparkContext, conf: Configuration) extends TaskSchedulerImpl(sc) { +private
[2/2] git commit: [SPARK-2933] [yarn] Refactor and cleanup Yarn AM code.
[SPARK-2933] [yarn] Refactor and cleanup Yarn AM code. This change modifies the Yarn module so that all the logic related to running the ApplicationMaster is localized. Instead of, previously, 4 different classes with mostly identical code, now we have: - A single, shared ApplicationMaster class, which can operate both in client and cluster mode, and substitutes the old ApplicationMaster (for cluster mode) and ExecutorLauncher (for client mode). The benefit here is that all different execution modes for all supported yarn versions use the same shared code for monitoring executor allocation, setting up configuration, and monitoring the process's lifecycle. - A new YarnRMClient interface, which defines basic RM functionality needed by the ApplicationMaster. This interface has concrete implementations for each supported Yarn version. - A new YarnAllocator interface, which just abstracts the existing interface of the YarnAllocationHandler class. This is to avoid having to touch the allocator code too much in this change, although it might benefit from a similar effort in the future. The end result is much easier to understand code, with much less duplication, making it much easier to fix bugs, add features, and test everything knowing that all supported versions will behave the same. Author: Marcelo Vanzin Closes #2020 from vanzin/SPARK-2933 and squashes the following commits: 3bbf3e7 [Marcelo Vanzin] Merge branch 'master' into SPARK-2933 ff389ed [Marcelo Vanzin] Do not interrupt reporter thread from within itself. 3a8ed37 [Marcelo Vanzin] Remote stale comment. 0f5142c [Marcelo Vanzin] Review feedback. 41f8c8a [Marcelo Vanzin] Fix app status reporting. c0794be [Marcelo Vanzin] Correctly clean up staging directory. 92770cc [Marcelo Vanzin] Merge branch 'master' into SPARK-2933 ecaf332 [Marcelo Vanzin] Small fix to shutdown code. f02d3f8 [Marcelo Vanzin] Merge branch 'master' into SPARK-2933 f581122 [Marcelo Vanzin] Review feedback. 557fdeb [Marcelo Vanzin] Cleanup a couple more constants. be6068d [Marcelo Vanzin] Restore shutdown hook to clean up staging dir. 5150993 [Marcelo Vanzin] Some more cleanup. b6289ab [Marcelo Vanzin] Move cluster/client code to separate methods. ecb23cd [Marcelo Vanzin] More trivial cleanup. 34f1e63 [Marcelo Vanzin] Fix some questionable error handling. 5657c7d [Marcelo Vanzin] Finish app if SparkContext initialization times out. 0e4be3d [Marcelo Vanzin] Keep "ExecutorLauncher" as the main class for client-mode AM. 91beabb [Marcelo Vanzin] Fix UI filter registration. 8c72239 [Marcelo Vanzin] Trivial cleanups. 99a52d5 [Marcelo Vanzin] Changes to the yarn-alpha project to use common AM code. 848ca6d [Marcelo Vanzin] [SPARK-2933] [yarn] Refactor and cleanup Yarn AM code. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b92d823a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b92d823a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b92d823a Branch: refs/heads/master Commit: b92d823ad13f6fcc325eeb99563bea543871c6aa Parents: 6f671d0 Author: Marcelo Vanzin Authored: Wed Aug 27 11:02:04 2014 -0500 Committer: Thomas Graves Committed: Wed Aug 27 11:02:04 2014 -0500 -- .../spark/deploy/yarn/ApplicationMaster.scala | 453 --- .../spark/deploy/yarn/ExecutorLauncher.scala| 315 - .../deploy/yarn/YarnAllocationHandler.scala | 192 ++-- .../spark/deploy/yarn/YarnRMClientImpl.scala| 103 + .../spark/deploy/yarn/ApplicationMaster.scala | 430 ++ .../yarn/ApplicationMasterArguments.scala | 26 +- .../spark/deploy/yarn/ClientArguments.scala | 9 +- .../apache/spark/deploy/yarn/ClientBase.scala | 54 ++- .../spark/deploy/yarn/YarnAllocator.scala | 34 ++ .../apache/spark/deploy/yarn/YarnRMClient.scala | 67 +++ .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 51 +++ .../cluster/YarnClientClusterScheduler.scala| 11 +- .../cluster/YarnClientSchedulerBackend.scala| 7 +- .../cluster/YarnClusterScheduler.scala | 17 +- .../spark/deploy/yarn/ApplicationMaster.scala | 413 - .../spark/deploy/yarn/ExecutorLauncher.scala| 276 --- .../deploy/yarn/YarnAllocationHandler.scala | 196 ++-- .../spark/deploy/yarn/YarnRMClientImpl.scala| 76 18 files changed, 892 insertions(+), 1838 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala deleted file mode 100644 index
git commit: [SPARK-3347] [yarn] Fix yarn-alpha compilation.
Repository: spark Updated Branches: refs/heads/master 8f1f9aaf4 -> 066f31a6b [SPARK-3347] [yarn] Fix yarn-alpha compilation. Missing import. Oops. Author: Marcelo Vanzin Closes #2236 from vanzin/SPARK-3347 and squashes the following commits: 594fc39 [Marcelo Vanzin] [SPARK-3347] [yarn] Fix yarn-alpha compilation. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/066f31a6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/066f31a6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/066f31a6 Branch: refs/heads/master Commit: 066f31a6b213121441fc9618abd5bae4a706a215 Parents: 8f1f9aa Author: Marcelo Vanzin Authored: Tue Sep 2 13:33:23 2014 -0500 Committer: Thomas Graves Committed: Tue Sep 2 13:33:23 2014 -0500 -- yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/066f31a6/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 12f1cd3..10fc39b 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{Apps, Records} import org.apache.spark.{Logging, SparkConf} +import org.apache.spark.deploy.SparkHadoopUtil /** * Version of [[org.apache.spark.deploy.yarn.ClientBase]] tailored to YARN's alpha API. - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-3187] [yarn] Cleanup allocator code.
Repository: spark Updated Branches: refs/heads/master c64cc435e -> 6a72a3694 [SPARK-3187] [yarn] Cleanup allocator code. Move all shared logic to the base YarnAllocator class, and leave the version-specific logic in the version-specific module. Author: Marcelo Vanzin Closes #2169 from vanzin/SPARK-3187 and squashes the following commits: 46c2826 [Marcelo Vanzin] Hide the privates. 4dc9c83 [Marcelo Vanzin] Actually release containers. 8b1a077 [Marcelo Vanzin] Changes to the Yarn alpha allocator. f3f5f1d [Marcelo Vanzin] [SPARK-3187] [yarn] Cleanup allocator code. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a72a369 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6a72a369 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6a72a369 Branch: refs/heads/master Commit: 6a72a36940311fcb3429bd34c8818bc7d513115c Parents: c64cc43 Author: Marcelo Vanzin Authored: Wed Sep 3 08:22:50 2014 -0500 Committer: Thomas Graves Committed: Wed Sep 3 08:22:50 2014 -0500 -- .../deploy/yarn/YarnAllocationHandler.scala | 462 +++ .../spark/deploy/yarn/YarnAllocator.scala | 425 - .../deploy/yarn/YarnAllocationHandler.scala | 402 +--- 3 files changed, 495 insertions(+), 794 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6a72a369/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala index 629cd13..9f9e16c 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala @@ -17,35 +17,21 @@ package org.apache.spark.deploy.yarn -import java.util.concurrent.{CopyOnWriteArrayList, ConcurrentHashMap} +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConversions._ -import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet} +import scala.collection.mutable.{ArrayBuffer, HashMap} -import org.apache.spark.{Logging, SparkConf, SparkEnv} -import org.apache.spark.scheduler.{SplitInfo, TaskSchedulerImpl} -import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend -import org.apache.spark.util.Utils +import org.apache.spark.SparkConf +import org.apache.spark.scheduler.SplitInfo import org.apache.hadoop.conf.Configuration import org.apache.hadoop.yarn.api.AMRMProtocol -import org.apache.hadoop.yarn.api.records.{AMResponse, ApplicationAttemptId} -import org.apache.hadoop.yarn.api.records.{Container, ContainerId} -import org.apache.hadoop.yarn.api.records.{Priority, Resource, ResourceRequest} -import org.apache.hadoop.yarn.api.protocolrecords.{AllocateRequest, AllocateResponse} +import org.apache.hadoop.yarn.api.records._ +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest import org.apache.hadoop.yarn.util.Records -// TODO: -// Too many params. -// Needs to be mt-safe -// Need to refactor this to make it 'cleaner' ... right now, all computation is reactive - should -// make it more proactive and decoupled. - -// Note that right now, we assume all node asks as uniform in terms of capabilities and priority -// Refer to http://developer.yahoo.com/blogs/hadoop/posts/2011/03/mapreduce-nextgen-scheduler/ for -// more info on how we are requesting for containers. - /** * Acquires resources for executors from a ResourceManager and launches executors in new containers. */ @@ -56,357 +42,20 @@ private[yarn] class YarnAllocationHandler( appAttemptId: ApplicationAttemptId, args: ApplicationMasterArguments, preferredNodes: collection.Map[String, collection.Set[SplitInfo]]) - extends YarnAllocator with Logging { - - // These three are locked on allocatedHostToContainersMap. Complementary data structures - // allocatedHostToContainersMap : containers which are running : host, Set - // allocatedContainerToHostMap: container to host mapping. - private val allocatedHostToContainersMap = -new HashMap[String, collection.mutable.Set[ContainerId]]() - - private val allocatedContainerToHostMap = new HashMap[ContainerId, String]() - - // allocatedRackCount is populated ONLY if allocation happens (or decremented if this is an - // allocated node) - // As with the two data structures above, tightly coupled with them, and to be locked on - // allocatedHostToContainersMap - private val allocatedRackCount = new HashMap[String, Int]() - - // Containers which have been released. - private val releas
git commit: [SPARK-3260] yarn - pass acls along with executor launch
Repository: spark Updated Branches: refs/heads/master 6a37ed838 -> 51b53a758 [SPARK-3260] yarn - pass acls along with executor launch Pass along the acl settings when we launch a container so that they can be applied to viewing the logs on a running NodeManager. Author: Thomas Graves Closes #2185 from tgravescs/SPARK-3260 and squashes the following commits: 6f94b5a [Thomas Graves] make unit test more robust 28b9dd3 [Thomas Graves] yarn - pass acls along with executor launch Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/51b53a75 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/51b53a75 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/51b53a75 Branch: refs/heads/master Commit: 51b53a758c85f2e20ad9bd73ed815fcfa9c7180b Parents: 6a37ed8 Author: Thomas Graves Authored: Fri Sep 5 09:54:40 2014 -0500 Committer: Thomas Graves Committed: Fri Sep 5 09:54:40 2014 -0500 -- .../spark/deploy/yarn/ExecutorRunnable.scala| 7 +- .../deploy/yarn/YarnAllocationHandler.scala | 7 +- .../spark/deploy/yarn/YarnRMClientImpl.scala| 7 +- .../spark/deploy/yarn/ApplicationMaster.scala | 13 ++-- .../apache/spark/deploy/yarn/ClientBase.scala | 6 +- .../spark/deploy/yarn/YarnAllocator.scala | 10 +-- .../apache/spark/deploy/yarn/YarnRMClient.scala | 5 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 11 ++- .../deploy/yarn/YarnSparkHadoopUtilSuite.scala | 76 +++- .../spark/deploy/yarn/ExecutorRunnable.scala| 7 +- .../deploy/yarn/YarnAllocationHandler.scala | 7 +- .../spark/deploy/yarn/YarnRMClientImpl.scala| 7 +- 12 files changed, 129 insertions(+), 34 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/51b53a75/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 7dae248..10cbeb8 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{Apps, ConverterUtils, Records, ProtoUtils} -import org.apache.spark.{SparkConf, Logging} +import org.apache.spark.{SecurityManager, SparkConf, Logging} class ExecutorRunnable( @@ -46,7 +46,8 @@ class ExecutorRunnable( slaveId: String, hostname: String, executorMemory: Int, -executorCores: Int) +executorCores: Int, +securityMgr: SecurityManager) extends Runnable with ExecutorRunnableUtil with Logging { var rpc: YarnRPC = YarnRPC.create(conf) @@ -86,6 +87,8 @@ class ExecutorRunnable( logInfo("Setting up executor with commands: " + commands) ctx.setCommands(commands) + ctx.setApplicationACLs(YarnSparkHadoopUtil.getApplicationAclsForYarn(securityMgr)) + // Send the start request to the ContainerManager val startReq = Records.newRecord(classOf[StartContainerRequest]) .asInstanceOf[StartContainerRequest] http://git-wip-us.apache.org/repos/asf/spark/blob/51b53a75/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala index 9f9e16c..85d6274 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConversions._ import scala.collection.mutable.{ArrayBuffer, HashMap} -import org.apache.spark.SparkConf +import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.scheduler.SplitInfo import org.apache.hadoop.conf.Configuration @@ -41,8 +41,9 @@ private[yarn] class YarnAllocationHandler( resourceManager: AMRMProtocol, appAttemptId: ApplicationAttemptId, args: ApplicationMasterArguments, -preferredNodes: collection.Map[String, collection.Set[SplitInfo]]) - extends YarnAllocator(conf, sparkConf, args, preferredNodes) { +preferredNodes: collection.Map[String, collection.Set[SplitInfo]], +securityMgr: SecurityManager) + extends YarnAllocator(conf, sparkConf, args, preferredNodes, securityMgr) { private val lastResponseId = new AtomicInteg
git commit: [SPARK-3375] spark on yarn container allocation issues
Repository: spark Updated Branches: refs/heads/master 51b53a758 -> 62c557609 [SPARK-3375] spark on yarn container allocation issues If yarn doesn't get the containers immediately it stops asking for them and the yarn application hangs with never getting any executors. The issue here is that we are sending the number of containers as 0 after we send the original one of X. on the yarn side this clears out the original request. For a ping we should just send empty asks. Author: Thomas Graves Closes #2275 from tgravescs/SPARK-3375 and squashes the following commits: 74b6820 [Thomas Graves] send empty resource requests when we aren't asking for containers Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/62c55760 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/62c55760 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/62c55760 Branch: refs/heads/master Commit: 62c557609929982eeec170fe12f810bedfcf97f2 Parents: 51b53a7 Author: Thomas Graves Authored: Fri Sep 5 09:56:22 2014 -0500 Committer: Thomas Graves Committed: Fri Sep 5 09:56:22 2014 -0500 -- .../spark/deploy/yarn/YarnAllocationHandler.scala | 13 +++-- .../spark/deploy/yarn/YarnAllocationHandler.scala | 8 +--- 2 files changed, 12 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/62c55760/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala index 85d6274..5a1b42c 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala @@ -51,12 +51,13 @@ private[yarn] class YarnAllocationHandler( override protected def allocateContainers(count: Int): YarnAllocateResponse = { var resourceRequests: List[ResourceRequest] = null -// default. -if (count <= 0 || preferredHostToCount.isEmpty) { - logDebug("numExecutors: " + count + ", host preferences: " + -preferredHostToCount.isEmpty) - resourceRequests = List(createResourceRequest( -AllocationType.ANY, null, count, YarnSparkHadoopUtil.RM_REQUEST_PRIORITY)) +logDebug("numExecutors: " + count) +if (count <= 0) { + resourceRequests = List() +} else if (preferredHostToCount.isEmpty) { +logDebug("host preferences is empty") +resourceRequests = List(createResourceRequest( + AllocationType.ANY, null, count, YarnSparkHadoopUtil.RM_REQUEST_PRIORITY)) } else { // request for all hosts in preferred nodes and for numExecutors - // candidates.size, request by default allocation policy. http://git-wip-us.apache.org/repos/asf/spark/blob/62c55760/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala -- diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala index c887cb5..5438f15 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala @@ -88,9 +88,11 @@ private[yarn] class YarnAllocationHandler( private def addResourceRequests(numExecutors: Int) { val containerRequests: List[ContainerRequest] = - if (numExecutors <= 0 || preferredHostToCount.isEmpty) { -logDebug("numExecutors: " + numExecutors + ", host preferences: " + - preferredHostToCount.isEmpty) + if (numExecutors <= 0) { +logDebug("numExecutors: " + numExecutors) +List() + } else if (preferredHostToCount.isEmpty) { +logDebug("host preferences is empty") createResourceRequests( AllocationType.ANY, resource = null, - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-3286] - Cannot view ApplicationMaster UI when Yarn’s url scheme i...
Repository: spark Updated Branches: refs/heads/master b734ed0c2 -> 6f7a76838 [SPARK-3286] - Cannot view ApplicationMaster UI when Yarnâs url scheme i... ...s https Author: Benoy Antony Closes #2276 from benoyantony/SPARK-3286 and squashes the following commits: c3d51ee [Benoy Antony] Use address with scheme, but Allpha version removes the scheme e82f94e [Benoy Antony] Use address with scheme, but Allpha version removes the scheme 92127c9 [Benoy Antony] rebasing from master 450c536 [Benoy Antony] [SPARK-3286] - Cannot view ApplicationMaster UI when Yarnâs url scheme is https f060c02 [Benoy Antony] [SPARK-3286] - Cannot view ApplicationMaster UI when Yarnâs url scheme is https Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6f7a7683 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6f7a7683 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6f7a7683 Branch: refs/heads/master Commit: 6f7a76838f15687583e3b0ab43309a3c079368c4 Parents: b734ed0 Author: Benoy Antony Authored: Wed Sep 10 11:59:39 2014 -0500 Committer: Thomas Graves Committed: Wed Sep 10 11:59:39 2014 -0500 -- .../scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala| 4 +++- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6f7a7683/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala index ad27a9a..fc30953 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala @@ -18,6 +18,7 @@ package org.apache.spark.deploy.yarn import scala.collection.{Map, Set} +import java.net.URI; import org.apache.hadoop.net.NetUtils import org.apache.hadoop.yarn.api._ @@ -97,7 +98,8 @@ private class YarnRMClientImpl(args: ApplicationMasterArguments) extends YarnRMC // Users can then monitor stderr/stdout on that node if required. appMasterRequest.setHost(Utils.localHostName()) appMasterRequest.setRpcPort(0) -appMasterRequest.setTrackingUrl(uiAddress) +//remove the scheme from the url if it exists since Hadoop does not expect scheme +appMasterRequest.setTrackingUrl(new URI(uiAddress).getAuthority()) resourceManager.registerApplicationMaster(appMasterRequest) } http://git-wip-us.apache.org/repos/asf/spark/blob/6f7a7683/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index a879c83..5756263 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -189,7 +189,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, if (sc == null) { finish(FinalApplicationStatus.FAILED, "Timed out waiting for SparkContext.") } else { - registerAM(sc.ui.appUIHostPort, securityMgr) + registerAM(sc.ui.appUIAddress, securityMgr) try { userThread.join() } finally { - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
git commit: SPARK-1713. Use a thread pool for launching executors.
Repository: spark Updated Branches: refs/heads/master 26503fdf2 -> 1f4a648d4 SPARK-1713. Use a thread pool for launching executors. This patch copies the approach used in the MapReduce application master for launching containers. Author: Sandy Ryza Closes #663 from sryza/sandy-spark-1713 and squashes the following commits: 036550d [Sandy Ryza] SPARK-1713. [YARN] Use a threadpool for launching executor containers Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1f4a648d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1f4a648d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1f4a648d Branch: refs/heads/master Commit: 1f4a648d4e30e837d6cf3ea8de1808e2254ad70b Parents: 26503fd Author: Sandy Ryza Authored: Wed Sep 10 14:34:24 2014 -0500 Committer: Thomas Graves Committed: Wed Sep 10 14:34:24 2014 -0500 -- docs/running-on-yarn.md | 7 +++ .../org/apache/spark/deploy/yarn/YarnAllocator.scala | 14 -- 2 files changed, 19 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1f4a648d/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 943f06b..d8b22f3 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -125,6 +125,13 @@ Most of the configs are the same for Spark on YARN as for other deployment modes the environment of the executor launcher. + + spark.yarn.containerLauncherMaxThreads + 25 + +The maximum number of threads to use in the application master for launching executor containers. + + # Launching Spark on YARN http://git-wip-us.apache.org/repos/asf/spark/blob/1f4a648d/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 02b9a81..0b8744f 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.yarn import java.util.{List => JList} -import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent._ import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConversions._ @@ -32,6 +32,8 @@ import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkEnv} import org.apache.spark.scheduler.{SplitInfo, TaskSchedulerImpl} import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend +import com.google.common.util.concurrent.ThreadFactoryBuilder + object AllocationType extends Enumeration { type AllocationType = Value val HOST, RACK, ANY = Value @@ -95,6 +97,14 @@ private[yarn] abstract class YarnAllocator( protected val (preferredHostToCount, preferredRackToCount) = generateNodeToWeight(conf, preferredNodes) + private val launcherPool = new ThreadPoolExecutor( +// max pool size of Integer.MAX_VALUE is ignored because we use an unbounded queue +sparkConf.getInt("spark.yarn.containerLauncherMaxThreads", 25), Integer.MAX_VALUE, +1, TimeUnit.MINUTES, +new LinkedBlockingQueue[Runnable](), +new ThreadFactoryBuilder().setNameFormat("ContainerLauncher #%d").setDaemon(true).build()) + launcherPool.allowCoreThreadTimeOut(true) + def getNumExecutorsRunning: Int = numExecutorsRunning.intValue def getNumExecutorsFailed: Int = numExecutorsFailed.intValue @@ -283,7 +293,7 @@ private[yarn] abstract class YarnAllocator( executorMemory, executorCores, securityMgr) - new Thread(executorRunnable).start() + launcherPool.execute(executorRunnable) } } logDebug(""" - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Repository: spark Updated Branches: refs/heads/master c27718f37 -> ed1980ffa [SPARK-2140] Updating heap memory calculation for YARN stable and alpha. Updated pull request, reflecting YARN stable and alpha states. I am getting intermittent test failures on my own test infrastructure. Is that tracked anywhere yet? Author: Chris Cope Closes #2253 from copester/master and squashes the following commits: 5ad89da [Chris Cope] [SPARK-2140] Removing calculateAMMemory functions since they are no longer needed. 52b4e45 [Chris Cope] [SPARK-2140] Updating heap memory calculation for YARN stable and alpha. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ed1980ff Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ed1980ff Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ed1980ff Branch: refs/heads/master Commit: ed1980ffa9ccb87d76694ba910ef22df034bca49 Parents: c27718f Author: Chris Cope Authored: Thu Sep 11 08:13:07 2014 -0500 Committer: Thomas Graves Committed: Thu Sep 11 08:13:07 2014 -0500 -- .../main/scala/org/apache/spark/deploy/yarn/Client.scala| 8 .../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 4 +--- .../org/apache/spark/deploy/yarn/ClientBaseSuite.scala | 3 --- .../main/scala/org/apache/spark/deploy/yarn/Client.scala| 9 - 4 files changed, 1 insertion(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 10fc39b..aff9ab7 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -103,14 +103,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa appContext } - def calculateAMMemory(newApp: GetNewApplicationResponse): Int = { -val minResMemory = newApp.getMinimumResourceCapability().getMemory() -val amMemory = ((args.amMemory / minResMemory) * minResMemory) + - ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) - - memoryOverhead) -amMemory - } - def setupSecurityToken(amContainer: ContainerLaunchContext) = { // Setup security tokens. val dob = new DataOutputBuffer() http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 8075b7a..c96f731 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -300,8 +300,6 @@ trait ClientBase extends Logging { retval.toString } - def calculateAMMemory(newApp: GetNewApplicationResponse): Int - def setupSecurityToken(amContainer: ContainerLaunchContext) def createContainerLaunchContext( @@ -346,7 +344,7 @@ trait ClientBase extends Logging { } amContainer.setEnvironment(env) -val amMemory = calculateAMMemory(newApp) +val amMemory = args.amMemory val javaOpts = ListBuffer[String]() http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala -- diff --git a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala index 68cc289..5480eca 100644 --- a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala +++ b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala @@ -238,9 +238,6 @@ class ClientBaseSuite extends FunSuite with Matchers { val sparkConf: SparkConf, val yarnConf: YarnConfiguration) extends ClientBase { -override def calculateAMMemory(newApp: GetNewApplicationResponse): Int = - throw new UnsupportedOperationException() - override def setupSecurityToken(amContainer: ContainerLaunchContext): Unit = throw new UnsupportedOperationException() http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala -- diff --git a/yarn/stable/src/main/scala/o
git commit: [SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Repository: spark Updated Branches: refs/heads/branch-1.1 e51ce9a55 -> 06fb2d057 [SPARK-2140] Updating heap memory calculation for YARN stable and alpha. Updated pull request, reflecting YARN stable and alpha states. I am getting intermittent test failures on my own test infrastructure. Is that tracked anywhere yet? Author: Chris Cope Closes #2253 from copester/master and squashes the following commits: 5ad89da [Chris Cope] [SPARK-2140] Removing calculateAMMemory functions since they are no longer needed. 52b4e45 [Chris Cope] [SPARK-2140] Updating heap memory calculation for YARN stable and alpha. (cherry picked from commit ed1980ffa9ccb87d76694ba910ef22df034bca49) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06fb2d05 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06fb2d05 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06fb2d05 Branch: refs/heads/branch-1.1 Commit: 06fb2d057beb50e9b690bf8b6d5bb7bdb16d8546 Parents: e51ce9a Author: Chris Cope Authored: Thu Sep 11 08:13:07 2014 -0500 Committer: Thomas Graves Committed: Thu Sep 11 08:13:27 2014 -0500 -- .../main/scala/org/apache/spark/deploy/yarn/Client.scala| 8 .../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 4 +--- .../org/apache/spark/deploy/yarn/ClientBaseSuite.scala | 3 --- .../main/scala/org/apache/spark/deploy/yarn/Client.scala| 9 - 4 files changed, 1 insertion(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 9be7854..3607eed 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -102,14 +102,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa appContext } - def calculateAMMemory(newApp: GetNewApplicationResponse): Int = { -val minResMemory = newApp.getMinimumResourceCapability().getMemory() -val amMemory = ((args.amMemory / minResMemory) * minResMemory) + - ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) - - memoryOverhead) -amMemory - } - def setupSecurityToken(amContainer: ContainerLaunchContext) = { // Setup security tokens. val dob = new DataOutputBuffer() http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 3897b3a..6da3b16 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -309,8 +309,6 @@ trait ClientBase extends Logging { retval.toString } - def calculateAMMemory(newApp: GetNewApplicationResponse): Int - def setupSecurityToken(amContainer: ContainerLaunchContext) def createContainerLaunchContext( @@ -353,7 +351,7 @@ trait ClientBase extends Logging { } amContainer.setEnvironment(env) -val amMemory = calculateAMMemory(newApp) +val amMemory = args.amMemory val javaOpts = ListBuffer[String]() http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala -- diff --git a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala index 68cc289..5480eca 100644 --- a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala +++ b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala @@ -238,9 +238,6 @@ class ClientBaseSuite extends FunSuite with Matchers { val sparkConf: SparkConf, val yarnConf: YarnConfiguration) extends ClientBase { -override def calculateAMMemory(newApp: GetNewApplicationResponse): Int = - throw new UnsupportedOperationException() - override def setupSecurityToken(amContainer: ContainerLaunchContext): Unit = throw new UnsupportedOperationException() http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala -
git commit: [SPARK-2558][DOCS] Add --queue example to YARN doc
Repository: spark Updated Branches: refs/heads/master b8634df1f -> f116f76bf [SPARK-2558][DOCS] Add --queue example to YARN doc Put original YARN queue spark-submit arg description in running-on-yarn html table and example command line Author: Mark G. Whitney Closes #2218 from kramimus/2258-yarndoc and squashes the following commits: 4b5d808 [Mark G. Whitney] remove yarn queue config f8cda0d [Mark G. Whitney] [SPARK-2558][DOCS] Add spark.yarn.queue description to YARN doc Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f116f76b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f116f76b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f116f76b Branch: refs/heads/master Commit: f116f76bf1f1610905ca094c8edc53151a78d2f4 Parents: b8634df Author: Mark G. Whitney Authored: Fri Sep 12 08:08:58 2014 -0500 Committer: Thomas Graves Committed: Fri Sep 12 08:08:58 2014 -0500 -- docs/running-on-yarn.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f116f76b/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index d8b22f3..212248b 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -155,6 +155,7 @@ For example: --driver-memory 4g \ --executor-memory 2g \ --executor-cores 1 \ +--queue thequeue \ lib/spark-examples*.jar \ 10 - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
git commit: SPARK-3014. Log a more informative messages in a couple failure scenario...
Repository: spark Updated Branches: refs/heads/master 15a564598 -> 1d767967e SPARK-3014. Log a more informative messages in a couple failure scenario... ...s Author: Sandy Ryza Closes #1934 from sryza/sandy-spark-3014 and squashes the following commits: ae19cc1 [Sandy Ryza] SPARK-3014. Log a more informative messages in a couple failure scenarios Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1d767967 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1d767967 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1d767967 Branch: refs/heads/master Commit: 1d767967e925f1d727957c2d43383ef6ad2c5d5e Parents: 15a5645 Author: Sandy Ryza Authored: Fri Sep 12 16:48:28 2014 -0500 Committer: Thomas Graves Committed: Fri Sep 12 16:48:28 2014 -0500 -- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 6 -- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 6 ++ 2 files changed, 6 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1d767967/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 0fdb5ae..5ed3575 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy import java.io.{File, PrintStream} -import java.lang.reflect.InvocationTargetException +import java.lang.reflect.{Modifier, InvocationTargetException} import java.net.URL import scala.collection.mutable.{ArrayBuffer, HashMap, Map} @@ -323,7 +323,9 @@ object SparkSubmit { } val mainMethod = mainClass.getMethod("main", new Array[String](0).getClass) - +if (!Modifier.isStatic(mainMethod.getModifiers)) { + throw new IllegalStateException("The main method in the given main class must be static") +} try { mainMethod.invoke(null, childArgs.toArray) } catch { http://git-wip-us.apache.org/repos/asf/spark/blob/1d767967/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 878b6db..735d772 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -283,11 +283,9 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, } val sparkContext = sparkContextRef.get() -assert(sparkContext != null || count >= numTries) if (sparkContext == null) { - logError( -"Unable to retrieve sparkContext inspite of waiting for %d, numTries = %d".format( - count * waitTime, numTries)) + logError(("SparkContext did not initialize after waiting for %d ms. Please check earlier" ++ " log output for errors. Failing the application.").format(numTries * waitTime)) } sparkContext } - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-3456] YarnAllocator on alpha can lose container requests to RM
Repository: spark Updated Branches: refs/heads/master af2583826 -> 25311c2c5 [SPARK-3456] YarnAllocator on alpha can lose container requests to RM Author: Thomas Graves Closes #2373 from tgravescs/SPARK-3456 and squashes the following commits: 77e9532 [Thomas Graves] [SPARK-3456] YarnAllocator on alpha can lose container requests to RM Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/25311c2c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/25311c2c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/25311c2c Branch: refs/heads/master Commit: 25311c2c545a60eb9dcf704814d4600987852155 Parents: af25838 Author: Thomas Graves Authored: Fri Sep 12 20:31:11 2014 -0500 Committer: Thomas Graves Committed: Fri Sep 12 20:31:11 2014 -0500 -- .../apache/spark/deploy/yarn/YarnAllocationHandler.scala | 11 ++- .../org/apache/spark/deploy/yarn/YarnAllocator.scala | 8 ++-- .../apache/spark/deploy/yarn/YarnAllocationHandler.scala | 3 ++- 3 files changed, 14 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/25311c2c/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala index 5a1b42c..6c93d85 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala @@ -48,16 +48,17 @@ private[yarn] class YarnAllocationHandler( private val lastResponseId = new AtomicInteger() private val releaseList: CopyOnWriteArrayList[ContainerId] = new CopyOnWriteArrayList() - override protected def allocateContainers(count: Int): YarnAllocateResponse = { + override protected def allocateContainers(count: Int, pending: Int): YarnAllocateResponse = { var resourceRequests: List[ResourceRequest] = null -logDebug("numExecutors: " + count) +logDebug("asking for additional executors: " + count + " with already pending: " + pending) +val totalNumAsk = count + pending if (count <= 0) { resourceRequests = List() } else if (preferredHostToCount.isEmpty) { logDebug("host preferences is empty") resourceRequests = List(createResourceRequest( - AllocationType.ANY, null, count, YarnSparkHadoopUtil.RM_REQUEST_PRIORITY)) + AllocationType.ANY, null, totalNumAsk, YarnSparkHadoopUtil.RM_REQUEST_PRIORITY)) } else { // request for all hosts in preferred nodes and for numExecutors - // candidates.size, request by default allocation policy. @@ -80,7 +81,7 @@ private[yarn] class YarnAllocationHandler( val anyContainerRequests: ResourceRequest = createResourceRequest( AllocationType.ANY, resource = null, -count, +totalNumAsk, YarnSparkHadoopUtil.RM_REQUEST_PRIORITY) val containerRequests: ArrayBuffer[ResourceRequest] = new ArrayBuffer[ResourceRequest]( @@ -103,7 +104,7 @@ private[yarn] class YarnAllocationHandler( req.addAllReleases(releasedContainerList) if (count > 0) { - logInfo("Allocating %d executor containers with %d of memory each.".format(count, + logInfo("Allocating %d executor containers with %d of memory each.".format(totalNumAsk, executorMemory + memoryOverhead)) } else { logDebug("Empty allocation req .. release : " + releasedContainerList) http://git-wip-us.apache.org/repos/asf/spark/blob/25311c2c/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 0b8744f..299e38a 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -112,6 +112,9 @@ private[yarn] abstract class YarnAllocator( def allocateResources() = { val missing = maxExecutors - numPendingAllocate.get() - numExecutorsRunning.get() +// this is needed by alpha, do it here since we add numPending right after this +val executorsPending = numPendingAllocate.get() + if (missing > 0) { numPendingAllocate.addAndGet(missing) logInfo("Will Allocate %d executor containers, each with %d memory".format( @@ -121,7 +124,7 @@ private[yarn] abstract class YarnAllocator( logDebug("Empty allocation req
git commit: [SPARK-3410] The priority of shutdownhook for ApplicationMaster should not be integer literal
Repository: spark Updated Branches: refs/heads/master f493f7982 -> cc1464446 [SPARK-3410] The priority of shutdownhook for ApplicationMaster should not be integer literal I think, it need to keep the priority of shutdown hook for ApplicationMaster than the priority of shutdown hook for o.a.h.FileSystem depending on changing the priority for FileSystem. Author: Kousuke Saruta Closes #2283 from sarutak/SPARK-3410 and squashes the following commits: 1d44fef [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into SPARK-3410 bd6cc53 [Kousuke Saruta] Modified style ee6f1aa [Kousuke Saruta] Added constant "SHUTDOWN_HOOK_PRIORITY" to ApplicationMaster 54eb68f [Kousuke Saruta] Changed Shutdown hook priority to 20 2f0aee3 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into SPARK-3410 4c5cb93 [Kousuke Saruta] Modified the priority for AM's shutdown hook 217d1a4 [Kousuke Saruta] Removed unused import statements 717aba2 [Kousuke Saruta] Modified ApplicationMaster to make to keep the priority of shutdown hook for ApplicationMaster higher than the priority of shutdown hook for HDFS Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cc146444 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cc146444 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cc146444 Branch: refs/heads/master Commit: cc14644460872efb344e8d895859d70213a40840 Parents: f493f79 Author: Kousuke Saruta Authored: Mon Sep 15 08:53:58 2014 -0500 Committer: Thomas Graves Committed: Mon Sep 15 08:53:58 2014 -0500 -- .../apache/spark/deploy/yarn/ApplicationMaster.scala | 13 +++-- 1 file changed, 7 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/cc146444/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 735d772..cde5fff 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -21,12 +21,8 @@ import java.io.IOException import java.net.Socket import java.util.concurrent.atomic.AtomicReference -import scala.collection.JavaConversions._ -import scala.util.Try - import akka.actor._ import akka.remote._ -import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.util.ShutdownHookManager import org.apache.hadoop.yarn.api._ @@ -107,8 +103,11 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, } } } -// Use priority 30 as it's higher than HDFS. It's the same priority MapReduce is using. -ShutdownHookManager.get().addShutdownHook(cleanupHook, 30) + +// Use higher priority than FileSystem. +assert(ApplicationMaster.SHUTDOWN_HOOK_PRIORITY > FileSystem.SHUTDOWN_HOOK_PRIORITY) +ShutdownHookManager + .get().addShutdownHook(cleanupHook, ApplicationMaster.SHUTDOWN_HOOK_PRIORITY) // Call this to force generation of secret so it gets populated into the // Hadoop UGI. This has to happen before the startUserClass which does a @@ -407,6 +406,8 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments, object ApplicationMaster extends Logging { + val SHUTDOWN_HOOK_PRIORITY: Int = 30 + private var master: ApplicationMaster = _ def main(args: Array[String]) = { - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH
Repository: spark Updated Branches: refs/heads/master c2f0821aa -> ebff7327a [SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH Based on https://github.com/apache/spark/pull/5478 that provide a PYSPARK_ARCHIVES_PATH env. within this PR, we just should export PYSPARK_ARCHIVES_PATH=/user/spark/pyspark.zip,/user/spark/python/lib/py4j-0.8.2.1-src.zip in conf/spark-env.sh when we don't install PySpark on each node of Yarn. i run python application successfully on yarn-client and yarn-cluster with this PR. andrewor14 sryza Sephiroth-Lin Can you take a look at this?thanks. Author: Lianhui Wang Closes #5580 from lianhuiwang/SPARK-6869 and squashes the following commits: 66ffa43 [Lianhui Wang] Update Client.scala c2ad0f9 [Lianhui Wang] Update Client.scala 1c8f664 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 008850a [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 f0b4ed8 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 150907b [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 20402cd [Lianhui Wang] use ZipEntry 9d87c3f [Lianhui Wang] update scala style e7bd971 [Lianhui Wang] address vanzin's comments 4b8a3ed [Lianhui Wang] use pyArchivesEnvOpt e6b573b [Lianhui Wang] address vanzin's comments f11f84a [Lianhui Wang] zip pyspark archives 5192cca [Lianhui Wang] update import path 3b1e4c8 [Lianhui Wang] address tgravescs's comments 9396346 [Lianhui Wang] put zip to make-distribution.sh 0d2baf7 [Lianhui Wang] update import paths e0179be [Lianhui Wang] add zip pyspark archives in build or sparksubmit 31e8e06 [Lianhui Wang] update code style 9f31dac [Lianhui Wang] update code and add comments f72987c [Lianhui Wang] add archives path to PYTHONPATH Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ebff7327 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ebff7327 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ebff7327 Branch: refs/heads/master Commit: ebff7327af5efa9f57c605284de4fae6b050ae0f Parents: c2f0821 Author: Lianhui Wang Authored: Fri May 8 08:44:46 2015 -0500 Committer: Thomas Graves Committed: Fri May 8 08:44:46 2015 -0500 -- assembly/pom.xml| 21 ++ .../org/apache/spark/deploy/SparkSubmit.scala | 41 project/SparkBuild.scala| 37 +- .../org/apache/spark/deploy/yarn/Client.scala | 23 --- 4 files changed, 114 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ebff7327/assembly/pom.xml -- diff --git a/assembly/pom.xml b/assembly/pom.xml index 2b4d0a9..626c857 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -92,6 +92,27 @@ true + + + org.apache.maven.plugins +maven-antrun-plugin + + +package + +run + + + + + + + + + + + + org.apache.maven.plugins http://git-wip-us.apache.org/repos/asf/spark/blob/ebff7327/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 8a03279..329fa06 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -332,6 +332,47 @@ object SparkSubmit { } } +// In yarn mode for a python app, add pyspark archives to files +// that can be distributed with the job +if (args.isPython && clusterManager == YARN) { + var pyArchives: String = null + val pyArchivesEnvOpt = sys.env.get("PYSPARK_ARCHIVES_PATH") + if (pyArchivesEnvOpt.isDefined) { +pyArchives = pyArchivesEnvOpt.get + } else { +if (!sys.env.contains("SPARK_HOME")) { + printErrorAndExit("SPARK_HOME does not exist for python application in yarn mode.") +} +val pythonPath = new ArrayBuffer[String] +for (sparkHome <- sys.env.get("SPARK_HOME")) { + val pyLibPath = Seq(sparkHome, "python", "lib").mkString(File.separator) + val pyArchivesFile = new File(pyLibPath, "pyspark.zip") + if (!pyArchivesFile.exists()) { +printErrorAndExit("pyspark.zip does not exist for pytho
spark git commit: [SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH
Repository: spark Updated Branches: refs/heads/branch-1.4 f5e9678e3 -> acf4bc1ca [SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH Based on https://github.com/apache/spark/pull/5478 that provide a PYSPARK_ARCHIVES_PATH env. within this PR, we just should export PYSPARK_ARCHIVES_PATH=/user/spark/pyspark.zip,/user/spark/python/lib/py4j-0.8.2.1-src.zip in conf/spark-env.sh when we don't install PySpark on each node of Yarn. i run python application successfully on yarn-client and yarn-cluster with this PR. andrewor14 sryza Sephiroth-Lin Can you take a look at this?thanks. Author: Lianhui Wang Closes #5580 from lianhuiwang/SPARK-6869 and squashes the following commits: 66ffa43 [Lianhui Wang] Update Client.scala c2ad0f9 [Lianhui Wang] Update Client.scala 1c8f664 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 008850a [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 f0b4ed8 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 150907b [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master' into SPARK-6869 20402cd [Lianhui Wang] use ZipEntry 9d87c3f [Lianhui Wang] update scala style e7bd971 [Lianhui Wang] address vanzin's comments 4b8a3ed [Lianhui Wang] use pyArchivesEnvOpt e6b573b [Lianhui Wang] address vanzin's comments f11f84a [Lianhui Wang] zip pyspark archives 5192cca [Lianhui Wang] update import path 3b1e4c8 [Lianhui Wang] address tgravescs's comments 9396346 [Lianhui Wang] put zip to make-distribution.sh 0d2baf7 [Lianhui Wang] update import paths e0179be [Lianhui Wang] add zip pyspark archives in build or sparksubmit 31e8e06 [Lianhui Wang] update code style 9f31dac [Lianhui Wang] update code and add comments f72987c [Lianhui Wang] add archives path to PYTHONPATH (cherry picked from commit ebff7327af5efa9f57c605284de4fae6b050ae0f) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/acf4bc1c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/acf4bc1c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/acf4bc1c Branch: refs/heads/branch-1.4 Commit: acf4bc1caa0818493f59da9991239d3caff806bd Parents: f5e9678 Author: Lianhui Wang Authored: Fri May 8 08:44:46 2015 -0500 Committer: Thomas Graves Committed: Fri May 8 08:45:13 2015 -0500 -- assembly/pom.xml| 21 ++ .../org/apache/spark/deploy/SparkSubmit.scala | 41 project/SparkBuild.scala| 37 +- .../org/apache/spark/deploy/yarn/Client.scala | 23 --- 4 files changed, 114 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/acf4bc1c/assembly/pom.xml -- diff --git a/assembly/pom.xml b/assembly/pom.xml index 2b4d0a9..626c857 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -92,6 +92,27 @@ true + + + org.apache.maven.plugins +maven-antrun-plugin + + +package + +run + + + + + + + + + + + + org.apache.maven.plugins http://git-wip-us.apache.org/repos/asf/spark/blob/acf4bc1c/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 8a03279..329fa06 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -332,6 +332,47 @@ object SparkSubmit { } } +// In yarn mode for a python app, add pyspark archives to files +// that can be distributed with the job +if (args.isPython && clusterManager == YARN) { + var pyArchives: String = null + val pyArchivesEnvOpt = sys.env.get("PYSPARK_ARCHIVES_PATH") + if (pyArchivesEnvOpt.isDefined) { +pyArchives = pyArchivesEnvOpt.get + } else { +if (!sys.env.contains("SPARK_HOME")) { + printErrorAndExit("SPARK_HOME does not exist for python application in yarn mode.") +} +val pythonPath = new ArrayBuffer[String] +for (sparkHome <- sys.env.get("SPARK_HOME")) { + val pyLibPath = Seq(sparkHome, "python", "lib").mkString(File.separator) + val pyArchivesFile = new File(pyLibPath, "pyspark.zip") +
spark git commit: [SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats.
Repository: spark Updated Branches: refs/heads/master 09265ad7c -> 3ddf051ee [SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats. Added faster RM-heartbeats on pending container allocations with multiplicative back-off. Also updated related documentations. Author: ehnalis Closes #6082 from ehnalis/yarn and squashes the following commits: a1d2101 [ehnalis] MIss-spell fixed. 90f8ba4 [ehnalis] Changed default HB values. 6120295 [ehnalis] Removed the bug, when allocation heartbeat would not start from initial value. 08bac63 [ehnalis] Refined style, grammar, removed duplicated code. 073d283 [ehnalis] [SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats. d4408c9 [ehnalis] [SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3ddf051e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3ddf051e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3ddf051e Branch: refs/heads/master Commit: 3ddf051ee7256f642f8a17768d161c7b5f55c7e1 Parents: 09265ad Author: ehnalis Authored: Wed May 20 08:27:39 2015 -0500 Committer: Thomas Graves Committed: Wed May 20 08:27:39 2015 -0500 -- docs/running-on-yarn.md | 15 - .../spark/deploy/yarn/ApplicationMaster.scala | 34 ++-- 2 files changed, 39 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3ddf051e/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 51c1339..9d55f43 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -71,9 +71,22 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.scheduler.heartbeat.interval-ms - 5000 + 3000 The interval in ms in which the Spark application master heartbeats into the YARN ResourceManager. +The value is capped at half the value of YARN's configuration for the expiry interval +(yarn.am.liveness-monitor.expiry-interval-ms). + + + + spark.yarn.scheduler.initial-allocation.interval + 200ms + +The initial interval in which the Spark application master eagerly heartbeats to the YARN ResourceManager +when there are pending container allocation requests. It should be no larger than +spark.yarn.scheduler.heartbeat.interval-ms. The allocation interval will doubled on +successive eager heartbeats if pending containers still exist, until +spark.yarn.scheduler.heartbeat.interval-ms is reached. http://git-wip-us.apache.org/repos/asf/spark/blob/3ddf051e/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 2975296..63a6f2e 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -300,11 +300,14 @@ private[spark] class ApplicationMaster( val expiryInterval = yarnConf.getInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 12) // we want to be reasonably responsive without causing too many requests to RM. -val schedulerInterval = - sparkConf.getTimeAsMs("spark.yarn.scheduler.heartbeat.interval-ms", "5s") +val heartbeatInterval = math.max(0, math.min(expiryInterval / 2, + sparkConf.getTimeAsMs("spark.yarn.scheduler.heartbeat.interval-ms", "3s"))) -// must be <= expiryInterval / 2. -val interval = math.max(0, math.min(expiryInterval / 2, schedulerInterval)) +// we want to check more frequently for pending containers +val initialAllocationInterval = math.min(heartbeatInterval, + sparkConf.getTimeAsMs("spark.yarn.scheduler.initial-allocation.interval", "200ms")) + +var nextAllocationInterval = initialAllocationInterval // The number of failures in a row until Reporter thread give up val reporterMaxFailures = sparkConf.getInt("spark.yarn.scheduler.reporterThread.maxFailures", 5) @@ -330,15 +333,27 @@ private[spark] class ApplicationMaster( if (!NonFatal(e) || failureCount >= reporterMaxFailures) { finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE, "Exception was thrown " + -s"${failureCount} time(s) from Reporter thread.") - +s"$failureCount time(s) from Reporter thread.") } else { -logWarning(s"Reporter thread fails ${failureCount} time(s) in a row.", e) +
spark git commit: [SPARK-7524] [SPARK-7846] add configs for keytab and principal, pass these two configs with different way in different modes
Repository: spark Updated Branches: refs/heads/master 8db40f671 -> a51b133de [SPARK-7524] [SPARK-7846] add configs for keytab and principal, pass these two configs with different way in different modes * As spark now supports long running service by updating tokens for namenode, but only accept parameters passed with "--k=v" format which is not very convinient. This patch add spark.* configs in properties file and system property. * --principal and --keytabl options are passed to client but when we started thrift server or spark-shell these two are also passed into the Main class (org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 and org.apache.spark.repl.Main). In these two main class, arguments passed in will be processed with some 3rd libraries, which will lead to some error: "Invalid option: --principal" or "Unrecgnised option: --principal". We should pass these command args in different forms, say system properties. Author: WangTaoTheTonic Closes #6051 from WangTaoTheTonic/SPARK-7524 and squashes the following commits: e65699a [WangTaoTheTonic] change logic to loadEnvironments ebd9ea0 [WangTaoTheTonic] merge master ecfe43a [WangTaoTheTonic] pass keytab and principal seperately in different mode 33a7f40 [WangTaoTheTonic] expand the use of the current configs 08bb4e8 [WangTaoTheTonic] fix wrong cite 73afa64 [WangTaoTheTonic] add configs for keytab and principal, move originals to internal Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a51b133d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a51b133d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a51b133d Branch: refs/heads/master Commit: a51b133de3c65a991ab105b6f020082080121b4c Parents: 8db40f6 Author: WangTaoTheTonic Authored: Fri May 29 11:06:11 2015 -0500 Committer: Thomas Graves Committed: Fri May 29 11:06:11 2015 -0500 -- .../scala/org/apache/spark/deploy/SparkSubmit.scala | 8 .../apache/spark/deploy/SparkSubmitArguments.scala | 2 ++ docs/running-on-yarn.md | 16 .../deploy/yarn/AMDelegationTokenRenewer.scala | 14 -- .../apache/spark/deploy/yarn/ClientArguments.scala | 6 ++ 5 files changed, 36 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a51b133d/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 92bb505..d1b32ea 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -428,6 +428,8 @@ object SparkSubmit { OptionAssigner(args.executorCores, YARN, CLIENT, sysProp = "spark.executor.cores"), OptionAssigner(args.files, YARN, CLIENT, sysProp = "spark.yarn.dist.files"), OptionAssigner(args.archives, YARN, CLIENT, sysProp = "spark.yarn.dist.archives"), + OptionAssigner(args.principal, YARN, CLIENT, sysProp = "spark.yarn.principal"), + OptionAssigner(args.keytab, YARN, CLIENT, sysProp = "spark.yarn.keytab"), // Yarn cluster only OptionAssigner(args.name, YARN, CLUSTER, clOption = "--name"), @@ -440,10 +442,8 @@ object SparkSubmit { OptionAssigner(args.files, YARN, CLUSTER, clOption = "--files"), OptionAssigner(args.archives, YARN, CLUSTER, clOption = "--archives"), OptionAssigner(args.jars, YARN, CLUSTER, clOption = "--addJars"), - - // Yarn client or cluster - OptionAssigner(args.principal, YARN, ALL_DEPLOY_MODES, clOption = "--principal"), - OptionAssigner(args.keytab, YARN, ALL_DEPLOY_MODES, clOption = "--keytab"), + OptionAssigner(args.principal, YARN, CLUSTER, clOption = "--principal"), + OptionAssigner(args.keytab, YARN, CLUSTER, clOption = "--keytab"), // Other options OptionAssigner(args.executorCores, STANDALONE, ALL_DEPLOY_MODES, http://git-wip-us.apache.org/repos/asf/spark/blob/a51b133d/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index c0e4c77..cc6a7bd 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -169,6 +169,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S deployMode = Option(deployMode).orElse(env.get("DEPLOY_MODE")).orNull nu
git commit: SPARK-1053. Don't require SPARK_YARN_APP_JAR
Repository: spark Updated Branches: refs/heads/master c852201ce -> b8a187195 SPARK-1053. Don't require SPARK_YARN_APP_JAR It looks this just requires taking out the checks. I verified that, with the patch, I was able to run spark-shell through yarn without setting the environment variable. Author: Sandy Ryza Closes #553 from sryza/sandy-spark-1053 and squashes the following commits: b037676 [Sandy Ryza] SPARK-1053. Don't require SPARK_YARN_APP_JAR Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b8a18719 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b8a18719 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b8a18719 Branch: refs/heads/master Commit: b8a1871953058c67b49b7f8455cbb417d5b50ab6 Parents: c852201 Author: Sandy Ryza Authored: Wed Feb 26 10:00:02 2014 -0600 Committer: Thomas Graves Committed: Wed Feb 26 10:00:02 2014 -0600 -- docs/running-on-yarn.md| 6 ++ .../scala/org/apache/spark/deploy/yarn/ClientArguments.scala | 4 ++-- .../main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 3 ++- .../spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 6 +- 4 files changed, 7 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b8a18719/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index cd4509e..ee1d892 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -99,13 +99,12 @@ With this mode, your application is actually run on the remote machine where the ## Launch spark application with yarn-client mode. -With yarn-client mode, the application will be launched locally. Just like running application or spark-shell on Local / Mesos / Standalone mode. The launch method is also the similar with them, just make sure that when you need to specify a master url, use "yarn-client" instead. And you also need to export the env value for SPARK_JAR and SPARK_YARN_APP_JAR +With yarn-client mode, the application will be launched locally. Just like running application or spark-shell on Local / Mesos / Standalone mode. The launch method is also the similar with them, just make sure that when you need to specify a master url, use "yarn-client" instead. And you also need to export the env value for SPARK_JAR. Configuration in yarn-client mode: In order to tune worker core/number/memory etc. You need to export environment variables or add them to the spark configuration file (./conf/spark_env.sh). The following are the list of options. -* `SPARK_YARN_APP_JAR`, Path to your application's JAR file (required) * `SPARK_WORKER_INSTANCES`, Number of workers to start (Default: 2) * `SPARK_WORKER_CORES`, Number of cores for the workers (Default: 1). * `SPARK_WORKER_MEMORY`, Memory per Worker (e.g. 1000M, 2G) (Default: 1G) @@ -118,12 +117,11 @@ In order to tune worker core/number/memory etc. You need to export environment v For example: SPARK_JAR=./assembly/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-assembly-{{site.SPARK_VERSION}}-hadoop2.0.5-alpha.jar \ - SPARK_YARN_APP_JAR=examples/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar \ ./bin/run-example org.apache.spark.examples.SparkPi yarn-client +or SPARK_JAR=./assembly/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-assembly-{{site.SPARK_VERSION}}-hadoop2.0.5-alpha.jar \ - SPARK_YARN_APP_JAR=examples/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar \ MASTER=yarn-client ./bin/spark-shell http://git-wip-us.apache.org/repos/asf/spark/blob/b8a18719/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala index 1419f21..fe37168 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala @@ -108,7 +108,7 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) { args = tail case Nil => - if (userJar == null || userClass == null) { + if (userClass == null) { printUsageAndExit(1) } @@ -129,7 +129,7 @@ class ClientArguments(val args: Array[String], val sparkConf: SparkConf) { System.err.println( "Usage: org.apache.spark.deploy.yarn.Client [options] \n" + "Options:\n" + - " --jar
git commit: SPARK-1032. If Yarn app fails before registering, app master stays aroun...
Repository: spark Updated Branches: refs/heads/master edf8a56ab -> 5f419bf9f SPARK-1032. If Yarn app fails before registering, app master stays aroun... ...d long after This reopens https://github.com/apache/incubator-spark/pull/648 against the new repo. Author: Sandy Ryza Closes #28 from sryza/sandy-spark-1032 and squashes the following commits: 5953f50 [Sandy Ryza] SPARK-1032. If Yarn app fails before registering, app master stays around long after Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5f419bf9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5f419bf9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5f419bf9 Branch: refs/heads/master Commit: 5f419bf9f433e8f057237f1d5bfed9f5f4e9427c Parents: edf8a56 Author: Sandy Ryza Authored: Fri Feb 28 09:40:47 2014 -0600 Committer: Thomas Graves Committed: Fri Feb 28 09:40:47 2014 -0600 -- .../spark/deploy/yarn/ApplicationMaster.scala | 34 +--- .../spark/deploy/yarn/ApplicationMaster.scala | 22 + 2 files changed, 38 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/5f419bf9/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 2e46d75..560e5de 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -65,6 +65,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, private val maxNumWorkerFailures = sparkConf.getInt("spark.yarn.max.worker.failures", math.max(args.numWorkers * 2, 3)) + private var registered = false + def run() { // Setup the directories so things go to yarn approved directories rather // then user specified and /tmp. @@ -110,7 +112,12 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, waitForSparkContextInitialized() // Do this after spark master is up and SparkContext is created so that we can register UI Url -val appMasterResponse: RegisterApplicationMasterResponse = registerApplicationMaster() +synchronized { + if (!isFinished) { +registerApplicationMaster() +registered = true + } +} // Allocate all containers allocateWorkers() @@ -208,7 +215,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, var count = 0 val waitTime = 1L val numTries = sparkConf.getInt("spark.yarn.ApplicationMaster.waitTries", 10) -while (ApplicationMaster.sparkContextRef.get() == null && count < numTries) { +while (ApplicationMaster.sparkContextRef.get() == null && count < numTries +&& !isFinished) { logInfo("Waiting for spark context initialization ... " + count) count = count + 1 ApplicationMaster.sparkContextRef.wait(waitTime) @@ -341,17 +349,19 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, return } isFinished = true + + logInfo("finishApplicationMaster with " + status) + if (registered) { +val finishReq = Records.newRecord(classOf[FinishApplicationMasterRequest]) + .asInstanceOf[FinishApplicationMasterRequest] +finishReq.setAppAttemptId(appAttemptId) +finishReq.setFinishApplicationStatus(status) +finishReq.setDiagnostics(diagnostics) +// Set tracking url to empty since we don't have a history server. +finishReq.setTrackingUrl("") +resourceManager.finishApplicationMaster(finishReq) + } } - -logInfo("finishApplicationMaster with " + status) -val finishReq = Records.newRecord(classOf[FinishApplicationMasterRequest]) - .asInstanceOf[FinishApplicationMasterRequest] -finishReq.setAppAttemptId(appAttemptId) -finishReq.setFinishApplicationStatus(status) -finishReq.setDiagnostics(diagnostics) -// Set tracking url to empty since we don't have a history server. -finishReq.setTrackingUrl("") -resourceManager.finishApplicationMaster(finishReq) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/5f419bf9/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Appl
git commit: SPARK-1051. On YARN, executors don't doAs submitting user
Repository: spark Updated Branches: refs/heads/master 5f419bf9f -> 46dff3445 SPARK-1051. On YARN, executors don't doAs submitting user This reopens https://github.com/apache/incubator-spark/pull/538 against the new repo Author: Sandy Ryza Closes #29 from sryza/sandy-spark-1051 and squashes the following commits: 708ce49 [Sandy Ryza] SPARK-1051. doAs submitting user in YARN Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/46dff344 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/46dff344 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/46dff344 Branch: refs/heads/master Commit: 46dff34458096e5330073ca58e0723da52aeddcd Parents: 5f419bf Author: Sandy Ryza Authored: Fri Feb 28 12:43:01 2014 -0600 Committer: Thomas Graves Committed: Fri Feb 28 12:43:01 2014 -0600 -- .../org/apache/spark/deploy/SparkHadoopUtil.scala | 18 ++ .../spark/deploy/yarn/ApplicationMaster.scala | 6 +- .../org/apache/spark/deploy/yarn/ClientBase.scala | 1 + .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 4 .../spark/deploy/yarn/ApplicationMaster.scala | 6 +- 5 files changed, 25 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/46dff344/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala index b479225..ec15647 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala @@ -25,6 +25,8 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.{SparkContext, SparkException} +import scala.collection.JavaConversions._ + /** * Contains util methods to interact with Hadoop from Spark. */ @@ -33,15 +35,9 @@ class SparkHadoopUtil { UserGroupInformation.setConfiguration(conf) def runAsUser(user: String)(func: () => Unit) { -// if we are already running as the user intended there is no reason to do the doAs. It -// will actually break secure HDFS access as it doesn't fill in the credentials. Also if -// the user is UNKNOWN then we shouldn't be creating a remote unknown user -// (this is actually the path spark on yarn takes) since SPARK_USER is initialized only -// in SparkContext. -val currentUser = Option(System.getProperty("user.name")). - getOrElse(SparkContext.SPARK_UNKNOWN_USER) -if (user != SparkContext.SPARK_UNKNOWN_USER && currentUser != user) { +if (user != SparkContext.SPARK_UNKNOWN_USER) { val ugi = UserGroupInformation.createRemoteUser(user) + transferCredentials(UserGroupInformation.getCurrentUser(), ugi) ugi.doAs(new PrivilegedExceptionAction[Unit] { def run: Unit = func() }) @@ -50,6 +46,12 @@ class SparkHadoopUtil { } } + def transferCredentials(source: UserGroupInformation, dest: UserGroupInformation) { +for (token <- source.getTokens()) { + dest.addToken(token) +} + } + /** * Return an appropriate (subclass) of Configuration. Creating config can initializes some Hadoop * subsystems. http://git-wip-us.apache.org/repos/asf/spark/blob/46dff344/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 560e5de..e045b9f 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{ConverterUtils, Records} import org.apache.spark.{SparkConf, SparkContext, Logging} +import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.util.Utils class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, @@ -67,6 +68,9 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, private var registered = false + private val sparkUser = Option(System.getenv("SPARK_USER")).getOrElse( +SparkContext.SPARK_UNKNOWN_USER) + def run() { // Setup the directories so things go to yarn approved directories rather // then user specified and /tmp. @@ -180,7 +184,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, false /* initialize */ , Thread.currentThread.getContextC
[1/3] SPARK-1189: Add Security to Spark - Akka, Http, ConnectionManager, UI use servlets
Repository: spark Updated Branches: refs/heads/master 40566e10a -> 7edbea41b http://git-wip-us.apache.org/repos/asf/spark/blob/7edbea41/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala b/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala index c1e8b29..96a5a12 100644 --- a/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala +++ b/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala @@ -18,21 +18,22 @@ package org.apache.spark.metrics import org.scalatest.{BeforeAndAfter, FunSuite} - -import org.apache.spark.SparkConf +import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.deploy.master.MasterSource class MetricsSystemSuite extends FunSuite with BeforeAndAfter { var filePath: String = _ var conf: SparkConf = null + var securityMgr: SecurityManager = null before { filePath = getClass.getClassLoader.getResource("test_metrics_system.properties").getFile() conf = new SparkConf(false).set("spark.metrics.conf", filePath) +securityMgr = new SecurityManager(conf) } test("MetricsSystem with default config") { -val metricsSystem = MetricsSystem.createMetricsSystem("default", conf) +val metricsSystem = MetricsSystem.createMetricsSystem("default", conf, securityMgr) val sources = metricsSystem.sources val sinks = metricsSystem.sinks @@ -42,7 +43,7 @@ class MetricsSystemSuite extends FunSuite with BeforeAndAfter { } test("MetricsSystem with sources add") { -val metricsSystem = MetricsSystem.createMetricsSystem("test", conf) +val metricsSystem = MetricsSystem.createMetricsSystem("test", conf, securityMgr) val sources = metricsSystem.sources val sinks = metricsSystem.sinks http://git-wip-us.apache.org/repos/asf/spark/blob/7edbea41/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala index 9f011d9..121e47c 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala @@ -28,7 +28,7 @@ import org.scalatest.concurrent.Timeouts._ import org.scalatest.matchers.ShouldMatchers._ import org.scalatest.time.SpanSugar._ -import org.apache.spark.{SparkConf, SparkContext} +import org.apache.spark.{SecurityManager, SparkConf, SparkContext} import org.apache.spark.serializer.{JavaSerializer, KryoSerializer} import org.apache.spark.util.{AkkaUtils, ByteBufferInputStream, SizeEstimator, Utils} @@ -39,6 +39,8 @@ class BlockManagerSuite extends FunSuite with BeforeAndAfter with PrivateMethodT var actorSystem: ActorSystem = null var master: BlockManagerMaster = null var oldArch: String = null + conf.set("spark.authenticate", "false") + val securityMgr = new SecurityManager(conf) // Reuse a serializer across tests to avoid creating a new thread-local buffer on each test conf.set("spark.kryoserializer.buffer.mb", "1") @@ -49,7 +51,8 @@ class BlockManagerSuite extends FunSuite with BeforeAndAfter with PrivateMethodT def rdd(rddId: Int, splitId: Int) = RDDBlockId(rddId, splitId) before { -val (actorSystem, boundPort) = AkkaUtils.createActorSystem("test", "localhost", 0, conf = conf) +val (actorSystem, boundPort) = AkkaUtils.createActorSystem("test", "localhost", 0, conf = conf, + securityManager = securityMgr) this.actorSystem = actorSystem conf.set("spark.driver.port", boundPort.toString) @@ -125,7 +128,7 @@ class BlockManagerSuite extends FunSuite with BeforeAndAfter with PrivateMethodT } test("master + 1 manager interaction") { -store = new BlockManager("", actorSystem, master, serializer, 2000, conf) +store = new BlockManager("", actorSystem, master, serializer, 2000, conf, securityMgr) val a1 = new Array[Byte](400) val a2 = new Array[Byte](400) val a3 = new Array[Byte](400) @@ -155,8 +158,9 @@ class BlockManagerSuite extends FunSuite with BeforeAndAfter with PrivateMethodT } test("master + 2 managers interaction") { -store = new BlockManager("exec1", actorSystem, master, serializer, 2000, conf) -store2 = new BlockManager("exec2", actorSystem, master, new KryoSerializer(conf), 2000, conf) +store = new BlockManager("exec1", actorSystem, master, serializer, 2000, conf, securityMgr) +store2 = new BlockManager("exec2", actorSystem, master, new KryoSerializer(conf), 2000, conf, + securityMgr) val peers = master.getPeers(store.blockManagerId, 1) assert(peers.size === 1, "master did not return the other manager as a peer") @@ -1
git commit: SPARK-1203 fix saving to hdfs from yarn
Repository: spark Updated Branches: refs/heads/master d55ec86de -> 6112270c9 SPARK-1203 fix saving to hdfs from yarn Author: Thomas Graves Closes #173 from tgravescs/SPARK-1203 and squashes the following commits: 4fd5ded [Thomas Graves] adding import 964e3f7 [Thomas Graves] SPARK-1203 fix saving to hdfs from yarn Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6112270c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6112270c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6112270c Branch: refs/heads/master Commit: 6112270c94f1a30a461a91f6e56485a5eaec2606 Parents: d55ec86 Author: Thomas Graves Authored: Wed Mar 19 08:09:20 2014 -0500 Committer: Thomas Graves Committed: Wed Mar 19 08:09:20 2014 -0500 -- core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6112270c/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 447deaf..75fc02a 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat => NewFileOutput import org.apache.hadoop.mapred.SparkHadoopWriter import org.apache.spark._ +import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.Partitioner.defaultPartitioner import org.apache.spark.SparkContext._ import org.apache.spark.partial.{BoundedDouble, PartialResult} @@ -723,6 +724,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) if (valueClass == null) { throw new SparkException("Output value class not set") } +SparkHadoopUtil.get.addCredentials(conf) logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", " + valueClass.getSimpleName + ")")
git commit: SPARK-1203 fix saving to hdfs from yarn
Repository: spark Updated Branches: refs/heads/branch-0.9 d385b5a19 -> 250ec271c SPARK-1203 fix saving to hdfs from yarn Author: Thomas Graves Closes #173 from tgravescs/SPARK-1203 and squashes the following commits: 4fd5ded [Thomas Graves] adding import 964e3f7 [Thomas Graves] SPARK-1203 fix saving to hdfs from yarn Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/250ec271 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/250ec271 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/250ec271 Branch: refs/heads/branch-0.9 Commit: 250ec271c6136b881f7ffcb9ce746c30d4abbf3c Parents: d385b5a Author: Thomas Graves Authored: Wed Mar 19 08:09:20 2014 -0500 Committer: Thomas Graves Committed: Wed Mar 19 08:19:47 2014 -0500 -- core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/250ec271/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala -- diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 0b2917b..0bc09b3 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -44,6 +44,7 @@ import com.clearspring.analytics.stream.cardinality.HyperLogLog import org.apache.hadoop.mapred.SparkHadoopWriter import org.apache.hadoop.mapreduce.SparkHadoopMapReduceUtil import org.apache.spark._ +import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.SparkContext._ import org.apache.spark.partial.{BoundedDouble, PartialResult} import org.apache.spark.Partitioner.defaultPartitioner @@ -710,6 +711,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) if (valueClass == null) { throw new SparkException("Output value class not set") } +SparkHadoopUtil.get.addCredentials(conf) logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", " + valueClass.getSimpleName+ ")")
git commit: Update the yarn alpha version to 0.9.1-incubating-SNAPSHOT
Repository: spark Updated Branches: refs/heads/branch-0.9 250ec271c -> 72875b29b Update the yarn alpha version to 0.9.1-incubating-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/72875b29 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/72875b29 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/72875b29 Branch: refs/heads/branch-0.9 Commit: 72875b29bb69f654f1dc33e50cc6431a1cfbc0a0 Parents: 250ec27 Author: Thomas Graves Authored: Wed Mar 19 12:41:11 2014 -0500 Committer: Thomas Graves Committed: Wed Mar 19 12:41:11 2014 -0500 -- dev/merge_spark_pr.py | 2 ++ yarn/alpha/pom.xml| 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/72875b29/dev/merge_spark_pr.py -- diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py index e8f78fc..e40a3b2 100755 --- a/dev/merge_spark_pr.py +++ b/dev/merge_spark_pr.py @@ -38,6 +38,8 @@ PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github") # Remote name which points to Apache git PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache") +print "%s" % PUSH_REMOTE_NAME + GIT_API_BASE = "https://api.github.com/repos/apache/spark"; # Prefix added to temporary branches BRANCH_PREFIX = "PR_TOOL" http://git-wip-us.apache.org/repos/asf/spark/blob/72875b29/yarn/alpha/pom.xml -- diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml index 349c835..8d60503 100644 --- a/yarn/alpha/pom.xml +++ b/yarn/alpha/pom.xml @@ -20,7 +20,7 @@ org.apache.spark yarn-parent_2.10 -0.9.0-incubating +0.9.1-incubating-SNAPSHOT ../pom.xml
git commit: SPARK-1051. On YARN, executors don't doAs submitting user
Repository: spark Updated Branches: refs/heads/branch-0.9 1e366904c -> 748f002b3 SPARK-1051. On YARN, executors don't doAs submitting user This reopens https://github.com/apache/incubator-spark/pull/538 against the new repo Author: Sandy Ryza Closes #29 from sryza/sandy-spark-1051 and squashes the following commits: 708ce49 [Sandy Ryza] SPARK-1051. doAs submitting user in YARN Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/748f002b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/748f002b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/748f002b Branch: refs/heads/branch-0.9 Commit: 748f002b3a58af118f7de0b6ea9170895b571c78 Parents: 1e36690 Author: Sandy Ryza Authored: Fri Feb 28 12:43:01 2014 -0600 Committer: Thomas Graves Committed: Thu Mar 20 14:48:05 2014 -0500 -- .../org/apache/spark/deploy/SparkHadoopUtil.scala | 18 ++ .../spark/deploy/yarn/ApplicationMaster.scala | 6 +- .../org/apache/spark/deploy/yarn/Client.scala | 1 + .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 4 .../spark/deploy/yarn/ApplicationMaster.scala | 6 +- .../org/apache/spark/deploy/yarn/Client.scala | 1 + 6 files changed, 26 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/748f002b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala index b479225..ec15647 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala @@ -25,6 +25,8 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.{SparkContext, SparkException} +import scala.collection.JavaConversions._ + /** * Contains util methods to interact with Hadoop from Spark. */ @@ -33,15 +35,9 @@ class SparkHadoopUtil { UserGroupInformation.setConfiguration(conf) def runAsUser(user: String)(func: () => Unit) { -// if we are already running as the user intended there is no reason to do the doAs. It -// will actually break secure HDFS access as it doesn't fill in the credentials. Also if -// the user is UNKNOWN then we shouldn't be creating a remote unknown user -// (this is actually the path spark on yarn takes) since SPARK_USER is initialized only -// in SparkContext. -val currentUser = Option(System.getProperty("user.name")). - getOrElse(SparkContext.SPARK_UNKNOWN_USER) -if (user != SparkContext.SPARK_UNKNOWN_USER && currentUser != user) { +if (user != SparkContext.SPARK_UNKNOWN_USER) { val ugi = UserGroupInformation.createRemoteUser(user) + transferCredentials(UserGroupInformation.getCurrentUser(), ugi) ugi.doAs(new PrivilegedExceptionAction[Unit] { def run: Unit = func() }) @@ -50,6 +46,12 @@ class SparkHadoopUtil { } } + def transferCredentials(source: UserGroupInformation, dest: UserGroupInformation) { +for (token <- source.getTokens()) { + dest.addToken(token) +} + } + /** * Return an appropriate (subclass) of Configuration. Creating config can initializes some Hadoop * subsystems. http://git-wip-us.apache.org/repos/asf/spark/blob/748f002b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 2e46d75..7aa1894 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{ConverterUtils, Records} import org.apache.spark.{SparkConf, SparkContext, Logging} +import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.util.Utils class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, @@ -65,6 +66,9 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, private val maxNumWorkerFailures = sparkConf.getInt("spark.yarn.max.worker.failures", math.max(args.numWorkers * 2, 3)) + private val sparkUser = Option(System.getenv("SPARK_USER")).getOrElse( +SparkContext.SPARK_UNKNOWN_USER) + def run() { // Setup the directories so things go to yarn approved directories rather // then user specified and /tmp. @@ -173,7 +1
git commit: SPARK-1032. If Yarn app fails before registering, app master stays aroun...
Repository: spark Updated Branches: refs/heads/branch-0.9 748f002b3 -> c6630d363 SPARK-1032. If Yarn app fails before registering, app master stays aroun... ...d long after This reopens https://github.com/apache/incubator-spark/pull/648 against the new repo. Author: Sandy Ryza Closes #28 from sryza/sandy-spark-1032 and squashes the following commits: 5953f50 [Sandy Ryza] SPARK-1032. If Yarn app fails before registering, app master stays around long after Conflicts: yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c6630d36 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c6630d36 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c6630d36 Branch: refs/heads/branch-0.9 Commit: c6630d363a73184c8dcca9f2c0c6fc3f5c8e47bf Parents: 748f002 Author: Sandy Ryza Authored: Fri Feb 28 09:40:47 2014 -0600 Committer: Thomas Graves Committed: Thu Mar 20 16:50:44 2014 -0500 -- .../spark/deploy/yarn/ApplicationMaster.scala | 34 +--- .../spark/deploy/yarn/ApplicationMaster.scala | 22 + 2 files changed, 38 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/c6630d36/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 7aa1894..e045b9f 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -66,6 +66,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, private val maxNumWorkerFailures = sparkConf.getInt("spark.yarn.max.worker.failures", math.max(args.numWorkers * 2, 3)) + private var registered = false + private val sparkUser = Option(System.getenv("SPARK_USER")).getOrElse( SparkContext.SPARK_UNKNOWN_USER) @@ -114,7 +116,12 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, waitForSparkContextInitialized() // Do this after spark master is up and SparkContext is created so that we can register UI Url -val appMasterResponse: RegisterApplicationMasterResponse = registerApplicationMaster() +synchronized { + if (!isFinished) { +registerApplicationMaster() +registered = true + } +} // Allocate all containers allocateWorkers() @@ -212,7 +219,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, var count = 0 val waitTime = 1L val numTries = sparkConf.getInt("spark.yarn.ApplicationMaster.waitTries", 10) -while (ApplicationMaster.sparkContextRef.get() == null && count < numTries) { +while (ApplicationMaster.sparkContextRef.get() == null && count < numTries +&& !isFinished) { logInfo("Waiting for spark context initialization ... " + count) count = count + 1 ApplicationMaster.sparkContextRef.wait(waitTime) @@ -345,17 +353,19 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, return } isFinished = true + + logInfo("finishApplicationMaster with " + status) + if (registered) { +val finishReq = Records.newRecord(classOf[FinishApplicationMasterRequest]) + .asInstanceOf[FinishApplicationMasterRequest] +finishReq.setAppAttemptId(appAttemptId) +finishReq.setFinishApplicationStatus(status) +finishReq.setDiagnostics(diagnostics) +// Set tracking url to empty since we don't have a history server. +finishReq.setTrackingUrl("") +resourceManager.finishApplicationMaster(finishReq) + } } - -logInfo("finishApplicationMaster with " + status) -val finishReq = Records.newRecord(classOf[FinishApplicationMasterRequest]) - .asInstanceOf[FinishApplicationMasterRequest] -finishReq.setAppAttemptId(appAttemptId) -finishReq.setFinishApplicationStatus(status) -finishReq.setDiagnostics(diagnostics) -// Set tracking url to empty since we don't have a history server. -finishReq.setTrackingUrl("") -resourceManager.finishApplicationMaster(finishReq) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/c6630d36/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala --
git commit: SPARK-1330 removed extra echo from comput_classpath.sh
Repository: spark Updated Branches: refs/heads/master 5b2d863e3 -> 426042ad2 SPARK-1330 removed extra echo from comput_classpath.sh remove the extra echo which prevents spark-class from working. Note that I did not update the comment above it, which is also wrong because I'm not sure what it should do. Should hive only be included if explicitly built with sbt hive/assembly or should sbt assembly build it? Author: Thomas Graves Closes #241 from tgravescs/SPARK-1330 and squashes the following commits: b10d708 [Thomas Graves] SPARK-1330 removed extra echo from comput_classpath.sh Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/426042ad Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/426042ad Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/426042ad Branch: refs/heads/master Commit: 426042ad24a54b4b776085cbf4e1896464efc613 Parents: 5b2d863 Author: Thomas Graves Authored: Thu Mar 27 11:54:43 2014 -0500 Committer: Thomas Graves Committed: Thu Mar 27 11:54:43 2014 -0500 -- bin/compute-classpath.sh | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/426042ad/bin/compute-classpath.sh -- diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index d6f1ff9..bef42df 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -36,7 +36,6 @@ CLASSPATH="$SPARK_CLASSPATH:$FWDIR/conf" # Hopefully we will find a way to avoid uber-jars entirely and deploy only the needed packages in # the future. if [ -f "$FWDIR"/sql/hive/target/scala-$SCALA_VERSION/spark-hive-assembly-*.jar ]; then - echo "Hive assembly found, including hive support. If this isn't desired run sbt hive/clean." # Datanucleus jars do not work if only included in the uberjar as plugin.xml metadata is lost. DATANUCLEUSJARS=$(JARS=("$FWDIR/lib_managed/jars"/datanucleus-*.jar); IFS=:; echo "${JARS[*]}")
git commit: SPARK-1350. Always use JAVA_HOME to run executor container JVMs.
Repository: spark Updated Branches: refs/heads/master ee6e9e7d8 -> 7f32fd42a SPARK-1350. Always use JAVA_HOME to run executor container JVMs. Author: Sandy Ryza Closes #313 from sryza/sandy-spark-1350 and squashes the following commits: bb6d187 [Sandy Ryza] SPARK-1350. Always use JAVA_HOME to run executor container JVMs. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7f32fd42 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7f32fd42 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7f32fd42 Branch: refs/heads/master Commit: 7f32fd42aaadcf6626b4d8bcf6295203b43b2037 Parents: ee6e9e7 Author: Sandy Ryza Authored: Fri Apr 4 08:54:04 2014 -0500 Committer: Thomas Graves Committed: Fri Apr 4 08:54:04 2014 -0500 -- .../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 8 +--- .../org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala | 9 ++--- 2 files changed, 3 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/7f32fd42/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 6568003..bc26790 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -345,14 +345,8 @@ trait ClientBase extends Logging { } // Command for the ApplicationMaster -var javaCommand = "java" -val javaHome = System.getenv("JAVA_HOME") -if ((javaHome != null && !javaHome.isEmpty()) || env.isDefinedAt("JAVA_HOME")) { - javaCommand = Environment.JAVA_HOME.$() + "/bin/java" -} - val commands = List[String]( - javaCommand + + Environment.JAVA_HOME.$() + "/bin/java" + " -server " + JAVA_OPTS + " " + args.amClass + http://git-wip-us.apache.org/repos/asf/spark/blob/7f32fd42/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala index da0a6f7..2079697 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala @@ -88,13 +88,8 @@ trait ExecutorRunnableUtil extends Logging { } */ -var javaCommand = "java" -val javaHome = System.getenv("JAVA_HOME") -if ((javaHome != null && !javaHome.isEmpty()) || env.isDefinedAt("JAVA_HOME")) { - javaCommand = Environment.JAVA_HOME.$() + "/bin/java" -} - -val commands = List[String](javaCommand + +val commands = List[String]( + Environment.JAVA_HOME.$() + "/bin/java" + " -server " + // Kill if OOM is raised - leverage yarn's failure handling to cause rescheduling. // Not killing the task leaves various aspects of the executor and (to some extent) the jvm in
git commit: SPARK-1252. On YARN, use container-log4j.properties for executors
Repository: spark Updated Branches: refs/heads/master 83f2a2f14 -> 9dd8b9166 SPARK-1252. On YARN, use container-log4j.properties for executors container-log4j.properties is a file that YARN provides so that containers can have log4j.properties distinct from that of the NodeManagers. Logs now go to syslog, and stderr and stdout just have the process's standard err and standard out. I tested this on pseudo-distributed clusters for both yarn (Hadoop 2.2) and yarn-alpha (Hadoop 0.23.7)/ Author: Sandy Ryza Closes #148 from sryza/sandy-spark-1252 and squashes the following commits: c0043b8 [Sandy Ryza] Put log4j.properties file under common 55823da [Sandy Ryza] Add license headers to new files 10934b8 [Sandy Ryza] Add log4j-spark-container.properties and support SPARK_LOG4J_CONF e74450b [Sandy Ryza] SPARK-1252. On YARN, use container-log4j.properties for executors Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9dd8b916 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9dd8b916 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9dd8b916 Branch: refs/heads/master Commit: 9dd8b9166225cbcfcab1f39268244c2feb42a658 Parents: 83f2a2f Author: Sandy Ryza Authored: Mon Apr 7 13:28:14 2014 -0500 Committer: Thomas Graves Committed: Mon Apr 7 13:28:14 2014 -0500 -- .../spark/deploy/yarn/ExecutorRunnable.scala| 3 ++- .../resources/log4j-spark-container.properties | 24 .../apache/spark/deploy/yarn/ClientBase.scala | 8 +-- .../deploy/yarn/ExecutorRunnableUtil.scala | 7 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 7 ++ yarn/pom.xml| 6 + .../spark/deploy/yarn/ExecutorRunnable.scala| 3 ++- 7 files changed, 53 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9dd8b916/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 981e8b0..3469b7d 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -81,7 +81,8 @@ class ExecutorRunnable( credentials.writeTokenStorageToStream(dob) ctx.setContainerTokens(ByteBuffer.wrap(dob.getData())) -val commands = prepareCommand(masterAddress, slaveId, hostname, executorMemory, executorCores) +val commands = prepareCommand(masterAddress, slaveId, hostname, executorMemory, executorCores, + localResources.contains(ClientBase.LOG4J_PROP)) logInfo("Setting up executor with commands: " + commands) ctx.setCommands(commands) http://git-wip-us.apache.org/repos/asf/spark/blob/9dd8b916/yarn/common/src/main/resources/log4j-spark-container.properties -- diff --git a/yarn/common/src/main/resources/log4j-spark-container.properties b/yarn/common/src/main/resources/log4j-spark-container.properties new file mode 100644 index 000..a1e37a0 --- /dev/null +++ b/yarn/common/src/main/resources/log4j-spark-container.properties @@ -0,0 +1,24 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +# Settings to quiet third party logs that are too verbose +log4j.logger.org.eclipse.jetty=WARN +log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO +log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO http://git-wip-us.apache.org/repos/asf/spark/blob/9dd8b916/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/Client
git commit: SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken
Repository: spark Updated Branches: refs/heads/branch-1.0 a1bb4c695 -> 9e90c464d SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken Author: Thomas Graves Closes #344 from tgravescs/SPARK-1417 and squashes the following commits: c450b5f [Thomas Graves] fix test e1c1d7e [Thomas Graves] add missing $ to appUIAddress e982ddb [Thomas Graves] use appUIHostPort in appUIAddress 0803ec2 [Thomas Graves] Review comment updates - remove extra newline, simplify assert in test 658a8ec [Thomas Graves] Add a appUIHostPort routine 0614208 [Thomas Graves] Fix test 2a6b1b7 [Thomas Graves] SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9e90c464 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9e90c464 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9e90c464 Branch: refs/heads/branch-1.0 Commit: 9e90c464dec3258b57ecad828d7a8609eb5528aa Parents: a1bb4c6 Author: Thomas Graves Authored: Fri Apr 11 13:17:48 2014 +0530 Committer: Thomas Graves Committed: Fri Apr 11 10:27:37 2014 -0500 -- .../scala/org/apache/spark/ui/SparkUI.scala | 7 +++- .../scala/org/apache/spark/SparkUISuite.scala | 35 .../spark/deploy/yarn/ApplicationMaster.scala | 2 +- .../spark/deploy/yarn/ApplicationMaster.scala | 2 +- 4 files changed, 43 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/9e90c464/core/src/main/scala/org/apache/spark/ui/SparkUI.scala -- diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index 4c891d7..7fa4fd3 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -113,7 +113,12 @@ private[spark] class SparkUI( logInfo("Stopped Spark Web UI at %s".format(appUIAddress)) } - private[spark] def appUIAddress = "http://"; + publicHost + ":" + boundPort + /** + * Return the application UI host:port. This does not include the scheme (http://). + */ + private[spark] def appUIHostPort = publicHost + ":" + boundPort + + private[spark] def appUIAddress = s"http://$appUIHostPort"; } http://git-wip-us.apache.org/repos/asf/spark/blob/9e90c464/core/src/test/scala/org/apache/spark/SparkUISuite.scala -- diff --git a/core/src/test/scala/org/apache/spark/SparkUISuite.scala b/core/src/test/scala/org/apache/spark/SparkUISuite.scala new file mode 100644 index 000..d0d119c --- /dev/null +++ b/core/src/test/scala/org/apache/spark/SparkUISuite.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark + +import java.net.URI + +import org.scalatest.FunSuite + +class SparkUISuite extends FunSuite with SharedSparkContext { + + test("verify appUIAddress contains the scheme") { +val uiAddress = sc.ui.appUIAddress +assert(uiAddress.equals("http://"; + sc.ui.appUIHostPort)) + } + + test("verify appUIAddress contains the port") { +val splitUIAddress = sc.ui.appUIAddress.split(':') +assert(splitUIAddress(2).toInt == sc.ui.boundPort) + } +} http://git-wip-us.apache.org/repos/asf/spark/blob/9e90c464/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 910484e..67ec95c 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -234,7 +234,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, assert(sparkContext != null || c
git commit: SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release
Repository: spark Updated Branches: refs/heads/master e269c24db -> 725925cf2 SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release YARN-1824 changes the APIs (addToEnvironment, setEnvFromInputString) in Apps, which causes the spark build to break if built against a version 2.4.0. To fix this, create the spark own function to do that functionality which will not break compiling against 2.3 and other 2.x versions. Author: xuan Author: xuan Closes #396 from xgong/master and squashes the following commits: 42b5984 [xuan] Remove two extra imports bc0926f [xuan] Remove usage of org.apache.hadoop.util.Shell be89fa7 [xuan] fix Spark compilation is broken with the latest hadoop-2.4.0 release Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/725925cf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/725925cf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/725925cf Branch: refs/heads/master Commit: 725925cf2120e998651f7d1406fdb34fc2405b9f Parents: e269c24 Author: xuan Authored: Wed Apr 16 14:41:22 2014 -0500 Committer: Thomas Graves Committed: Wed Apr 16 14:41:22 2014 -0500 -- .../apache/spark/deploy/yarn/ClientBase.scala | 33 +- .../deploy/yarn/ExecutorRunnableUtil.scala | 4 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 63 3 files changed, 85 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/725925cf/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index eb42922..628dd98 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -17,6 +17,7 @@ package org.apache.spark.deploy.yarn +import java.io.File import java.net.{InetAddress, UnknownHostException, URI} import java.nio.ByteBuffer @@ -280,7 +281,8 @@ trait ClientBase extends Logging { distCacheMgr.setDistArchivesEnv(env) // Allow users to specify some environment variables. -Apps.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV")) +YarnSparkHadoopUtil.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV"), + File.pathSeparator) // Add each SPARK_* key to the environment. System.getenv().filterKeys(_.startsWith("SPARK")).foreach { case (k,v) => env(k) = v } @@ -382,7 +384,8 @@ object ClientBase { YarnConfiguration.YARN_APPLICATION_CLASSPATH)).getOrElse( getDefaultYarnApplicationClasspath()) for (c <- classpathEntries) { - Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim) + YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, c.trim, +File.pathSeparator) } val mrClasspathEntries = Option(conf.getStrings( @@ -390,7 +393,8 @@ object ClientBase { getDefaultMRApplicationClasspath()) if (mrClasspathEntries != null) { for (c <- mrClasspathEntries) { -Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim) +YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, c.trim, + File.pathSeparator) } } } @@ -425,28 +429,29 @@ object ClientBase { } def populateClasspath(conf: Configuration, sparkConf: SparkConf, addLog4j: Boolean, env: HashMap[String, String]) { -Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$()) +YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$(), + File.pathSeparator) // If log4j present, ensure ours overrides all others if (addLog4j) { - Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + -Path.SEPARATOR + LOG4J_PROP) + YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + +Path.SEPARATOR + LOG4J_PROP, File.pathSeparator) } // Normally the users app.jar is last in case conflicts with spark jars val userClasspathFirst = sparkConf.get("spark.yarn.user.classpath.first", "false") .toBoolean if (userClasspathFirst) { - Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + -Path.SEPARATOR + APP_JAR) + YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + +Path.SEPARATOR + APP_JAR, File.pathSeparator) } -Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + - Path.SEPARATOR + SPARK_JAR) +YarnSparkH
git commit: SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release
Repository: spark Updated Branches: refs/heads/branch-1.0 b75301f1f -> d8fc4a4b6 SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release YARN-1824 changes the APIs (addToEnvironment, setEnvFromInputString) in Apps, which causes the spark build to break if built against a version 2.4.0. To fix this, create the spark own function to do that functionality which will not break compiling against 2.3 and other 2.x versions. Author: xuan Author: xuan Closes #396 from xgong/master and squashes the following commits: 42b5984 [xuan] Remove two extra imports bc0926f [xuan] Remove usage of org.apache.hadoop.util.Shell be89fa7 [xuan] fix Spark compilation is broken with the latest hadoop-2.4.0 release (cherry picked from commit 725925cf2120e998651f7d1406fdb34fc2405b9f) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d8fc4a4b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d8fc4a4b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d8fc4a4b Branch: refs/heads/branch-1.0 Commit: d8fc4a4b6c3871efb2cc024eeb01f5a988f6c91b Parents: b75301f Author: xuan Authored: Wed Apr 16 14:41:22 2014 -0500 Committer: Thomas Graves Committed: Wed Apr 16 14:43:07 2014 -0500 -- .../apache/spark/deploy/yarn/ClientBase.scala | 33 +- .../deploy/yarn/ExecutorRunnableUtil.scala | 4 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 63 3 files changed, 85 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d8fc4a4b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index eb42922..628dd98 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -17,6 +17,7 @@ package org.apache.spark.deploy.yarn +import java.io.File import java.net.{InetAddress, UnknownHostException, URI} import java.nio.ByteBuffer @@ -280,7 +281,8 @@ trait ClientBase extends Logging { distCacheMgr.setDistArchivesEnv(env) // Allow users to specify some environment variables. -Apps.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV")) +YarnSparkHadoopUtil.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV"), + File.pathSeparator) // Add each SPARK_* key to the environment. System.getenv().filterKeys(_.startsWith("SPARK")).foreach { case (k,v) => env(k) = v } @@ -382,7 +384,8 @@ object ClientBase { YarnConfiguration.YARN_APPLICATION_CLASSPATH)).getOrElse( getDefaultYarnApplicationClasspath()) for (c <- classpathEntries) { - Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim) + YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, c.trim, +File.pathSeparator) } val mrClasspathEntries = Option(conf.getStrings( @@ -390,7 +393,8 @@ object ClientBase { getDefaultMRApplicationClasspath()) if (mrClasspathEntries != null) { for (c <- mrClasspathEntries) { -Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim) +YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, c.trim, + File.pathSeparator) } } } @@ -425,28 +429,29 @@ object ClientBase { } def populateClasspath(conf: Configuration, sparkConf: SparkConf, addLog4j: Boolean, env: HashMap[String, String]) { -Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$()) +YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$(), + File.pathSeparator) // If log4j present, ensure ours overrides all others if (addLog4j) { - Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + -Path.SEPARATOR + LOG4J_PROP) + YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + +Path.SEPARATOR + LOG4J_PROP, File.pathSeparator) } // Normally the users app.jar is last in case conflicts with spark jars val userClasspathFirst = sparkConf.get("spark.yarn.user.classpath.first", "false") .toBoolean if (userClasspathFirst) { - Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + -Path.SEPARATOR + APP_JAR) + YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$() + +Path.SEPARATOR + APP_JAR, File.pathSeparator) } -Apps.addToEnvironmen
git commit: [SPARK-1395] Allow "local:" URIs to work on Yarn.
Repository: spark Updated Branches: refs/heads/master bb76eae1b -> 69047506b [SPARK-1395] Allow "local:" URIs to work on Yarn. This only works for the three paths defined in the environment (SPARK_JAR, SPARK_YARN_APP_JAR and SPARK_LOG4J_CONF). Tested by running SparkPi with local: and file: URIs against Yarn cluster (no "upload" shows up in logs in the local case). Author: Marcelo Vanzin Closes #303 from vanzin/yarn-local and squashes the following commits: 82219c1 [Marcelo Vanzin] [SPARK-1395] Allow "local:" URIs to work on Yarn. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/69047506 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/69047506 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/69047506 Branch: refs/heads/master Commit: 69047506bf97e6e37e4079c87cb0327d3760ac41 Parents: bb76eae Author: Marcelo Vanzin Authored: Thu Apr 17 10:29:38 2014 -0500 Committer: Thomas Graves Committed: Thu Apr 17 10:29:38 2014 -0500 -- .../org/apache/spark/deploy/SparkSubmit.scala | 4 +- .../spark/deploy/yarn/ExecutorRunnable.scala| 2 +- .../apache/spark/deploy/yarn/ClientBase.scala | 190 +-- .../deploy/yarn/ExecutorRunnableUtil.scala | 17 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 6 +- .../spark/deploy/yarn/ExecutorRunnable.scala| 2 +- 6 files changed, 142 insertions(+), 79 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/69047506/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index e05fbfe..e5d593c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy import java.io.{PrintStream, File} -import java.net.URL +import java.net.{URI, URL} import org.apache.spark.executor.ExecutorURLClassLoader @@ -216,7 +216,7 @@ object SparkSubmit { } private def addJarToClasspath(localJar: String, loader: ExecutorURLClassLoader) { -val localJarFile = new File(localJar) +val localJarFile = new File(new URI(localJar).getPath()) if (!localJarFile.exists()) { printWarning(s"Jar $localJar does not exist, skipping.") } http://git-wip-us.apache.org/repos/asf/spark/blob/69047506/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 3469b7d..7dae248 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -82,7 +82,7 @@ class ExecutorRunnable( ctx.setContainerTokens(ByteBuffer.wrap(dob.getData())) val commands = prepareCommand(masterAddress, slaveId, hostname, executorMemory, executorCores, - localResources.contains(ClientBase.LOG4J_PROP)) + localResources) logInfo("Setting up executor with commands: " + commands) ctx.setCommands(commands) http://git-wip-us.apache.org/repos/asf/spark/blob/69047506/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 628dd98..566de71 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.yarn import java.io.File -import java.net.{InetAddress, UnknownHostException, URI} +import java.net.{InetAddress, UnknownHostException, URI, URISyntaxException} import java.nio.ByteBuffer import scala.collection.JavaConversions._ @@ -209,53 +209,35 @@ trait ClientBase extends Logging { Map( ClientBase.SPARK_JAR -> System.getenv("SPARK_JAR"), ClientBase.APP_JAR -> args.userJar, - ClientBase.LOG4J_PROP -> System.getenv("SPARK_LOG4J_CONF") + ClientBase.LOG4J_PROP -> System.getenv(ClientBase.LOG4J_CONF_ENV_KEY) ).foreach { case(destName, _localPath) => val localPath: String = if (_localPath != null) _localPath.trim() else "" if (! localPath.isEmpty()) { val localURI = new URI(localPath) -val setPermissions = if (destNam
git commit: [SPARK-1395] Allow "local:" URIs to work on Yarn.
Repository: spark Updated Branches: refs/heads/branch-1.0 b3ad707c4 -> a83a794f1 [SPARK-1395] Allow "local:" URIs to work on Yarn. This only works for the three paths defined in the environment (SPARK_JAR, SPARK_YARN_APP_JAR and SPARK_LOG4J_CONF). Tested by running SparkPi with local: and file: URIs against Yarn cluster (no "upload" shows up in logs in the local case). Author: Marcelo Vanzin Closes #303 from vanzin/yarn-local and squashes the following commits: 82219c1 [Marcelo Vanzin] [SPARK-1395] Allow "local:" URIs to work on Yarn. (cherry picked from commit 69047506bf97e6e37e4079c87cb0327d3760ac41) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a83a794f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a83a794f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a83a794f Branch: refs/heads/branch-1.0 Commit: a83a794f1accc616cfccde78af44c5cbf066c647 Parents: b3ad707 Author: Marcelo Vanzin Authored: Thu Apr 17 10:29:38 2014 -0500 Committer: Thomas Graves Committed: Thu Apr 17 10:33:15 2014 -0500 -- .../org/apache/spark/deploy/SparkSubmit.scala | 4 +- .../spark/deploy/yarn/ExecutorRunnable.scala| 2 +- .../apache/spark/deploy/yarn/ClientBase.scala | 190 +-- .../deploy/yarn/ExecutorRunnableUtil.scala | 17 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 6 +- .../spark/deploy/yarn/ExecutorRunnable.scala| 2 +- 6 files changed, 142 insertions(+), 79 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/a83a794f/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala -- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index e05fbfe..e5d593c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy import java.io.{PrintStream, File} -import java.net.URL +import java.net.{URI, URL} import org.apache.spark.executor.ExecutorURLClassLoader @@ -216,7 +216,7 @@ object SparkSubmit { } private def addJarToClasspath(localJar: String, loader: ExecutorURLClassLoader) { -val localJarFile = new File(localJar) +val localJarFile = new File(new URI(localJar).getPath()) if (!localJarFile.exists()) { printWarning(s"Jar $localJar does not exist, skipping.") } http://git-wip-us.apache.org/repos/asf/spark/blob/a83a794f/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 3469b7d..7dae248 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -82,7 +82,7 @@ class ExecutorRunnable( ctx.setContainerTokens(ByteBuffer.wrap(dob.getData())) val commands = prepareCommand(masterAddress, slaveId, hostname, executorMemory, executorCores, - localResources.contains(ClientBase.LOG4J_PROP)) + localResources) logInfo("Setting up executor with commands: " + commands) ctx.setCommands(commands) http://git-wip-us.apache.org/repos/asf/spark/blob/a83a794f/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index 628dd98..566de71 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.yarn import java.io.File -import java.net.{InetAddress, UnknownHostException, URI} +import java.net.{InetAddress, UnknownHostException, URI, URISyntaxException} import java.nio.ByteBuffer import scala.collection.JavaConversions._ @@ -209,53 +209,35 @@ trait ClientBase extends Logging { Map( ClientBase.SPARK_JAR -> System.getenv("SPARK_JAR"), ClientBase.APP_JAR -> args.userJar, - ClientBase.LOG4J_PROP -> System.getenv("SPARK_LOG4J_CONF") + ClientBase.LOG4J_PROP -> System.getenv(ClientBase.LOG4J_CONF_ENV_KEY) ).foreach { case(destName, _localPath) => val localPath: String = if (_localPath != null) _localPath.trim() else "" if (!
git commit: SPARK-1408 Modify Spark on Yarn to point to the history server when app ...
Repository: spark Updated Branches: refs/heads/master 69047506b -> 0058b5d2c SPARK-1408 Modify Spark on Yarn to point to the history server when app ... ...finishes Note this is dependent on https://github.com/apache/spark/pull/204 to have a working history server, but there are no code dependencies. This also fixes SPARK-1288 yarn stable finishApplicationMaster incomplete. Since I was in there I made the diagnostic message be passed properly. Author: Thomas Graves Closes #362 from tgravescs/SPARK-1408 and squashes the following commits: ec89705 [Thomas Graves] Fix typo. 446122d [Thomas Graves] Make config yarn specific f5d5373 [Thomas Graves] SPARK-1408 Modify Spark on Yarn to point to the history server when app finishes Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0058b5d2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0058b5d2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0058b5d2 Branch: refs/heads/master Commit: 0058b5d2c74147d24b127a5432f89ebc7050dc18 Parents: 6904750 Author: Thomas Graves Authored: Thu Apr 17 16:36:37 2014 -0500 Committer: Thomas Graves Committed: Thu Apr 17 16:36:37 2014 -0500 -- docs/running-on-yarn.md | 1 + .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 3 +-- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/0058b5d2/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 9825143..9765062 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -42,6 +42,7 @@ System Properties: * `spark.yarn.preserve.staging.files`, set to true to preserve the staged files(spark jar, app jar, distributed cache files) at the end of the job rather then delete them. * `spark.yarn.scheduler.heartbeat.interval-ms`, the interval in ms in which the Spark application master heartbeats into the YARN ResourceManager. Default is 5 seconds. * `spark.yarn.max.executor.failures`, the maximum number of executor failures before failing the application. Default is the number of executors requested times 2 with minimum of 3. +* `spark.yarn.historyServer.address`, the address of the Spark history server (i.e. host.com:18080). The address should not contain a scheme (http://). Defaults to not being set since the history server is an optional service. This address is given to the Yarn ResourceManager when the Spark application finishes to link the application from the ResourceManager UI to the Spark history server UI. # Launching Spark on YARN http://git-wip-us.apache.org/repos/asf/spark/blob/0058b5d2/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 67ec95c..f078d06 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -366,8 +366,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, finishReq.setAppAttemptId(appAttemptId) finishReq.setFinishApplicationStatus(status) finishReq.setDiagnostics(diagnostics) -// Set tracking url to empty since we don't have a history server. -finishReq.setTrackingUrl("") + finishReq.setTrackingUrl(sparkConf.get("spark.yarn.historyServer.address", "")) resourceManager.finishApplicationMaster(finishReq) } } http://git-wip-us.apache.org/repos/asf/spark/blob/0058b5d2/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 581cfe4..b225be6 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -347,8 +347,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, logInfo("finishApplicationMaster with " + status) if (registered) { -// Set tracking URL to empty since we don't have a history server. -amClient.unregisterApplicationMaster(status, "" /* appMessa
git commit: SPARK-1408 Modify Spark on Yarn to point to the history server when app ...
Repository: spark Updated Branches: refs/heads/branch-1.0 a83a794f1 -> 6195fb8db SPARK-1408 Modify Spark on Yarn to point to the history server when app ... ...finishes Note this is dependent on https://github.com/apache/spark/pull/204 to have a working history server, but there are no code dependencies. This also fixes SPARK-1288 yarn stable finishApplicationMaster incomplete. Since I was in there I made the diagnostic message be passed properly. Author: Thomas Graves Closes #362 from tgravescs/SPARK-1408 and squashes the following commits: ec89705 [Thomas Graves] Fix typo. 446122d [Thomas Graves] Make config yarn specific f5d5373 [Thomas Graves] SPARK-1408 Modify Spark on Yarn to point to the history server when app finishes (cherry picked from commit 0058b5d2c74147d24b127a5432f89ebc7050dc18) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6195fb8d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6195fb8d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6195fb8d Branch: refs/heads/branch-1.0 Commit: 6195fb8db9f194856df7f5c02f5023596d0c3253 Parents: a83a794 Author: Thomas Graves Authored: Thu Apr 17 16:36:37 2014 -0500 Committer: Thomas Graves Committed: Thu Apr 17 16:36:50 2014 -0500 -- docs/running-on-yarn.md | 1 + .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 3 +-- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6195fb8d/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 9825143..9765062 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -42,6 +42,7 @@ System Properties: * `spark.yarn.preserve.staging.files`, set to true to preserve the staged files(spark jar, app jar, distributed cache files) at the end of the job rather then delete them. * `spark.yarn.scheduler.heartbeat.interval-ms`, the interval in ms in which the Spark application master heartbeats into the YARN ResourceManager. Default is 5 seconds. * `spark.yarn.max.executor.failures`, the maximum number of executor failures before failing the application. Default is the number of executors requested times 2 with minimum of 3. +* `spark.yarn.historyServer.address`, the address of the Spark history server (i.e. host.com:18080). The address should not contain a scheme (http://). Defaults to not being set since the history server is an optional service. This address is given to the Yarn ResourceManager when the Spark application finishes to link the application from the ResourceManager UI to the Spark history server UI. # Launching Spark on YARN http://git-wip-us.apache.org/repos/asf/spark/blob/6195fb8d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 67ec95c..f078d06 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -366,8 +366,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, finishReq.setAppAttemptId(appAttemptId) finishReq.setFinishApplicationStatus(status) finishReq.setDiagnostics(diagnostics) -// Set tracking url to empty since we don't have a history server. -finishReq.setTrackingUrl("") + finishReq.setTrackingUrl(sparkConf.get("spark.yarn.historyServer.address", "")) resourceManager.finishApplicationMaster(finishReq) } } http://git-wip-us.apache.org/repos/asf/spark/blob/6195fb8d/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 581cfe4..b225be6 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -347,8 +347,8 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration, logInfo("finishApplicationMaster with " + status) if (registered) { -// Set tracking URL to empt
git commit: SPARK-1557 Set permissions on event log files/directories
Repository: spark Updated Branches: refs/heads/master 9a1184a8a -> 8db0f7e28 SPARK-1557 Set permissions on event log files/directories This adds minimal setting of event log directory/files permissions. To have a secure environment the user must manually create the top level event log directory and set permissions up. We can add logic to do that automatically later if we want. Author: Thomas Graves Closes #538 from tgravescs/SPARK-1557 and squashes the following commits: e471d8e [Thomas Graves] rework d8b6620 [Thomas Graves] update use of octal 3ca9b79 [Thomas Graves] Updated based on comments 5a09709 [Thomas Graves] add in missing import 3150ed6 [Thomas Graves] SPARK-1557 Set permissions on event log files/directories Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8db0f7e2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8db0f7e2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8db0f7e2 Branch: refs/heads/master Commit: 8db0f7e28f5f0330a3344705ff48d8e7b97c383f Parents: 9a1184a Author: Thomas Graves Authored: Tue Apr 29 09:19:48 2014 -0500 Committer: Thomas Graves Committed: Tue Apr 29 09:19:48 2014 -0500 -- .../spark/scheduler/EventLoggingListener.scala | 6 +- .../org/apache/spark/util/FileLogger.scala | 22 +++- docs/security.md| 2 ++ 3 files changed, 24 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/8db0f7e2/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala index 2fe65cd..d822a8e 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala @@ -21,6 +21,7 @@ import scala.collection.mutable import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.hadoop.fs.permission.FsPermission import org.json4s.jackson.JsonMethods._ import org.apache.spark.{Logging, SparkConf, SparkContext} @@ -54,7 +55,7 @@ private[spark] class EventLoggingListener( private val logger = new FileLogger(logDir, conf, hadoopConfiguration, outputBufferSize, shouldCompress, - shouldOverwrite) + shouldOverwrite, Some(LOG_FILE_PERMISSIONS)) /** * Begin logging events. @@ -124,6 +125,9 @@ private[spark] object EventLoggingListener extends Logging { val SPARK_VERSION_PREFIX = "SPARK_VERSION_" val COMPRESSION_CODEC_PREFIX = "COMPRESSION_CODEC_" val APPLICATION_COMPLETE = "APPLICATION_COMPLETE" + val LOG_FILE_PERMISSIONS: FsPermission = +FsPermission.createImmutable(Integer.parseInt("770", 8).toShort) + // A cache for compression codecs to avoid creating the same codec many times private val codecMap = new mutable.HashMap[String, CompressionCodec] http://git-wip-us.apache.org/repos/asf/spark/blob/8db0f7e2/core/src/main/scala/org/apache/spark/util/FileLogger.scala -- diff --git a/core/src/main/scala/org/apache/spark/util/FileLogger.scala b/core/src/main/scala/org/apache/spark/util/FileLogger.scala index 1ed3b70..0965e0f 100644 --- a/core/src/main/scala/org/apache/spark/util/FileLogger.scala +++ b/core/src/main/scala/org/apache/spark/util/FileLogger.scala @@ -24,6 +24,7 @@ import java.util.Date import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path} +import org.apache.hadoop.fs.permission.FsPermission import org.apache.spark.{Logging, SparkConf} import org.apache.spark.io.CompressionCodec @@ -42,7 +43,8 @@ private[spark] class FileLogger( hadoopConfiguration: Configuration, outputBufferSize: Int = 8 * 1024, // 8 KB compress: Boolean = false, -overwrite: Boolean = true) +overwrite: Boolean = true, +dirPermissions: Option[FsPermission] = None) extends Logging { private val dateFormat = new ThreadLocal[SimpleDateFormat]() { @@ -79,16 +81,25 @@ private[spark] class FileLogger( if (!fileSystem.mkdirs(path)) { throw new IOException("Error in creating log directory: %s".format(logDir)) } +if (dirPermissions.isDefined) { + val fsStatus = fileSystem.getFileStatus(path) + if (fsStatus.getPermission().toShort() != dirPermissions.get.toShort) { +fileSystem.setPermission(path, dirPermissions.get) + } +} } /** * Create a new writer for the file identified by the given path. + * If the permissions are not passed in, it will def
git commit: SPARK-1557 Set permissions on event log files/directories
Repository: spark Updated Branches: refs/heads/branch-1.0 c27ce2b4f -> bccd13ec1 SPARK-1557 Set permissions on event log files/directories This adds minimal setting of event log directory/files permissions. To have a secure environment the user must manually create the top level event log directory and set permissions up. We can add logic to do that automatically later if we want. Author: Thomas Graves Closes #538 from tgravescs/SPARK-1557 and squashes the following commits: e471d8e [Thomas Graves] rework d8b6620 [Thomas Graves] update use of octal 3ca9b79 [Thomas Graves] Updated based on comments 5a09709 [Thomas Graves] add in missing import 3150ed6 [Thomas Graves] SPARK-1557 Set permissions on event log files/directories (cherry picked from commit 8db0f7e28f5f0330a3344705ff48d8e7b97c383f) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bccd13ec Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bccd13ec Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bccd13ec Branch: refs/heads/branch-1.0 Commit: bccd13ec1825016b7f072370344a4c98ff078dea Parents: c27ce2b Author: Thomas Graves Authored: Tue Apr 29 09:19:48 2014 -0500 Committer: Thomas Graves Committed: Tue Apr 29 09:20:31 2014 -0500 -- .../spark/scheduler/EventLoggingListener.scala | 6 +- .../org/apache/spark/util/FileLogger.scala | 22 +++- docs/security.md| 2 ++ 3 files changed, 24 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/bccd13ec/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala -- diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala index 2fe65cd..d822a8e 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala @@ -21,6 +21,7 @@ import scala.collection.mutable import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.hadoop.fs.permission.FsPermission import org.json4s.jackson.JsonMethods._ import org.apache.spark.{Logging, SparkConf, SparkContext} @@ -54,7 +55,7 @@ private[spark] class EventLoggingListener( private val logger = new FileLogger(logDir, conf, hadoopConfiguration, outputBufferSize, shouldCompress, - shouldOverwrite) + shouldOverwrite, Some(LOG_FILE_PERMISSIONS)) /** * Begin logging events. @@ -124,6 +125,9 @@ private[spark] object EventLoggingListener extends Logging { val SPARK_VERSION_PREFIX = "SPARK_VERSION_" val COMPRESSION_CODEC_PREFIX = "COMPRESSION_CODEC_" val APPLICATION_COMPLETE = "APPLICATION_COMPLETE" + val LOG_FILE_PERMISSIONS: FsPermission = +FsPermission.createImmutable(Integer.parseInt("770", 8).toShort) + // A cache for compression codecs to avoid creating the same codec many times private val codecMap = new mutable.HashMap[String, CompressionCodec] http://git-wip-us.apache.org/repos/asf/spark/blob/bccd13ec/core/src/main/scala/org/apache/spark/util/FileLogger.scala -- diff --git a/core/src/main/scala/org/apache/spark/util/FileLogger.scala b/core/src/main/scala/org/apache/spark/util/FileLogger.scala index 1ed3b70..0965e0f 100644 --- a/core/src/main/scala/org/apache/spark/util/FileLogger.scala +++ b/core/src/main/scala/org/apache/spark/util/FileLogger.scala @@ -24,6 +24,7 @@ import java.util.Date import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path} +import org.apache.hadoop.fs.permission.FsPermission import org.apache.spark.{Logging, SparkConf} import org.apache.spark.io.CompressionCodec @@ -42,7 +43,8 @@ private[spark] class FileLogger( hadoopConfiguration: Configuration, outputBufferSize: Int = 8 * 1024, // 8 KB compress: Boolean = false, -overwrite: Boolean = true) +overwrite: Boolean = true, +dirPermissions: Option[FsPermission] = None) extends Logging { private val dateFormat = new ThreadLocal[SimpleDateFormat]() { @@ -79,16 +81,25 @@ private[spark] class FileLogger( if (!fileSystem.mkdirs(path)) { throw new IOException("Error in creating log directory: %s".format(logDir)) } +if (dirPermissions.isDefined) { + val fsStatus = fileSystem.getFileStatus(path) + if (fsStatus.getPermission().toShort() != dirPermissions.get.toShort) { +fileSystem.setPermission(path, dirPermissions.get) + } +} } /** * Create a n
git commit: [SPARK-1522] : YARN ClientBase throws a NPE if there is no YARN Application CP
Repository: spark Updated Branches: refs/heads/master 6cf335d79 -> e27344768 [SPARK-1522] : YARN ClientBase throws a NPE if there is no YARN Application CP The current implementation of ClientBase.getDefaultYarnApplicationClasspath inspects the MRJobConfig class for the field DEFAULT_YARN_APPLICATION_CLASSPATH when it should be really looking into YarnConfiguration. If the Application Configuration has no yarn.application.classpath defined a NPE exception will be thrown. Additional Changes include: * Test Suite for ClientBase added [ticket: SPARK-1522] : https://issues.apache.org/jira/browse/SPARK-1522 Author : bernardo.gomezpala...@gmail.com Testing : SPARK_HADOOP_VERSION=2.3.0 SPARK_YARN=true ./sbt/sbt test Author: Bernardo Gomez Palacio Closes #433 from berngp/feature/SPARK-1522 and squashes the following commits: 2c2e118 [Bernardo Gomez Palacio] [SPARK-1522]: YARN ClientBase throws a NPE if there is no YARN Application specific CP Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e2734476 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e2734476 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e2734476 Branch: refs/heads/master Commit: e273447684779a18bd61d733bfe7958b78657ffd Parents: 6cf335d Author: Bernardo Gomez Palacio Authored: Mon Jun 9 16:14:54 2014 -0500 Committer: Thomas Graves Committed: Mon Jun 9 16:14:54 2014 -0500 -- .../apache/spark/deploy/yarn/ClientBase.scala | 89 +-- .../spark/deploy/yarn/ClientBaseSuite.scala | 112 +++ 2 files changed, 167 insertions(+), 34 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e2734476/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala -- diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index aeb3f00..4b5e0ef 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -23,6 +23,7 @@ import java.nio.ByteBuffer import scala.collection.JavaConversions._ import scala.collection.mutable.{HashMap, ListBuffer, Map} +import scala.util.{Try, Success, Failure} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs._ @@ -378,7 +379,7 @@ trait ClientBase extends Logging { } } -object ClientBase { +object ClientBase extends Logging { val SPARK_JAR: String = "__spark__.jar" val APP_JAR: String = "__app__.jar" val LOG4J_PROP: String = "log4j.properties" @@ -388,37 +389,47 @@ object ClientBase { def getSparkJar = sys.env.get("SPARK_JAR").getOrElse(SparkContext.jarOfClass(this.getClass).head) - // Based on code from org.apache.hadoop.mapreduce.v2.util.MRApps - def populateHadoopClasspath(conf: Configuration, env: HashMap[String, String]) { -val classpathEntries = Option(conf.getStrings( - YarnConfiguration.YARN_APPLICATION_CLASSPATH)).getOrElse( -getDefaultYarnApplicationClasspath()) -if (classpathEntries != null) { - for (c <- classpathEntries) { -YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, c.trim, - File.pathSeparator) - } + def populateHadoopClasspath(conf: Configuration, env: HashMap[String, String]) = { +val classPathElementsToAdd = getYarnAppClasspath(conf) ++ getMRAppClasspath(conf) +for (c <- classPathElementsToAdd.flatten) { + YarnSparkHadoopUtil.addToEnvironment( +env, +Environment.CLASSPATH.name, +c.trim, +File.pathSeparator) } +classPathElementsToAdd + } -val mrClasspathEntries = Option(conf.getStrings( - "mapreduce.application.classpath")).getOrElse( -getDefaultMRApplicationClasspath()) -if (mrClasspathEntries != null) { - for (c <- mrClasspathEntries) { -YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, c.trim, - File.pathSeparator) - } -} + private def getYarnAppClasspath(conf: Configuration): Option[Seq[String]] = +Option(conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) match { + case Some(s) => Some(s.toSeq) + case None => getDefaultYarnApplicationClasspath } - def getDefaultYarnApplicationClasspath(): Array[String] = { -try { - val field = classOf[MRJobConfig].getField("DEFAULT_YARN_APPLICATION_CLASSPATH") - field.get(null).asInstanceOf[Array[String]] -} catch { - case err: NoSuchFieldError => null - case err: NoSuchFieldException => null + private def getMRAppClasspath(conf: Configuration): Option[Seq[String]] = +Option(conf