[1/3] spark-website git commit: Spark 2.2.2 release announcements
Repository: spark-website Updated Branches: refs/heads/asf-site 390743e8a -> 179a5897a http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-1-6-2.html -- diff --git a/site/releases/spark-release-1-6-2.html b/site/releases/spark-release-1-6-2.html index 5f459ce..d23f38d 100644 --- a/site/releases/spark-release-1-6-2.html +++ b/site/releases/spark-release-1-6-2.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-1-6-3.html -- diff --git a/site/releases/spark-release-1-6-3.html b/site/releases/spark-release-1-6-3.html index 6d9efd9..fb678c2 100644 --- a/site/releases/spark-release-1-6-3.html +++ b/site/releases/spark-release-1-6-3.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-0-0.html -- diff --git a/site/releases/spark-release-2-0-0.html b/site/releases/spark-release-2-0-0.html index 8d1ef52..40bd9f3 100644 --- a/site/releases/spark-release-2-0-0.html +++ b/site/releases/spark-release-2-0-0.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-0-1.html -- diff --git a/site/releases/spark-release-2-0-1.html b/site/releases/spark-release-2-0-1.html index 6adef9a..1860b0c 100644 --- a/site/releases/spark-release-2-0-1.html +++ b/site/releases/spark-release-2-0-1.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-0-2.html -- diff --git a/site/releases/spark-release-2-0-2.html b/site/releases/spark-release-2-0-2.html index 275bed6..9d903dd 100644 --- a/site/releases/spark-release-2-0-2.html +++ b/site/releases/spark-release-2-0-2.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/releases/spark-release-2-1-0.html -- diff --git a/site/releases/spark-release-2-1-0.html b/site/releases/spark-release-2-1-0.html index ddaff2f..bf6980a 100644 --- a/site/releases/spark-release-2-1-0.html +++ b/site/releases/spark-release-2-1-0.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http:/
[3/3] spark-website git commit: Spark 2.2.2 release announcements
Spark 2.2.2 release announcements 2.2.2 docs look fine, created the announcement and regenerated the docs. Author: Thomas Graves Closes #129 from tgravescs/spark222-announce. Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/179a5897 Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/179a5897 Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/179a5897 Branch: refs/heads/asf-site Commit: 179a5897a6b056ba84b3ef6fefac5cee020214ac Parents: 390743e Author: Thomas Graves Authored: Mon Jul 9 10:29:50 2018 -0500 Committer: Thomas Graves Committed: Mon Jul 9 10:29:50 2018 -0500 -- news/_posts/2018-07-02-spark-2-2-2-released.md | 14 ++ site/committers.html | 6 +++--- site/community.html | 6 +++--- site/contributing.html | 6 +++--- site/developer-tools.html| 6 +++--- site/documentation.html | 6 +++--- site/downloads.html | 6 +++--- site/examples.html | 6 +++--- site/faq.html| 6 +++--- site/graphx/index.html | 6 +++--- site/history.html| 6 +++--- site/improvement-proposals.html | 6 +++--- site/index.html | 6 +++--- site/mailing-lists.html | 6 +++--- site/mllib/index.html| 6 +++--- site/news/amp-camp-2013-registration-ope.html| 6 +++--- site/news/announcing-the-first-spark-summit.html | 6 +++--- site/news/fourth-spark-screencast-published.html | 6 +++--- site/news/index.html | 15 --- site/news/nsdi-paper.html| 6 +++--- site/news/one-month-to-spark-summit-2015.html| 6 +++--- site/news/proposals-open-for-spark-summit-east.html | 6 +++--- .../registration-open-for-spark-summit-east.html | 6 +++--- site/news/run-spark-and-shark-on-amazon-emr.html | 6 +++--- site/news/spark-0-6-1-and-0-5-2-released.html| 6 +++--- site/news/spark-0-6-2-released.html | 6 +++--- site/news/spark-0-7-0-released.html | 6 +++--- site/news/spark-0-7-2-released.html | 6 +++--- site/news/spark-0-7-3-released.html | 6 +++--- site/news/spark-0-8-0-released.html | 6 +++--- site/news/spark-0-8-1-released.html | 6 +++--- site/news/spark-0-9-0-released.html | 6 +++--- site/news/spark-0-9-1-released.html | 6 +++--- site/news/spark-0-9-2-released.html | 6 +++--- site/news/spark-1-0-0-released.html | 6 +++--- site/news/spark-1-0-1-released.html | 6 +++--- site/news/spark-1-0-2-released.html | 6 +++--- site/news/spark-1-1-0-released.html | 6 +++--- site/news/spark-1-1-1-released.html | 6 +++--- site/news/spark-1-2-0-released.html | 6 +++--- site/news/spark-1-2-1-released.html | 6 +++--- site/news/spark-1-2-2-released.html | 6 +++--- site/news/spark-1-3-0-released.html | 6 +++--- site/news/spark-1-4-0-released.html | 6 +++--- site/news/spark-1-4-1-released.html | 6 +++--- site/news/spark-1-5-0-released.html | 6 +++--- site/news/spark-1-5-1-released.html | 6 +++--- site/news/spark-1-5-2-released.html | 6 +++--- site/news/spark-1-6-0-released.html | 6 +++--- site/news/spark-1-6-1-released.html | 6 +++--- site/news/spark-1-6-2-released.html | 6 +++--- site/news/spark-1-6-3-released.html | 6 +++--- site/news/spark-2-0-0-released.html | 6 +++--- site/news/spark-2-0-1-released.html | 6 +++--- site/news/spark-2-0-2-released.html | 6 +++--- site/news/spark-2-1-0-released.html | 6 +++--- site/news/spark-2-1-1-released.html | 6 +++--- site/news/spark-2-1-2-released.html | 6 +++--- site/news/spark-2-1-3-released.html | 6 +++--- site/news/spark-2-2-0-released.html | 6 +++--- site/news/spark-2-2-1-released.html | 6 +++--- site/news/spark-2-3-0-released.html | 6 +++--- site/news/spark-2-3-1-released.html | 6 +++--- site/news/spark-2.0.0-preview.html |
[2/3] spark-website git commit: Spark 2.2.2 release announcements
http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-2-3-1-released.html -- diff --git a/site/news/spark-2-3-1-released.html b/site/news/spark-2-3-1-released.html index a6ab537..06b718a 100644 --- a/site/news/spark-2-3-1-released.html +++ b/site/news/spark-2-3-1-released.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-2.0.0-preview.html -- diff --git a/site/news/spark-2.0.0-preview.html b/site/news/spark-2.0.0-preview.html index e704bab..da54cef 100644 --- a/site/news/spark-2.0.0-preview.html +++ b/site/news/spark-2.0.0-preview.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-accepted-into-apache-incubator.html -- diff --git a/site/news/spark-accepted-into-apache-incubator.html b/site/news/spark-accepted-into-apache-incubator.html index 06ebb8d..696c6d0 100644 --- a/site/news/spark-accepted-into-apache-incubator.html +++ b/site/news/spark-accepted-into-apache-incubator.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-and-shark-in-the-news.html -- diff --git a/site/news/spark-and-shark-in-the-news.html b/site/news/spark-and-shark-in-the-news.html index f3f5f41..328d3c4 100644 --- a/site/news/spark-and-shark-in-the-news.html +++ b/site/news/spark-and-shark-in-the-news.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-becomes-tlp.html -- diff --git a/site/news/spark-becomes-tlp.html b/site/news/spark-becomes-tlp.html index 24051c2..981aaa7 100644 --- a/site/news/spark-becomes-tlp.html +++ b/site/news/spark-becomes-tlp.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/spark-featured-in-wired.html -- diff --git a/site/news/spark-featured-in-wired.html b/site/news/spark-featured-in-wired.html index 3bd546f..709c5dc 100644 --- a/site/news/spark-featured-in-wired.html +++ b/site/news/spark-featured-in-wired.html @@ -162,6 +162,9 @@ Latest News + Spark 2.2.2 released + (Jul 02, 2018) + Spark 2.1.3 released (Jun 29, 2018) @@ -171,9 +174,6 @@ Spark+AI Summit (June 4-6th, 2018, San Francisco) agenda posted (Mar 01, 2018) - Spark 2.3.0 released - (Feb 28, 2018) - Archive http://git-wip-us.apache.org/repos/asf/spark-website/blob/179a5897/site/news/s
spark-website git commit: Adding missing spark 2.2.2 announcement html files
Repository: spark-website
Updated Branches:
refs/heads/asf-site 179a5897a -> 2b5ba2f62
Adding missing spark 2.2.2 announcement html files
Project: http://git-wip-us.apache.org/repos/asf/spark-website/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark-website/commit/2b5ba2f6
Tree: http://git-wip-us.apache.org/repos/asf/spark-website/tree/2b5ba2f6
Diff: http://git-wip-us.apache.org/repos/asf/spark-website/diff/2b5ba2f6
Branch: refs/heads/asf-site
Commit: 2b5ba2f6247a97374acc3c800d4a4179a0c0a334
Parents: 179a589
Author: Thomas Graves
Authored: Mon Jul 9 11:04:33 2018 -0500
Committer: Thomas Graves
Committed: Mon Jul 9 11:04:33 2018 -0500
--
site/news/spark-2-2-2-released.html| 232 +++
site/releases/spark-release-2-2-2.html | 238
site/sitemap.xml | 8 +-
3 files changed, 474 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark-website/blob/2b5ba2f6/site/news/spark-2-2-2-released.html
--
diff --git a/site/news/spark-2-2-2-released.html
b/site/news/spark-2-2-2-released.html
new file mode 100644
index 000..1c46315
--- /dev/null
+++ b/site/news/spark-2-2-2-released.html
@@ -0,0 +1,232 @@
+
+
+
+
+
+
+
+
+ Spark 2.2.2 released | Apache Spark
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-32518208-2']);
+ _gaq.push(['_trackPageview']);
+ (function() {
+var ga = document.createElement('script'); ga.type = 'text/javascript';
ga.async = true;
+ga.src = ('https:' == document.location.protocol ? 'https://ssl' :
'http://www') + '.google-analytics.com/ga.js';
+var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ga, s);
+ })();
+
+
+ function trackOutboundLink(link, category, action) {
+try {
+ _gaq.push(['_trackEvent', category , action]);
+} catch(err){}
+
+setTimeout(function() {
+ document.location.href = link.href;
+}, 100);
+ }
+
+
+
+
+
+
+
+
+https://code.jquery.com/jquery.js";>
+https://netdna.bootstrapcdn.com/bootstrap/3.0.3/js/bootstrap.min.js";>
+
+
+
+
+
+
+
+
+
+
+ Lightning-fast unified analytics engine
+
+
+
+
+
+
+
+
+
+ Toggle navigation
+
+
+
+
+
+
+
+
+
+ Download
+
+
+ Libraries
+
+
+ SQL and DataFrames
+ Spark Streaming
+ MLlib (machine learning)
+ GraphX (graph)
+
+ Third-Party
Projects
+
+
+
+
+ Documentation
+
+
+ Latest Release (Spark 2.3.1)
+ Older Versions and Other
Resources
+ Frequently Asked Questions
+
+
+ Examples
+
+
+ Community
+
+
+ Mailing Lists & Resources
+ Contributing to Spark
+ Improvement Proposals
(SPIP)
+ https://issues.apache.org/jira/browse/SPARK";>Issue
Tracker
+ Powered By
+ Project Committers
+ Project History
+
+
+
+
+ Developers
+
+
+ Useful Developer Tools
+ Versioning Policy
+ Release Process
+ Security
+
+
+
+
+
+https://www.apache.org/"; class="dropdown-toggle"
data-toggle="dropdown">
+ Apache Software Foundation
+
+ https://www.apache.org/";>Apache Homepage
+ https://www.apache.org/licenses/";>License
+ https://www.apache.org/foundation/sponsorship.html";>Sponsorship
+ https://www.apache.org/foundation/thanks.html";>Thanks
+ https://www.apache.org/security/";>Security
+
+
+
+
+
+
+
+
+
+
+
+ Latest News
+
+
+ Spark 2.2.2
released
+ (Jul 02, 2018)
+
+ Spark 2.1.3
released
+ (Jun 29, 2018)
+
+ Spark 2.3.1
released
+ (Jun 08, 2018)
+
+ Spark+AI Summit (June
4-6th, 2018, San Francisco) agenda posted
+ (Mar 01, 2018)
+
+
+ Archive
+
+
+ https://www.apache.org/events/current-event.html";>
+https://www.apache.org/events/current-event-234x60.png"/>
+
+
+
+
+Download Spark
+
+
+Built-in Libraries:
+
+
+SQL and DataFrames
+Spark Streaming
+MLlib (machine learning)
+
svn commit: r28003 - /dev/spark/v2.2.2-rc2-docs/
Author: tgraves Date: Mon Jul 9 16:12:21 2018 New Revision: 28003 Log: Removing RC artifacts. Removed: dev/spark/v2.2.2-rc2-docs/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24610] fix reading small files via wholeTextFiles
Repository: spark
Updated Branches:
refs/heads/master 9fa4a1ed3 -> 1055c94cd
[SPARK-24610] fix reading small files via wholeTextFiles
## What changes were proposed in this pull request?
The `WholeTextFileInputFormat` determines the `maxSplitSize` for the file/s
being read using the `wholeTextFiles` method. While this works well for large
files, for smaller files where the maxSplitSize is smaller than the defaults
being used with configs like hive-site.xml or explicitly passed in the form of
`mapreduce.input.fileinputformat.split.minsize.per.node` or
`mapreduce.input.fileinputformat.split.minsize.per.rack` , it just throws up an
exception.
```java
java.io.IOException: Minimum split size pernode 123456 cannot be larger than
maximum split size 9962
at
org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.getSplits(CombineFileInputFormat.java:200)
at
org.apache.spark.rdd.WholeTextFileRDD.getPartitions(WholeTextFileRDD.scala:50)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
at
org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2096)
at org.apache.spark.rdd.RDD.count(RDD.scala:1158)
... 48 elided
`
This change checks the maxSplitSize against the minSplitSizePerNode and
minSplitSizePerRack and set them if `maxSplitSize < minSplitSizePerNode/Rack`
## How was this patch tested?
Test manually setting the conf while launching the job and added unit test.
Author: Dhruve Ashar
Closes #21601 from dhruve/bug/SPARK-24610.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1055c94c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1055c94c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1055c94c
Branch: refs/heads/master
Commit: 1055c94cdf072bfce5e36bb6552fe9b148bb9d17
Parents: 9fa4a1e
Author: Dhruve Ashar
Authored: Thu Jul 12 15:36:02 2018 -0500
Committer: Thomas Graves
Committed: Thu Jul 12 15:36:02 2018 -0500
--
.../spark/input/WholeTextFileInputFormat.scala | 13 +++
.../input/WholeTextFileInputFormatSuite.scala | 96
2 files changed, 109 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1055c94c/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
b/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
index f47cd38..04c5c4b 100644
--- a/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
+++ b/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
@@ -53,6 +53,19 @@ private[spark] class WholeTextFileInputFormat
val totalLen = files.map(file => if (file.isDirectory) 0L else
file.getLen).sum
val maxSplitSize = Math.ceil(totalLen * 1.0 /
(if (minPartitions == 0) 1 else minPartitions)).toLong
+
+// For small files we need to ensure the min split size per node & rack <=
maxSplitSize
+val config = context.getConfiguration
+val minSplitSizePerNode =
config.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERNODE, 0L)
+val minSplitSizePerRack =
config.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERRACK, 0L)
+
+if (maxSplitSize < minSplitSizePerNode) {
+ super.setMinSplitSizeNode(maxSplitSize)
+}
+
+if (maxSplitSize < minSplitSizePerRack) {
+ super.setMinSplitSizeRack(maxSplitSize)
+}
super.setMaxSplitSize(maxSplitSize)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1055c94c/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala
b/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala
new file mode 100644
index 000..817dc08
--- /dev/null
+++
b/core/src/test/scala/org/apache/spark/input/WholeTextFileInputFormatSuite.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You unde
spark git commit: Update for spark 2.2.2 release
Repository: spark
Updated Branches:
refs/heads/branch-2.2 4bc4ccd63 -> 17db57213
Update for spark 2.2.2 release
Release process for Spark 2.2.2 say to update the test.
Author: Thomas Graves
Closes #21707 from tgravescs/spark222-release.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/17db5721
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/17db5721
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/17db5721
Branch: refs/heads/branch-2.2
Commit: 17db57213aabc13cb59d13f0f570c7539dae
Parents: 4bc4ccd
Author: Thomas Graves
Authored: Mon Jul 16 09:29:20 2018 -0500
Committer: Thomas Graves
Committed: Mon Jul 16 09:29:20 2018 -0500
--
.../apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/17db5721/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 313059b..e6a6cac 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -170,7 +170,7 @@ class HiveExternalCatalogVersionsSuite extends
SparkSubmitTestUtils {
object PROCESS_TABLES extends QueryTest with SQLTestUtils {
// Tests the latest version of every release line.
- val testingVersions = Seq("2.0.2", "2.1.2", "2.2.1")
+ val testingVersions = Seq("2.0.2", "2.1.3", "2.2.2")
protected var spark: SparkSession = _
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
Repository: spark
Updated Branches:
refs/heads/master fc0c8c971 -> c8bee932c
[SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
## What changes were proposed in this pull request?
When speculation is enabled,
TaskSetManager#markPartitionCompleted should write successful task duration to
MedianHeap,
not just increase tasksSuccessful.
Otherwise when TaskSetManager#checkSpeculatableTasks,tasksSuccessful non-zero,
but MedianHeap is empty.
Then throw an exception successfulTaskDurations.median
java.util.NoSuchElementException: MedianHeap is empty.
Finally led to stopping SparkContext.
## How was this patch tested?
TaskSetManagerSuite.scala
unit test:[SPARK-24677] MedianHeap should not be empty when speculation is
enabled
Author: sychen
Closes #21656 from cxzl25/fix_MedianHeap_empty.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c8bee932
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c8bee932
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c8bee932
Branch: refs/heads/master
Commit: c8bee932cb644627c4049b5a07dd8028968572d9
Parents: fc0c8c9
Author: sychen
Authored: Wed Jul 18 13:24:41 2018 -0500
Committer: Thomas Graves
Committed: Wed Jul 18 13:24:41 2018 -0500
--
.../spark/scheduler/TaskSchedulerImpl.scala | 7 ++-
.../apache/spark/scheduler/TaskSetManager.scala | 7 ++-
.../spark/scheduler/TaskSetManagerSuite.scala | 49
3 files changed, 59 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c8bee932/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 598b62f..56c0bf6 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -697,9 +697,12 @@ private[spark] class TaskSchedulerImpl(
* do not also submit those same tasks. That also means that a task
completion from an earlier
* attempt can lead to the entire stage getting marked as successful.
*/
- private[scheduler] def markPartitionCompletedInAllTaskSets(stageId: Int,
partitionId: Int) = {
+ private[scheduler] def markPartitionCompletedInAllTaskSets(
+ stageId: Int,
+ partitionId: Int,
+ taskInfo: TaskInfo) = {
taskSetsByStageIdAndAttempt.getOrElse(stageId, Map()).values.foreach { tsm
=>
- tsm.markPartitionCompleted(partitionId)
+ tsm.markPartitionCompleted(partitionId, taskInfo)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c8bee932/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index a18c665..6071605 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -758,7 +758,7 @@ private[spark] class TaskSetManager(
}
// There may be multiple tasksets for this stage -- we let all of them
know that the partition
// was completed. This may result in some of the tasksets getting
completed.
-sched.markPartitionCompletedInAllTaskSets(stageId,
tasks(index).partitionId)
+sched.markPartitionCompletedInAllTaskSets(stageId,
tasks(index).partitionId, info)
// This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which
holds the
// "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue,
we should not
// "deserialize" the value when holding a lock to avoid blocking other
threads. So we call
@@ -769,9 +769,12 @@ private[spark] class TaskSetManager(
maybeFinishTaskSet()
}
- private[scheduler] def markPartitionCompleted(partitionId: Int): Unit = {
+ private[scheduler] def markPartitionCompleted(partitionId: Int, taskInfo:
TaskInfo): Unit = {
partitionToIndex.get(partitionId).foreach { index =>
if (!successful(index)) {
+if (speculationEnabled && !isZombie) {
+ successfulTaskDurations.insert(taskInfo.duration)
+}
tasksSuccessful += 1
successful(index) = true
if (tasksSuccessful == numTasks) {
http://git-wip-us.apache.org/repos/asf/spark/blob/c8bee932/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/TaskSetMa
spark git commit: [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
Repository: spark
Updated Branches:
refs/heads/branch-2.3 e31b4766b -> 7be70e29d
[SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
## What changes were proposed in this pull request?
When speculation is enabled,
TaskSetManager#markPartitionCompleted should write successful task duration to
MedianHeap,
not just increase tasksSuccessful.
Otherwise when TaskSetManager#checkSpeculatableTasks,tasksSuccessful non-zero,
but MedianHeap is empty.
Then throw an exception successfulTaskDurations.median
java.util.NoSuchElementException: MedianHeap is empty.
Finally led to stopping SparkContext.
## How was this patch tested?
TaskSetManagerSuite.scala
unit test:[SPARK-24677] MedianHeap should not be empty when speculation is
enabled
Author: sychen
Closes #21656 from cxzl25/fix_MedianHeap_empty.
(cherry picked from commit c8bee932cb644627c4049b5a07dd8028968572d9)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7be70e29
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7be70e29
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7be70e29
Branch: refs/heads/branch-2.3
Commit: 7be70e29dd92de36dbb30ce39623d588f48e4cac
Parents: e31b476
Author: sychen
Authored: Wed Jul 18 13:24:41 2018 -0500
Committer: Thomas Graves
Committed: Wed Jul 18 13:24:54 2018 -0500
--
.../spark/scheduler/TaskSchedulerImpl.scala | 7 ++-
.../apache/spark/scheduler/TaskSetManager.scala | 7 ++-
.../spark/scheduler/TaskSetManagerSuite.scala | 49
3 files changed, 59 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/7be70e29/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 598b62f..56c0bf6 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -697,9 +697,12 @@ private[spark] class TaskSchedulerImpl(
* do not also submit those same tasks. That also means that a task
completion from an earlier
* attempt can lead to the entire stage getting marked as successful.
*/
- private[scheduler] def markPartitionCompletedInAllTaskSets(stageId: Int,
partitionId: Int) = {
+ private[scheduler] def markPartitionCompletedInAllTaskSets(
+ stageId: Int,
+ partitionId: Int,
+ taskInfo: TaskInfo) = {
taskSetsByStageIdAndAttempt.getOrElse(stageId, Map()).values.foreach { tsm
=>
- tsm.markPartitionCompleted(partitionId)
+ tsm.markPartitionCompleted(partitionId, taskInfo)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7be70e29/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index b52e376..c90ae50 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -759,7 +759,7 @@ private[spark] class TaskSetManager(
}
// There may be multiple tasksets for this stage -- we let all of them
know that the partition
// was completed. This may result in some of the tasksets getting
completed.
-sched.markPartitionCompletedInAllTaskSets(stageId,
tasks(index).partitionId)
+sched.markPartitionCompletedInAllTaskSets(stageId,
tasks(index).partitionId, info)
// This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which
holds the
// "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue,
we should not
// "deserialize" the value when holding a lock to avoid blocking other
threads. So we call
@@ -770,9 +770,12 @@ private[spark] class TaskSetManager(
maybeFinishTaskSet()
}
- private[scheduler] def markPartitionCompleted(partitionId: Int): Unit = {
+ private[scheduler] def markPartitionCompleted(partitionId: Int, taskInfo:
TaskInfo): Unit = {
partitionToIndex.get(partitionId).foreach { index =>
if (!successful(index)) {
+if (speculationEnabled && !isZombie) {
+ successfulTaskDurations.insert(taskInfo.duration)
+}
tasksSuccessful += 1
successful(index) = true
if (tasksSuccessful == numTasks) {
http://git-wip-us.apache.org/repos/asf/spark/blob/7be70e29/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
-
spark git commit: [SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
Repository: spark
Updated Branches:
refs/heads/branch-2.2 17db57213 -> 144426cff
[SPARK-24677][CORE] Avoid NoSuchElementException from MedianHeap
## What changes were proposed in this pull request?
When speculation is enabled,
TaskSetManager#markPartitionCompleted should write successful task duration to
MedianHeap,
not just increase tasksSuccessful.
Otherwise when TaskSetManager#checkSpeculatableTasks,tasksSuccessful non-zero,
but MedianHeap is empty.
Then throw an exception successfulTaskDurations.median
java.util.NoSuchElementException: MedianHeap is empty.
Finally led to stopping SparkContext.
## How was this patch tested?
TaskSetManagerSuite.scala
unit test:[SPARK-24677] MedianHeap should not be empty when speculation is
enabled
Author: sychen
Closes #21656 from cxzl25/fix_MedianHeap_empty.
(cherry picked from commit c8bee932cb644627c4049b5a07dd8028968572d9)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/144426cf
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/144426cf
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/144426cf
Branch: refs/heads/branch-2.2
Commit: 144426cffd6e4b26b676004f5489e218140f7df2
Parents: 17db572
Author: sychen
Authored: Wed Jul 18 13:24:41 2018 -0500
Committer: Thomas Graves
Committed: Wed Jul 18 13:26:24 2018 -0500
--
.../spark/scheduler/TaskSchedulerImpl.scala | 7 ++-
.../apache/spark/scheduler/TaskSetManager.scala | 7 ++-
.../spark/scheduler/TaskSetManagerSuite.scala | 49
3 files changed, 59 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/144426cf/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index df6407b..f8c62b4 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -701,9 +701,12 @@ private[spark] class TaskSchedulerImpl private[scheduler](
* do not also submit those same tasks. That also means that a task
completion from an earlier
* attempt can lead to the entire stage getting marked as successful.
*/
- private[scheduler] def markPartitionCompletedInAllTaskSets(stageId: Int,
partitionId: Int) = {
+ private[scheduler] def markPartitionCompletedInAllTaskSets(
+ stageId: Int,
+ partitionId: Int,
+ taskInfo: TaskInfo) = {
taskSetsByStageIdAndAttempt.getOrElse(stageId, Map()).values.foreach { tsm
=>
- tsm.markPartitionCompleted(partitionId)
+ tsm.markPartitionCompleted(partitionId, taskInfo)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/144426cf/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index d9515fb..705b896 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -748,7 +748,7 @@ private[spark] class TaskSetManager(
}
// There may be multiple tasksets for this stage -- we let all of them
know that the partition
// was completed. This may result in some of the tasksets getting
completed.
-sched.markPartitionCompletedInAllTaskSets(stageId,
tasks(index).partitionId)
+sched.markPartitionCompletedInAllTaskSets(stageId,
tasks(index).partitionId, info)
// This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which
holds the
// "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue,
we should not
// "deserialize" the value when holding a lock to avoid blocking other
threads. So we call
@@ -759,9 +759,12 @@ private[spark] class TaskSetManager(
maybeFinishTaskSet()
}
- private[scheduler] def markPartitionCompleted(partitionId: Int): Unit = {
+ private[scheduler] def markPartitionCompleted(partitionId: Int, taskInfo:
TaskInfo): Unit = {
partitionToIndex.get(partitionId).foreach { index =>
if (!successful(index)) {
+if (speculationEnabled && !isZombie) {
+ successfulTaskDurations.insert(taskInfo.duration)
+}
tasksSuccessful += 1
successful(index) = true
if (tasksSuccessful == numTasks) {
http://git-wip-us.apache.org/repos/asf/spark/blob/144426cf/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
-
spark git commit: [SPARK-22151] PYTHONPATH not picked up from the spark.yarn.appMaste…
Repository: spark
Updated Branches:
refs/heads/master c8bee932c -> 1272b2034
[SPARK-22151] PYTHONPATH not picked up from the spark.yarn.appMasteâ¦
â¦rEnv properly
Running in yarn cluster mode and trying to set pythonpath via
spark.yarn.appMasterEnv.PYTHONPATH doesn't work.
the yarn Client code looks at the env variables:
val pythonPathStr = (sys.env.get("PYTHONPATH") ++ pythonPath)
But when you set spark.yarn.appMasterEnv it puts it into the local env.
So the python path set in spark.yarn.appMasterEnv isn't properly set.
You can work around if you are running in cluster mode by setting it on the
client like:
PYTHONPATH=./addon/python/ spark-submit
## What changes were proposed in this pull request?
In Client.scala, PYTHONPATH was being overridden, so changed code to append
values to PYTHONPATH instead of overriding them.
## How was this patch tested?
Added log statements to ApplicationMaster.scala to check for environment
variable PYTHONPATH, ran a spark job in cluster mode before the change and
verified the issue. Performed the same test after the change and verified the
fix.
Author: pgandhi
Closes #21468 from pgandhi999/SPARK-22151.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1272b203
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1272b203
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1272b203
Branch: refs/heads/master
Commit: 1272b2034d4eed4bfe60a49e1065871b3a3f96e0
Parents: c8bee93
Author: pgandhi
Authored: Wed Jul 18 14:07:03 2018 -0500
Committer: Thomas Graves
Committed: Wed Jul 18 14:07:03 2018 -0500
--
.../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1272b203/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
--
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 793d012..ed9879c 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -811,10 +811,12 @@ private[spark] class Client(
// Finally, update the Spark config to propagate PYTHONPATH to the AM and
executors.
if (pythonPath.nonEmpty) {
- val pythonPathStr = (sys.env.get("PYTHONPATH") ++ pythonPath)
+ val pythonPathList = (sys.env.get("PYTHONPATH") ++ pythonPath)
+ env("PYTHONPATH") = (env.get("PYTHONPATH") ++ pythonPathList)
.mkString(ApplicationConstants.CLASS_PATH_SEPARATOR)
- env("PYTHONPATH") = pythonPathStr
- sparkConf.setExecutorEnv("PYTHONPATH", pythonPathStr)
+ val pythonPathExecutorEnv =
(sparkConf.getExecutorEnv.toMap.get("PYTHONPATH") ++
+pythonPathList).mkString(ApplicationConstants.CLASS_PATH_SEPARATOR)
+ sparkConf.setExecutorEnv("PYTHONPATH", pythonPathExecutorEnv)
}
if (isClusterMode) {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24755][CORE] Executor loss can cause task to not be resubmitted
Repository: spark
Updated Branches:
refs/heads/master 6a9a058e0 -> 8d707b060
[SPARK-24755][CORE] Executor loss can cause task to not be resubmitted
**Description**
As described in
[SPARK-24755](https://issues.apache.org/jira/browse/SPARK-24755), when
speculation is enabled, there is scenario that executor loss can cause task to
not be resubmitted.
This patch changes the variable killedByOtherAttempt to keeps track of the
taskId of tasks that are killed by other attempt. By doing this, we can still
prevent resubmitting task killed by other attempt while resubmit successful
attempt when executor lost.
**How was this patch tested?**
A UT is added based on the UT written by xuanyuanking with modification to
simulate the scenario described in SPARK-24755.
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Closes #21729 from hthuynh2/SPARK_24755.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8d707b06
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8d707b06
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8d707b06
Branch: refs/heads/master
Commit: 8d707b06003bc97d06630b22e6ae7c35f99b3cdd
Parents: 6a9a058
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Authored: Thu Jul 19 09:52:07 2018 -0500
Committer: Thomas Graves
Committed: Thu Jul 19 09:52:07 2018 -0500
--
.../apache/spark/scheduler/TaskSetManager.scala | 10 +-
.../spark/scheduler/TaskSetManagerSuite.scala | 112 +++
2 files changed, 117 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/8d707b06/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index 6071605..defed1e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -84,10 +84,10 @@ private[spark] class TaskSetManager(
val successful = new Array[Boolean](numTasks)
private val numFailures = new Array[Int](numTasks)
- // Set the coresponding index of Boolean var when the task killed by other
attempt tasks,
- // this happened while we set the `spark.speculation` to true. The task
killed by others
+ // Add the tid of task into this HashSet when the task is killed by other
attempt tasks.
+ // This happened while we set the `spark.speculation` to true. The task
killed by others
// should not resubmit while executor lost.
- private val killedByOtherAttempt: Array[Boolean] = new
Array[Boolean](numTasks)
+ private val killedByOtherAttempt = new HashSet[Long]
val taskAttempts = Array.fill[List[TaskInfo]](numTasks)(Nil)
private[scheduler] var tasksSuccessful = 0
@@ -735,7 +735,7 @@ private[spark] class TaskSetManager(
logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for task
${attemptInfo.id} " +
s"in stage ${taskSet.id} (TID ${attemptInfo.taskId}) on
${attemptInfo.host} " +
s"as the attempt ${info.attemptNumber} succeeded on ${info.host}")
- killedByOtherAttempt(index) = true
+ killedByOtherAttempt += attemptInfo.taskId
sched.backend.killTask(
attemptInfo.taskId,
attemptInfo.executorId,
@@ -947,7 +947,7 @@ private[spark] class TaskSetManager(
&& !isZombie) {
for ((tid, info) <- taskInfos if info.executorId == execId) {
val index = taskInfos(tid).index
-if (successful(index) && !killedByOtherAttempt(index)) {
+if (successful(index) && !killedByOtherAttempt.contains(tid)) {
successful(index) = false
copiesRunning(index) -= 1
tasksSuccessful -= 1
http://git-wip-us.apache.org/repos/asf/spark/blob/8d707b06/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
index ae571e5..206b9f4 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
@@ -1414,6 +1414,118 @@ class TaskSetManagerSuite extends SparkFunSuite with
LocalSparkContext with Logg
taskSetManager2.checkSpeculatableTasks(0)
}
+
+ test("SPARK-24755 Executor loss can cause task to not be resubmitted") {
+val conf = new SparkConf().set("spark.speculation", "true")
+sc = new SparkContext("local", "test", conf)
+// Set the speculation multiplier to be 0 so speculative tasks are
launched imm
spark git commit: [SPARK-24755][CORE] Executor loss can cause task to not be resubmitted
Repository: spark
Updated Branches:
refs/heads/branch-2.3 7be70e29d -> d0280ab81
[SPARK-24755][CORE] Executor loss can cause task to not be resubmitted
**Description**
As described in
[SPARK-24755](https://issues.apache.org/jira/browse/SPARK-24755), when
speculation is enabled, there is scenario that executor loss can cause task to
not be resubmitted.
This patch changes the variable killedByOtherAttempt to keeps track of the
taskId of tasks that are killed by other attempt. By doing this, we can still
prevent resubmitting task killed by other attempt while resubmit successful
attempt when executor lost.
**How was this patch tested?**
A UT is added based on the UT written by xuanyuanking with modification to
simulate the scenario described in SPARK-24755.
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Closes #21729 from hthuynh2/SPARK_24755.
(cherry picked from commit 8d707b06003bc97d06630b22e6ae7c35f99b3cdd)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d0280ab8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d0280ab8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d0280ab8
Branch: refs/heads/branch-2.3
Commit: d0280ab818391fd11662647459f1e9e683b2bc8e
Parents: 7be70e2
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Authored: Thu Jul 19 09:52:07 2018 -0500
Committer: Thomas Graves
Committed: Thu Jul 19 09:52:30 2018 -0500
--
.../apache/spark/scheduler/TaskSetManager.scala | 10 +-
.../spark/scheduler/TaskSetManagerSuite.scala | 112 +++
2 files changed, 117 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d0280ab8/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index c90ae50..df8d914 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -85,10 +85,10 @@ private[spark] class TaskSetManager(
val successful = new Array[Boolean](numTasks)
private val numFailures = new Array[Int](numTasks)
- // Set the coresponding index of Boolean var when the task killed by other
attempt tasks,
- // this happened while we set the `spark.speculation` to true. The task
killed by others
+ // Add the tid of task into this HashSet when the task is killed by other
attempt tasks.
+ // This happened while we set the `spark.speculation` to true. The task
killed by others
// should not resubmit while executor lost.
- private val killedByOtherAttempt: Array[Boolean] = new
Array[Boolean](numTasks)
+ private val killedByOtherAttempt = new HashSet[Long]
val taskAttempts = Array.fill[List[TaskInfo]](numTasks)(Nil)
private[scheduler] var tasksSuccessful = 0
@@ -736,7 +736,7 @@ private[spark] class TaskSetManager(
logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for task
${attemptInfo.id} " +
s"in stage ${taskSet.id} (TID ${attemptInfo.taskId}) on
${attemptInfo.host} " +
s"as the attempt ${info.attemptNumber} succeeded on ${info.host}")
- killedByOtherAttempt(index) = true
+ killedByOtherAttempt += attemptInfo.taskId
sched.backend.killTask(
attemptInfo.taskId,
attemptInfo.executorId,
@@ -942,7 +942,7 @@ private[spark] class TaskSetManager(
&& !isZombie) {
for ((tid, info) <- taskInfos if info.executorId == execId) {
val index = taskInfos(tid).index
-if (successful(index) && !killedByOtherAttempt(index)) {
+if (successful(index) && !killedByOtherAttempt.contains(tid)) {
successful(index) = false
copiesRunning(index) -= 1
tasksSuccessful -= 1
http://git-wip-us.apache.org/repos/asf/spark/blob/d0280ab8/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
index 0f4d056..b4acccf 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
@@ -1411,6 +1411,118 @@ class TaskSetManagerSuite extends SparkFunSuite with
LocalSparkContext with Logg
taskSetManager2.checkSpeculatableTasks(0)
}
+
+ test("SPARK-24755 Executor loss can cause task to not be resubmitted") {
+val conf = new SparkConf().set("spark.speculation", "true")
+sc = new SparkContext(
spark git commit: [SPARK-13343] speculative tasks that didn't commit shouldn't be marked as success
Repository: spark
Updated Branches:
refs/heads/master ee5a5a092 -> 5828f41a5
[SPARK-13343] speculative tasks that didn't commit shouldn't be marked as
success
**Description**
Currently Speculative tasks that didn't commit can show up as success
(depending on timing of commit). This is a bit confusing because that task
didn't really succeed in the sense it didn't write anything.
I think these tasks should be marked as KILLED or something that is more
obvious to the user exactly what happened. it is happened to hit the timing
where it got a commit denied exception then it shows up as failed and counts
against your task failures. It shouldn't count against task failures since that
failure really doesn't matter.
MapReduce handles these situation so perhaps we can look there for a model.
https://user-images.githubusercontent.com/15680678/42013170-99db48c2-7a61-11e8-8c7b-ef94c84e36ea.png";>
**How can this issue happen?**
When both attempts of a task finish before the driver sends command to kill one
of them, both of them send the status update FINISHED to the driver. The driver
calls TaskSchedulerImpl to handle one successful task at a time. When it
handles the first successful task, it sends the command to kill the other copy
of the task, however, because that task is already finished, the executor will
ignore the command. After finishing handling the first attempt, it processes
the second one, although all actions on the result of this task are skipped,
this copy of the task is still marked as SUCCESS. As a result, even though this
issue does not affect the result of the job, it might cause confusing to user
because both of them appear to be successful.
**How does this PR fix the issue?**
The simple way to fix this issue is that when taskSetManager handles successful
task, it checks if any other attempt succeeded. If this is the case, it will
call handleFailedTask with state==KILLED and reason==TaskKilled(âanother
attempt succeededâ) to handle this task as begin killed.
**How was this patch tested?**
I tested this manually by running applications, that caused the issue before, a
few times, and observed that the issue does not happen again. Also, I added a
unit test in TaskSetManagerSuite to test that if we call handleSuccessfulTask
to handle status update for 2 copies of a task, only the one that is handled
first will be mark as SUCCESS
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Author: hthuynh2
Closes #21653 from hthuynh2/SPARK_13343.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5828f41a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5828f41a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5828f41a
Branch: refs/heads/master
Commit: 5828f41a52c446b774a909e96eff8d8c5831b394
Parents: ee5a5a0
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Authored: Fri Jul 27 12:34:14 2018 -0500
Committer: Thomas Graves
Committed: Fri Jul 27 12:34:14 2018 -0500
--
.../apache/spark/scheduler/TaskSetManager.scala | 19 +-
.../spark/scheduler/TaskSetManagerSuite.scala | 70
2 files changed, 88 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5828f41a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index 0b21256..8b77641 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -29,7 +29,7 @@ import org.apache.spark._
import org.apache.spark.TaskState.TaskState
import org.apache.spark.internal.{config, Logging}
import org.apache.spark.scheduler.SchedulingMode._
-import org.apache.spark.util.{AccumulatorV2, Clock, SystemClock, Utils}
+import org.apache.spark.util.{AccumulatorV2, Clock, LongAccumulator,
SystemClock, Utils}
import org.apache.spark.util.collection.MedianHeap
/**
@@ -728,6 +728,23 @@ private[spark] class TaskSetManager(
def handleSuccessfulTask(tid: Long, result: DirectTaskResult[_]): Unit = {
val info = taskInfos(tid)
val index = info.index
+// Check if any other attempt succeeded before this and this attempt has
not been handled
+if (successful(index) && killedByOtherAttempt.contains(tid)) {
+ // Undo the effect on calculatedTasks and totalResultSize made earlier
when
+ // checking if can fetch more results
+ calculatedTasks -= 1
+ val resultSizeAcc = result.accumUpdates.find(a =>
+a.name == Some(InternalAccumulator.RESULT_SIZE))
+ if (resultSizeAcc.isDefined) {
+totalResultSize -
spark git commit: [SPARK-24981][CORE] ShutdownHook timeout causes job to fail when succeeded when SparkContext stop() not called by user program
Repository: spark
Updated Branches:
refs/heads/master c1760da5d -> 35700bb7f
[SPARK-24981][CORE] ShutdownHook timeout causes job to fail when succeeded when
SparkContext stop() not called by user program
**Description**
The issue is described in
[SPARK-24981](https://issues.apache.org/jira/browse/SPARK-24981).
**How does this PR fix the issue?**
This PR catch the Exception that is thrown while the Sparkcontext.stop() is
running (when it is called by the ShutdownHookManager).
**How was this patch tested?**
I manually tested it by adding delay (60s) inside the stop(). This make the
shutdownHookManger interrupt the thread that is running stop(). The Interrupted
Exception was catched and the job succeed.
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Author: Hieu Tri Huynh
Closes #21936 from hthuynh2/SPARK_24981.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/35700bb7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/35700bb7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/35700bb7
Branch: refs/heads/master
Commit: 35700bb7f2e3008ff781a1b3a1da8147d26371be
Parents: c1760da
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Authored: Mon Aug 6 09:01:51 2018 -0500
Committer: Thomas Graves
Committed: Mon Aug 6 09:01:51 2018 -0500
--
core/src/main/scala/org/apache/spark/SparkContext.scala | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/35700bb7/core/src/main/scala/org/apache/spark/SparkContext.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 03e91cd..e8bacee 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -571,7 +571,12 @@ class SparkContext(config: SparkConf) extends Logging {
_shutdownHookRef = ShutdownHookManager.addShutdownHook(
ShutdownHookManager.SPARK_CONTEXT_SHUTDOWN_PRIORITY) { () =>
logInfo("Invoking stop() from shutdown hook")
- stop()
+ try {
+stop()
+ } catch {
+case e: Throwable =>
+ logWarning("Ignoring Exception while stopping SparkContext from
shutdown hook", e)
+ }
}
} catch {
case NonFatal(e) =>
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24992][CORE] spark should randomize yarn local dir selection
Repository: spark
Updated Branches:
refs/heads/master 1a5e46076 -> 51e2b38d9
[SPARK-24992][CORE] spark should randomize yarn local dir selection
**Description:
[SPARK-24992](https://issues.apache.org/jira/browse/SPARK-24992)**
Utils.getLocalDir is used to get path of a temporary directory. However, it
always returns the the same directory, which is the first element in the array
localRootDirs. When running on YARN, this might causes the case that we always
write to one disk, which makes it busy while other disks are free. We should
randomize the selection to spread out the loads.
**What changes were proposed in this pull request?**
This PR randomized the selection of local directory inside the method
Utils.getLocalDir. This change affects the Utils.fetchFile method since it
based on the fact that Utils.getLocalDir always return the same directory to
cache file. Therefore, a new variable cachedLocalDir is used to cache the first
localDirectory that it gets from Utils.getLocalDir. Also, when getting the
configured local directories (inside Utils. getConfiguredLocalDirs), in case we
are in yarn mode, the array of directories are also randomized before return.
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Closes #21953 from hthuynh2/SPARK_24992.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/51e2b38d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/51e2b38d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/51e2b38d
Branch: refs/heads/master
Commit: 51e2b38d93df8cb0cc151d5e68a2190eab52644c
Parents: 1a5e460
Author: Hieu Huynh <âhieu.hu...@oath.comâ>
Authored: Mon Aug 6 13:58:28 2018 -0500
Committer: Thomas Graves
Committed: Mon Aug 6 13:58:28 2018 -0500
--
.../scala/org/apache/spark/util/Utils.scala | 21
1 file changed, 17 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/51e2b38d/core/src/main/scala/org/apache/spark/util/Utils.scala
--
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index a6fd363..7ec707d 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -83,6 +83,7 @@ private[spark] object Utils extends Logging {
val random = new Random()
private val sparkUncaughtExceptionHandler = new SparkUncaughtExceptionHandler
+ @volatile private var cachedLocalDir: String = ""
/**
* Define a default value for driver memory here since this value is
referenced across the code
@@ -462,7 +463,15 @@ private[spark] object Utils extends Logging {
if (useCache && fetchCacheEnabled) {
val cachedFileName = s"${url.hashCode}${timestamp}_cache"
val lockFileName = s"${url.hashCode}${timestamp}_lock"
- val localDir = new File(getLocalDir(conf))
+ // Set the cachedLocalDir for the first time and re-use it later
+ if (cachedLocalDir.isEmpty) {
+this.synchronized {
+ if (cachedLocalDir.isEmpty) {
+cachedLocalDir = getLocalDir(conf)
+ }
+}
+ }
+ val localDir = new File(cachedLocalDir)
val lockFile = new File(localDir, lockFileName)
val lockFileChannel = new RandomAccessFile(lockFile, "rw").getChannel()
// Only one executor entry.
@@ -767,13 +776,17 @@ private[spark] object Utils extends Logging {
* - Otherwise, this will return java.io.tmpdir.
*
* Some of these configuration options might be lists of multiple paths, but
this method will
- * always return a single directory.
+ * always return a single directory. The return directory is chosen randomly
from the array
+ * of directories it gets from getOrCreateLocalRootDirs.
*/
def getLocalDir(conf: SparkConf): String = {
-getOrCreateLocalRootDirs(conf).headOption.getOrElse {
+val localRootDirs = getOrCreateLocalRootDirs(conf)
+if (localRootDirs.isEmpty) {
val configuredLocalDirs = getConfiguredLocalDirs(conf)
throw new IOException(
s"Failed to get a temp directory under
[${configuredLocalDirs.mkString(",")}].")
+} else {
+ localRootDirs(scala.util.Random.nextInt(localRootDirs.length))
}
}
@@ -815,7 +828,7 @@ private[spark] object Utils extends Logging {
// to what Yarn on this system said was available. Note this assumes
that Yarn has
// created the directories already, and that they are secured so that
only the
// user has access to them.
- getYarnLocalDirs(conf).split(",")
+ randomizeInPlace(getYarnLocalDirs(conf).split(","))
} else if (conf.getenv("SPARK_EXECUTOR_DIRS") != null) {
conf.getenv("SPARK_EXECUTOR
spark git commit: [SPARK-15703][SCHEDULER][CORE][WEBUI] Make ListenerBus event queue size configurable
Repository: spark
Updated Branches:
refs/heads/master 0869b3a5f -> 0b71d9ae0
[SPARK-15703][SCHEDULER][CORE][WEBUI] Make ListenerBus event queue size
configurable
## What changes were proposed in this pull request?
This change adds a new configuration entry to specify the size of the spark
listener bus event queue. The value for this config
("spark.scheduler.listenerbus.eventqueue.size") is set to a default to 1.
Note:
I haven't currently documented the configuration entry. We can decide whether
it would be appropriate to make it a public configuration or keep it as an
undocumented one. Refer JIRA for more details.
## How was this patch tested?
Ran existing jobs and verified the event queue size with debug logs and from
the Spark WebUI Environment tab.
Author: Dhruve Ashar
Closes #14269 from dhruve/bug/SPARK-15703.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0b71d9ae
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0b71d9ae
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0b71d9ae
Branch: refs/heads/master
Commit: 0b71d9ae0804b0394e4abd02c7cebf52a9102216
Parents: 0869b3a
Author: Dhruve Ashar
Authored: Tue Jul 26 13:23:33 2016 -0500
Committer: Tom Graves
Committed: Tue Jul 26 13:23:33 2016 -0500
--
.../scala/org/apache/spark/SparkContext.scala | 4 +--
.../apache/spark/internal/config/package.scala | 5
.../spark/scheduler/LiveListenerBus.scala | 23 +--
.../scheduler/EventLoggingListenerSuite.scala | 4 +--
.../spark/scheduler/SparkListenerSuite.scala| 30 +++-
.../storage/BlockManagerReplicationSuite.scala | 9 --
.../spark/storage/BlockManagerSuite.scala | 6 ++--
.../spark/ui/storage/StorageTabSuite.scala | 11 +++
.../streaming/ReceivedBlockHandlerSuite.scala | 5 +++-
9 files changed, 60 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0b71d9ae/core/src/main/scala/org/apache/spark/SparkContext.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 6d7f05d..d48e2b4 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -249,7 +249,7 @@ class SparkContext(config: SparkConf) extends Logging with
ExecutorAllocationCli
def isStopped: Boolean = stopped.get()
// An asynchronous listener bus for Spark events
- private[spark] val listenerBus = new LiveListenerBus
+ private[spark] val listenerBus = new LiveListenerBus(this)
// This function allows components created by SparkEnv to be mocked in unit
tests:
private[spark] def createSparkEnv(
@@ -2148,7 +2148,7 @@ class SparkContext(config: SparkConf) extends Logging
with ExecutorAllocationCli
}
}
-listenerBus.start(this)
+listenerBus.start()
_listenerBusStarted = true
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0b71d9ae/core/src/main/scala/org/apache/spark/internal/config/package.scala
--
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 05dd683..ebb21e9 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -103,4 +103,9 @@ package object config {
.stringConf
.checkValues(Set("hive", "in-memory"))
.createWithDefault("in-memory")
+
+ private[spark] val LISTENER_BUS_EVENT_QUEUE_SIZE =
+ConfigBuilder("spark.scheduler.listenerbus.eventqueue.size")
+ .intConf
+ .createWithDefault(1)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0b71d9ae/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
index 1c21313..bfa3c40 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
@@ -22,7 +22,8 @@ import java.util.concurrent.atomic.AtomicBoolean
import scala.util.DynamicVariable
-import org.apache.spark.SparkContext
+import org.apache.spark.{SparkContext, SparkException}
+import org.apache.spark.internal.config._
import org.apache.spark.util.Utils
/**
@@ -32,18 +33,24 @@ import org.apache.spark.util.Utils
* has started will events be actually propagated to all attached lis
spark git commit: [SPARK-16673][WEB UI] New Executor Page removed conditional for Logs and Thread Dump columns
Repository: spark
Updated Branches:
refs/heads/master 67e59d464 -> e98eb2146
[SPARK-16673][WEB UI] New Executor Page removed conditional for Logs and Thread
Dump columns
## What changes were proposed in this pull request?
When #13670 switched `ExecutorsPage` to use JQuery DataTables it incidentally
removed the conditional for the Logs and Thread Dump columns. I reimplemented
the conditional display of the Logs and Thread dump columns as it was before
the switch.
## How was this patch tested?
Manually tested and dev/run-tests
Author: Alex Bozarth
Closes #14382 from ajbozarth/spark16673.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e98eb214
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e98eb214
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e98eb214
Branch: refs/heads/master
Commit: e98eb2146f1363956bfc3e5adcc11c246182d617
Parents: 67e59d4
Author: Alex Bozarth
Authored: Fri Aug 19 10:04:20 2016 -0500
Committer: Tom Graves
Committed: Fri Aug 19 10:04:20 2016 -0500
--
.../org/apache/spark/ui/static/executorspage.js | 38
.../apache/spark/ui/exec/ExecutorsPage.scala| 7 ++--
2 files changed, 34 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e98eb214/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
--
diff --git
a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index b2b2363..1df6733 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -15,6 +15,16 @@
* limitations under the License.
*/
+var threadDumpEnabled = false;
+
+function setThreadDumpEnabled(val) {
+threadDumpEnabled = val;
+}
+
+function getThreadDumpEnabled() {
+return threadDumpEnabled;
+}
+
function formatStatus(status, type) {
if (type !== 'display') return status;
if (status) {
@@ -116,6 +126,12 @@ function formatLogsCells(execLogs, type) {
return result;
}
+function logsExist(execs) {
+return execs.some(function(exec) {
+return !($.isEmptyObject(exec["executorLogs"]));
+});
+}
+
// Determine Color Opacity from 0.5-1
// activeTasks range from 0 to maxTasks
function activeTasksAlpha(activeTasks, maxTasks) {
@@ -143,18 +159,16 @@ function totalDurationAlpha(totalGCTime, totalDuration) {
(Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1;
}
+// When GCTimePercent is edited change ToolTips.TASK_TIME to match
+var GCTimePercent = 0.1;
+
function totalDurationStyle(totalGCTime, totalDuration) {
// Red if GC time over GCTimePercent of total time
-// When GCTimePercent is edited change ToolTips.TASK_TIME to match
-var GCTimePercent = 0.1;
return (totalGCTime > GCTimePercent * totalDuration) ?
("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime,
totalDuration) + ")") : "";
}
function totalDurationColor(totalGCTime, totalDuration) {
-// Red if GC time over GCTimePercent of total time
-// When GCTimePercent is edited change ToolTips.TASK_TIME to match
-var GCTimePercent = 0.1;
return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black";
}
@@ -392,8 +406,18 @@ $(document).ready(function () {
{data: 'executorLogs', render: formatLogsCells},
{
data: 'id', render: function (data, type) {
-return type === 'display' ? ("Thread Dump" ) : data;
+return type === 'display' ? ("Thread Dump" ) : data;
+}
}
+],
+"columnDefs": [
+{
+"targets": [ 15 ],
+"visible": logsExist(response)
+},
+{
+"targets": [ 16 ],
+"visible": getThreadDumpEnabled()
}
],
"order": [[0, "asc"]]
@@ -458,7 +482,7 @@ $(document).ready(function () {
"paging": false,
"searching": false,
"info": false
-
+
spark git commit: [SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is enabled.
Repository: spark
Updated Branches:
refs/heads/master e98eb2146 -> 071eaaf9d
[SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is
enabled.
## What changes were proposed in this pull request?
If the following conditions are satisfied, executors don't load properties in
`hdfs-site.xml` and UnknownHostException can be thrown.
(1) NameNode HA is enabled
(2) spark.eventLogging is disabled or logging path is NOT on HDFS
(3) Using Standalone or Mesos for the cluster manager
(4) There are no code to load `HdfsCondition` class in the driver regardless of
directly or indirectly.
(5) The tasks access to HDFS
(There might be some more conditions...)
For example, following code causes UnknownHostException when the conditions
above are satisfied.
```
sc.textFile("").collect
```
```
java.lang.IllegalArgumentException: java.net.UnknownHostException: hacluster
at
org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:378)
at
org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:310)
at
org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176)
at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:678)
at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:619)
at
org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:149)
at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:170)
at
org.apache.hadoop.mapred.JobConf.getWorkingDirectory(JobConf.java:656)
at
org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:438)
at
org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:411)
at
org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986)
at
org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986)
at
org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177)
at
org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177)
at scala.Option.map(Option.scala:146)
at org.apache.spark.rdd.HadoopRDD.getJobConf(HadoopRDD.scala:177)
at org.apache.spark.rdd.HadoopRDD$$anon$1.(HadoopRDD.scala:213)
at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:209)
at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:102)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:282)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:282)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70)
at org.apache.spark.scheduler.Task.run(Task.scala:85)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.UnknownHostException: hacluster
```
But following code doesn't cause the Exception because `textFile` method loads
`HdfsConfiguration` indirectly.
```
sc.textFile("").collect
```
When a job includes some operations which access to HDFS, the object of
`org.apache.hadoop.Configuration` is wrapped by `SerializableConfiguration`,
serialized and broadcasted from driver to executors and each executor
deserialize the object with `loadDefaults` false so HDFS related properties
should be set before broadcasted.
## How was this patch tested?
Tested manually on my standalone cluster.
Author: Kousuke Saruta
Closes #13738 from sarutak/SPARK-11227.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/071eaaf9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/071eaaf9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/071eaaf9
Branch: refs/heads/master
Commit: 071eaaf9d2b63589f2e66e5279a16a5a484de6f5
Parents: e98eb21
Author: Kousuke Saruta
Authored: Fri Aug 19 10:11:25 2016 -0500
Committer: Tom Graves
Committed: Fri Aug 19 10:11:25 2016 -0500
--
.../scala/org/apache/spark/SparkContext.scala | 22 +++-
1 file changed, 21 insertions(+),
spark git commit: [SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is enabled.
Repository: spark
Updated Branches:
refs/heads/branch-2.0 e0c60f185 -> d0707c6ba
[SPARK-11227][CORE] UnknownHostException can be thrown when NameNode HA is
enabled.
## What changes were proposed in this pull request?
If the following conditions are satisfied, executors don't load properties in
`hdfs-site.xml` and UnknownHostException can be thrown.
(1) NameNode HA is enabled
(2) spark.eventLogging is disabled or logging path is NOT on HDFS
(3) Using Standalone or Mesos for the cluster manager
(4) There are no code to load `HdfsCondition` class in the driver regardless of
directly or indirectly.
(5) The tasks access to HDFS
(There might be some more conditions...)
For example, following code causes UnknownHostException when the conditions
above are satisfied.
```
sc.textFile("").collect
```
```
java.lang.IllegalArgumentException: java.net.UnknownHostException: hacluster
at
org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:378)
at
org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:310)
at
org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:176)
at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:678)
at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:619)
at
org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:149)
at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:170)
at
org.apache.hadoop.mapred.JobConf.getWorkingDirectory(JobConf.java:656)
at
org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:438)
at
org.apache.hadoop.mapred.FileInputFormat.setInputPaths(FileInputFormat.java:411)
at
org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986)
at
org.apache.spark.SparkContext$$anonfun$hadoopFile$1$$anonfun$32.apply(SparkContext.scala:986)
at
org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177)
at
org.apache.spark.rdd.HadoopRDD$$anonfun$getJobConf$6.apply(HadoopRDD.scala:177)
at scala.Option.map(Option.scala:146)
at org.apache.spark.rdd.HadoopRDD.getJobConf(HadoopRDD.scala:177)
at org.apache.spark.rdd.HadoopRDD$$anon$1.(HadoopRDD.scala:213)
at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:209)
at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:102)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:282)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:318)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:282)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70)
at org.apache.spark.scheduler.Task.run(Task.scala:85)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.UnknownHostException: hacluster
```
But following code doesn't cause the Exception because `textFile` method loads
`HdfsConfiguration` indirectly.
```
sc.textFile("").collect
```
When a job includes some operations which access to HDFS, the object of
`org.apache.hadoop.Configuration` is wrapped by `SerializableConfiguration`,
serialized and broadcasted from driver to executors and each executor
deserialize the object with `loadDefaults` false so HDFS related properties
should be set before broadcasted.
## How was this patch tested?
Tested manually on my standalone cluster.
Author: Kousuke Saruta
Closes #13738 from sarutak/SPARK-11227.
(cherry picked from commit 071eaaf9d2b63589f2e66e5279a16a5a484de6f5)
Signed-off-by: Tom Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d0707c6b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d0707c6b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d0707c6b
Branch: refs/heads/branch-2.0
Commit: d0707c6baeb4003735a508f98db370984354
Parents: e0c60f1
Author: Kousuke Saruta
Authored: Fri Aug 19 10:11:25 2016 -0500
Committer: Tom Graves
Committed: Fri Aug 19 10:11:41 2016 -0500
--
...
spark git commit: [SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData
Repository: spark
Updated Branches:
refs/heads/master 40b30fcf4 -> 891ac2b91
[SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData
## What changes were proposed in this pull request?
Based on #12990 by tankkyo
Since the History Server currently loads all application's data it can OOM if
too many applications have a significant task count. `spark.ui.trimTasks`
(default: false) can be set to true to trim tasks by `spark.ui.retainedTasks`
(default: 1)
(This is a "quick fix" to help those running into the problem until a update of
how the history server loads app data can be done)
## How was this patch tested?
Manual testing and dev/run-tests
Author: Alex Bozarth
Closes #14673 from ajbozarth/spark15083.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/891ac2b9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/891ac2b9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/891ac2b9
Branch: refs/heads/master
Commit: 891ac2b914fb6f90a62c6fbc0a3960a89d1c1d92
Parents: 40b30fc
Author: Alex Bozarth
Authored: Wed Aug 24 14:39:41 2016 -0500
Committer: Tom Graves
Committed: Wed Aug 24 14:39:41 2016 -0500
--
.../apache/spark/internal/config/package.scala | 5 +
.../spark/ui/jobs/JobProgressListener.scala | 9 +-
.../org/apache/spark/ui/jobs/StagePage.scala| 12 +-
.../scala/org/apache/spark/ui/jobs/UIData.scala | 4 +-
.../stage_task_list_w__sortBy_expectation.json | 130 ++---
...ortBy_short_names___runtime_expectation.json | 130 ++---
...sortBy_short_names__runtime_expectation.json | 182 +--
.../status/api/v1/AllStagesResourceSuite.scala | 4 +-
docs/configuration.md | 8 +
9 files changed, 256 insertions(+), 228 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/891ac2b9/core/src/main/scala/org/apache/spark/internal/config/package.scala
--
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index be3dac4..47174e4 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -114,4 +114,9 @@ package object config {
private[spark] val PYSPARK_PYTHON = ConfigBuilder("spark.pyspark.python")
.stringConf
.createOptional
+
+ // To limit memory usage, we only track information for a fixed number of
tasks
+ private[spark] val UI_RETAINED_TASKS =
ConfigBuilder("spark.ui.retainedTasks")
+.intConf
+.createWithDefault(10)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/891ac2b9/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index 491f716..d3a4f9d 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -19,12 +19,13 @@ package org.apache.spark.ui.jobs
import java.util.concurrent.TimeoutException
-import scala.collection.mutable.{HashMap, HashSet, ListBuffer}
+import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap, ListBuffer}
import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
import org.apache.spark.scheduler._
import org.apache.spark.scheduler.SchedulingMode.SchedulingMode
import org.apache.spark.storage.BlockManagerId
@@ -93,6 +94,7 @@ class JobProgressListener(conf: SparkConf) extends
SparkListener with Logging {
val retainedStages = conf.getInt("spark.ui.retainedStages",
SparkUI.DEFAULT_RETAINED_STAGES)
val retainedJobs = conf.getInt("spark.ui.retainedJobs",
SparkUI.DEFAULT_RETAINED_JOBS)
+ val retainedTasks = conf.get(UI_RETAINED_TASKS)
// We can test for memory leaks by ensuring that collections that track
non-active jobs and
// stages do not grow without bound and that collections for active
jobs/stages eventually become
@@ -405,6 +407,11 @@ class JobProgressListener(conf: SparkConf) extends
SparkListener with Logging {
taskData.updateTaskMetrics(taskMetrics)
taskData.errorMessage = errorMessage
+ // If Tasks is too large, remove and garbage collect old tasks
+ if (stageData.taskData.size >
spark git commit: [SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData
Repository: spark
Updated Branches:
refs/heads/branch-2.0 356a359de -> 55db26245
[SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData
## What changes were proposed in this pull request?
This is a back port of #14673 addressing merge conflicts in package.scala that
prevented a cherry-pick to `branch-2.0` when it was merged to `master`
Since the History Server currently loads all application's data it can OOM if
too many applications have a significant task count. This trims tasks by
`spark.ui.retainedTasks` (default: 10)
## How was this patch tested?
Manual testing and dev/run-tests
Author: Alex Bozarth
Closes #14794 from ajbozarth/spark15083-branch-2.0.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/55db2624
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/55db2624
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/55db2624
Branch: refs/heads/branch-2.0
Commit: 55db26245d69bb02b7d7d5f25029b1a1cd571644
Parents: 356a359
Author: Alex Bozarth
Authored: Thu Aug 25 11:54:55 2016 -0500
Committer: Tom Graves
Committed: Thu Aug 25 11:54:55 2016 -0500
--
.../apache/spark/internal/config/package.scala | 5 +
.../spark/ui/jobs/JobProgressListener.scala | 9 +-
.../org/apache/spark/ui/jobs/StagePage.scala| 12 +-
.../scala/org/apache/spark/ui/jobs/UIData.scala | 4 +-
.../stage_task_list_w__sortBy_expectation.json | 130 ++---
...ortBy_short_names___runtime_expectation.json | 130 ++---
...sortBy_short_names__runtime_expectation.json | 182 +--
.../status/api/v1/AllStagesResourceSuite.scala | 4 +-
docs/configuration.md | 8 +
9 files changed, 256 insertions(+), 228 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/55db2624/core/src/main/scala/org/apache/spark/internal/config/package.scala
--
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 05dd683..5a59aee 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -103,4 +103,9 @@ package object config {
.stringConf
.checkValues(Set("hive", "in-memory"))
.createWithDefault("in-memory")
+
+ // To limit memory usage, we only track information for a fixed number of
tasks
+ private[spark] val UI_RETAINED_TASKS =
ConfigBuilder("spark.ui.retainedTasks")
+.intConf
+.createWithDefault(10)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/55db2624/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index 842f42b..38ad6e9 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -19,12 +19,13 @@ package org.apache.spark.ui.jobs
import java.util.concurrent.TimeoutException
-import scala.collection.mutable.{HashMap, HashSet, ListBuffer}
+import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap, ListBuffer}
import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
import org.apache.spark.scheduler._
import org.apache.spark.scheduler.SchedulingMode.SchedulingMode
import org.apache.spark.storage.BlockManagerId
@@ -93,6 +94,7 @@ class JobProgressListener(conf: SparkConf) extends
SparkListener with Logging {
val retainedStages = conf.getInt("spark.ui.retainedStages",
SparkUI.DEFAULT_RETAINED_STAGES)
val retainedJobs = conf.getInt("spark.ui.retainedJobs",
SparkUI.DEFAULT_RETAINED_JOBS)
+ val retainedTasks = conf.get(UI_RETAINED_TASKS)
// We can test for memory leaks by ensuring that collections that track
non-active jobs and
// stages do not grow without bound and that collections for active
jobs/stages eventually become
@@ -400,6 +402,11 @@ class JobProgressListener(conf: SparkConf) extends
SparkListener with Logging {
taskData.updateTaskMetrics(taskMetrics)
taskData.errorMessage = errorMessage
+ // If Tasks is too large, remove and garbage collect old tasks
+ if (stageData.taskData.size > retainedTasks) {
+stageData.taskData = stageData.taskData.drop(stageData.taskData.size -
retainedTasks)
+ }
+
for (
activeJobsDependentOnStage <-
sta
spark git commit: [SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large application history
Repository: spark
Updated Branches:
refs/heads/master 02ac379e8 -> f7beae6da
[SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large
application history
## What changes were proposed in this pull request?
With the new History Server the summary page loads the application list via the
the REST API, this makes it very slow to impossible to load with large (10K+)
application history. This pr fixes this by adding the
`spark.history.ui.maxApplications` conf to limit the number of applications the
History Server displays. This is accomplished using a new optional `limit`
param for the `applications` api. (Note this only applies to what the summary
page displays, all the Application UI's are still accessible if the user knows
the App ID and goes to the Application UI directly.)
I've also added a new test for the `limit` param in `HistoryServerSuite.scala`
## How was this patch tested?
Manual testing and dev/run-tests
Author: Alex Bozarth
Closes #14835 from ajbozarth/spark17243.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f7beae6d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f7beae6d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f7beae6d
Branch: refs/heads/master
Commit: f7beae6da02e6b70a34c714e93136becbde7679b
Parents: 02ac379
Author: Alex Bozarth
Authored: Tue Aug 30 16:33:54 2016 -0500
Committer: Tom Graves
Committed: Tue Aug 30 16:33:54 2016 -0500
--
.../org/apache/spark/ui/static/historypage.js | 8 ++-
.../spark/deploy/history/HistoryPage.scala | 3 +-
.../spark/deploy/history/HistoryServer.scala| 4 ++
.../apache/spark/internal/config/package.scala | 4 ++
.../status/api/v1/ApplicationListResource.scala | 10 ++-
.../limit_app_list_json_expectation.json| 67
.../deploy/history/HistoryServerSuite.scala | 1 +
docs/monitoring.md | 16 -
8 files changed, 106 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f7beae6d/core/src/main/resources/org/apache/spark/ui/static/historypage.js
--
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index 5b9afb5..c809400 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -15,6 +15,12 @@
* limitations under the License.
*/
+var appLimit = -1;
+
+function setAppLimit(val) {
+appLimit = val;
+}
+
function makeIdNumeric(id) {
var strs = id.split("_");
if (strs.length < 3) {
@@ -89,7 +95,7 @@ $(document).ready(function() {
requestedIncomplete = getParameterByName("showIncomplete", searchString);
requestedIncomplete = (requestedIncomplete == "true" ? true : false);
-$.getJSON("api/v1/applications", function(response,status,jqXHR) {
+$.getJSON("api/v1/applications?limit=" + appLimit,
function(response,status,jqXHR) {
var array = [];
var hasMultipleAttempts = false;
for (i in response) {
http://git-wip-us.apache.org/repos/asf/spark/blob/f7beae6d/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
index 74f7802..b4f5a61 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -45,7 +45,8 @@ private[history] class HistoryPage(parent: HistoryServer)
extends WebUIPage("")
++
++
++
-
+ ++
+setAppLimit({parent.maxApplications})
} else if (requestedIncomplete) {
No incomplete applications found!
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/f7beae6d/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index d821474..c178917 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -28,6 +28,7 @@ import org.eclipse.jetty.servlet.{ServletContextHandler,
ServletHolder}
import org.apache.spark.{SecurityManager, SparkConf}
import org
spark git commit: [SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large application history
Repository: spark
Updated Branches:
refs/heads/branch-2.0 bc6c0d9f9 -> 021aa28f4
[SPARK-17243][WEB UI] Spark 2.0 History Server won't load with very large
application history
## What changes were proposed in this pull request?
back port of #14835 addressing merge conflicts
With the new History Server the summary page loads the application list via the
the REST API, this makes it very slow to impossible to load with large (10K+)
application history. This pr fixes this by adding the
`spark.history.ui.maxApplications` conf to limit the number of applications the
History Server displays. This is accomplished using a new optional `limit`
param for the `applications` api. (Note this only applies to what the summary
page displays, all the Application UI's are still accessible if the user knows
the App ID and goes to the Application UI directly.)
I've also added a new test for the `limit` param in `HistoryServerSuite.scala`
## How was this patch tested?
Manual testing and dev/run-tests
Author: Alex Bozarth
Closes #14886 from ajbozarth/spark17243-branch-2.0.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/021aa28f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/021aa28f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/021aa28f
Branch: refs/heads/branch-2.0
Commit: 021aa28f439443cda1bc7c5e3eee7c85b40c1a2d
Parents: bc6c0d9
Author: Alex Bozarth
Authored: Wed Aug 31 08:50:42 2016 -0500
Committer: Tom Graves
Committed: Wed Aug 31 08:50:42 2016 -0500
--
.../org/apache/spark/ui/static/historypage.js | 8 ++-
.../spark/deploy/history/HistoryPage.scala | 3 +-
.../spark/deploy/history/HistoryServer.scala| 4 ++
.../apache/spark/internal/config/package.scala | 4 ++
.../status/api/v1/ApplicationListResource.scala | 10 ++-
.../limit_app_list_json_expectation.json| 67
.../deploy/history/HistoryServerSuite.scala | 1 +
docs/monitoring.md | 16 -
8 files changed, 106 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/021aa28f/core/src/main/resources/org/apache/spark/ui/static/historypage.js
--
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index d216166..177120a 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -15,6 +15,12 @@
* limitations under the License.
*/
+var appLimit = -1;
+
+function setAppLimit(val) {
+appLimit = val;
+}
+
// this function works exactly the same as UIUtils.formatDuration
function formatDuration(milliseconds) {
if (milliseconds < 100) {
@@ -111,7 +117,7 @@ $(document).ready(function() {
requestedIncomplete = getParameterByName("showIncomplete", searchString);
requestedIncomplete = (requestedIncomplete == "true" ? true : false);
-$.getJSON("api/v1/applications", function(response,status,jqXHR) {
+$.getJSON("api/v1/applications?limit=" + appLimit,
function(response,status,jqXHR) {
var array = [];
var hasMultipleAttempts = false;
for (i in response) {
http://git-wip-us.apache.org/repos/asf/spark/blob/021aa28f/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
index 2fad112..a120b6c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -44,7 +44,8 @@ private[history] class HistoryPage(parent: HistoryServer)
extends WebUIPage("")
if (allAppsSize > 0) {
++
++
-
+
++
+ setAppLimit({parent.maxApplications})
} else if (requestedIncomplete) {
No incomplete applications found!
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/021aa28f/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index d821474..c178917 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -28,6 +28,7 @@ import
spark git commit: [SPARK-16711] YarnShuffleService doesn't re-init properly on YARN rolling upgrade
Repository: spark
Updated Branches:
refs/heads/branch-2.0 28377da38 -> e169085cd
[SPARK-16711] YarnShuffleService doesn't re-init properly on YARN rolling
upgrade
branch-2.0 version of this patch. The differences are in the
YarnShuffleService for finding the location to put the DB. branch-2.0 does not
use the yarn nm recovery path like master does.
Tested in manually on 8 node yarn cluster and ran unit tests. Manually tests
verified DB created properly and it found them if already existed. Verified
that during rolling upgrade credentials were reloaded and running application
was not affected.
Author: Thomas Graves
Closes #14997 from tgravescs/SPARK-16711-branch2.0.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e169085c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e169085c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e169085c
Branch: refs/heads/branch-2.0
Commit: e169085cd33ff498ecd5aab180af036ca644c9e0
Parents: 28377da
Author: Thomas Graves
Authored: Thu Sep 8 08:16:19 2016 -0500
Committer: Tom Graves
Committed: Thu Sep 8 08:16:19 2016 -0500
--
common/network-common/pom.xml | 16 ++
.../spark/network/util/LevelDBProvider.java | 152 ++
common/network-shuffle/pom.xml | 16 --
.../shuffle/ExternalShuffleBlockResolver.java | 120 +--
.../spark/network/yarn/YarnShuffleService.java | 198 +++
.../network/yarn/YarnShuffleServiceSuite.scala | 10 +
6 files changed, 345 insertions(+), 167 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e169085c/common/network-common/pom.xml
--
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index bc3b0fe..e170b9b 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -42,6 +42,22 @@
netty-all
+
+ org.fusesource.leveldbjni
+ leveldbjni-all
+ 1.8
+
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
org.slf4j
http://git-wip-us.apache.org/repos/asf/spark/blob/e169085c/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java
--
diff --git
a/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java
b/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java
new file mode 100644
index 000..ec900a7
--- /dev/null
+++
b/common/network-common/src/main/java/org/apache/spark/network/util/LevelDBProvider.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.fusesource.leveldbjni.internal.NativeDB;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * LevelDB utility class available in the network package.
+ */
+public class LevelDBProvider {
+ private static final Logger logger =
LoggerFactory.getLogger(LevelDBProvider.class);
+
+ public static DB initLevelDB(File dbFile, StoreVersion version, ObjectMapper
mapper) throws
+ IOException {
+DB tmpDb = null;
+if (dbFile != null) {
+ Options options = new Options();
+ options.createIfMissing(false);
+ options.logger(new LevelDBLogger());
+ try {
+tmpDb = JniDBFactory.factory.open(dbFile, options);
+ } catch (NativeDB.DBException e) {
+if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
+
spark git commit: [SPARK-17433] YarnShuffleService doesn't handle moving credentials levelDb
Repository: spark
Updated Branches:
refs/heads/master 7098a1294 -> a3981c28c
[SPARK-17433] YarnShuffleService doesn't handle moving credentials levelDb
The secrets leveldb isn't being moved if you run spark shuffle services without
yarn nm recovery on and then turn it on. This fixes that. I unfortunately
missed this when I ported the patch from our internal branch 2 to master branch
due to the changes for the recovery path. Note this only applies to master
since it is the only place the yarn nm recovery dir is used.
Unit tests ran and tested on 8 node cluster. Fresh startup with NM recovery,
fresh startup no nm recovery, switching between no nm recovery and recovery.
Also tested running applications to make sure wasn't affected by rolling
upgrade.
Author: Thomas Graves
Author: Tom Graves
Closes #14999 from tgravescs/SPARK-17433.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a3981c28
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a3981c28
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a3981c28
Branch: refs/heads/master
Commit: a3981c28c956a82ccf5b1c61d45b6bd252d4abed
Parents: 7098a12
Author: Thomas Graves
Authored: Fri Sep 9 13:43:32 2016 -0500
Committer: Tom Graves
Committed: Fri Sep 9 13:43:32 2016 -0500
--
.../spark/network/yarn/YarnShuffleService.java | 56 ++--
.../network/yarn/YarnShuffleServiceSuite.scala | 12 -
2 files changed, 50 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a3981c28/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
--
diff --git
a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
index df082e4..43c8df7 100644
---
a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
+++
b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
import java.util.List;
import java.util.Map;
@@ -159,8 +160,7 @@ public class YarnShuffleService extends AuxiliaryService {
// If we don't find one, then we choose a file to use to save the state
next time. Even if
// an application was stopped while the NM was down, we expect yarn to
call stopApplication()
// when it comes back
- registeredExecutorFile =
-new File(getRecoveryPath().toUri().getPath(), RECOVERY_FILE_NAME);
+ registeredExecutorFile = initRecoveryDb(RECOVERY_FILE_NAME);
TransportConf transportConf = new TransportConf("shuffle", new
HadoopConfigProvider(conf));
blockHandler = new ExternalShuffleBlockHandler(transportConf,
registeredExecutorFile);
@@ -196,7 +196,7 @@ public class YarnShuffleService extends AuxiliaryService {
private void createSecretManager() throws IOException {
secretManager = new ShuffleSecretManager();
-secretsFile = new File(getRecoveryPath().toUri().getPath(),
SECRETS_RECOVERY_FILE_NAME);
+secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME);
// Make sure this is protected in case its not in the NM recovery dir
FileSystem fs = FileSystem.getLocal(_conf);
@@ -328,37 +328,59 @@ public class YarnShuffleService extends AuxiliaryService {
}
/**
- * Get the recovery path, this will override the default one to get our own
maintained
- * recovery path.
+ * Get the path specific to this auxiliary service to use for recovery.
+ */
+ protected Path getRecoveryPath(String fileName) {
+return _recoveryPath;
+ }
+
+ /**
+ * Figure out the recovery path and handle moving the DB if YARN NM recovery
gets enabled
+ * when it previously was not. If YARN NM recovery is enabled it uses that
path, otherwise
+ * it will uses a YARN local dir.
*/
- protected Path getRecoveryPath() {
+ protected File initRecoveryDb(String dbFileName) {
+if (_recoveryPath != null) {
+File recoveryFile = new File(_recoveryPath.toUri().getPath(),
dbFileName);
+if (recoveryFile.exists()) {
+ return recoveryFile;
+}
+}
+// db doesn't exist in recovery path go check local dirs for it
String[] localDirs =
_conf.getTrimmedStrings("yarn.nodemanager.local-dirs");
for (String dir : localDirs) {
- File f = new File(new Path(dir).toUri().getPath(), RECOVERY_FILE_NAME);
+ File f = new File(new Path(dir).toUri().getPath(), dbFileName);
if (f.exists()) {
if (_recoveryPath == null) {
spark git commit: [SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as Failed
Repository: spark
Updated Branches:
refs/heads/master 040e46979 -> ff6e4cbdc
[SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as
Failed
## What changes were proposed in this pull request?
Due to race conditions, the ` assert(numExecutorsRunning <=
targetNumExecutors)` can fail causing `AssertionError`. So removed the
assertion, instead moved the conditional check before launching new container:
```
java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:156)
at
org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1.org$apache$spark$deploy$yarn$YarnAllocator$$anonfun$$updateInternalState$1(YarnAllocator.scala:489)
at
org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1$$anon$1.run(YarnAllocator.scala:519)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
```
## How was this patch tested?
This was manually tested using a large ForkAndJoin job with Dynamic Allocation
enabled to validate the failing job succeeds, without any such exception.
Author: Kishor Patil
Closes #15069 from kishorvpatil/SPARK-17511.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ff6e4cbd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ff6e4cbd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ff6e4cbd
Branch: refs/heads/master
Commit: ff6e4cbdc80e2ad84c5d70ee07f323fad9374e3e
Parents: 040e469
Author: Kishor Patil
Authored: Wed Sep 14 14:19:35 2016 -0500
Committer: Tom Graves
Committed: Wed Sep 14 14:19:35 2016 -0500
--
.../spark/deploy/yarn/YarnAllocator.scala | 62 +++-
.../spark/deploy/yarn/YarnAllocatorSuite.scala | 19 ++
2 files changed, 52 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ff6e4cbd/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
--
diff --git
a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index 2f4b498..0b66d1c 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -496,7 +496,6 @@ private[yarn] class YarnAllocator(
def updateInternalState(): Unit = synchronized {
numExecutorsRunning += 1
-assert(numExecutorsRunning <= targetNumExecutors)
executorIdToContainer(executorId) = container
containerIdToExecutorId(container.getId) = executorId
@@ -506,36 +505,41 @@ private[yarn] class YarnAllocator(
allocatedContainerToHostMap.put(containerId, executorHostname)
}
- if (launchContainers) {
-launcherPool.execute(new Runnable {
- override def run(): Unit = {
-try {
- new ExecutorRunnable(
-Some(container),
-conf,
-sparkConf,
-driverUrl,
-executorId,
-executorHostname,
-executorMemory,
-executorCores,
-appAttemptId.getApplicationId.toString,
-securityMgr,
-localResources
- ).run()
- updateInternalState()
-} catch {
- case NonFatal(e) =>
-logError(s"Failed to launch executor $executorId on container
$containerId", e)
-// Assigned container should be released immediately to avoid
unnecessary resource
-// occupation.
-amClient.releaseAssignedContainer(containerId)
+ if (numExecutorsRunning < targetNumExecutors) {
+if (launchContainers) {
+ launcherPool.execute(new Runnable {
+override def run(): Unit = {
+ try {
+new ExecutorRunnable(
+ Some(container),
+ conf,
+ sparkConf,
+ driverUrl,
+ executorId,
+ executorHostname,
+ executorMemory,
+ executorCores,
+ appAttemptId.getApplicationId.toString,
+ securityMgr,
+ localResources
+).run()
+updateInternalState()
+ } catch {
+case NonFatal(e) =>
+ logError(s"Failed to launch executor $executorId on
container $containerId", e)
+ // Assigned container should be released i
spark git commit: [SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as Failed
Repository: spark
Updated Branches:
refs/heads/branch-2.0 6fe5972e6 -> fab77dadf
[SPARK-17511] Yarn Dynamic Allocation: Avoid marking released container as
Failed
Due to race conditions, the ` assert(numExecutorsRunning <=
targetNumExecutors)` can fail causing `AssertionError`. So removed the
assertion, instead moved the conditional check before launching new container:
```
java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:156)
at
org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1.org$apache$spark$deploy$yarn$YarnAllocator$$anonfun$$updateInternalState$1(YarnAllocator.scala:489)
at
org.apache.spark.deploy.yarn.YarnAllocator$$anonfun$runAllocatedContainers$1$$anon$1.run(YarnAllocator.scala:519)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
```
This was manually tested using a large ForkAndJoin job with Dynamic Allocation
enabled to validate the failing job succeeds, without any such exception.
Author: Kishor Patil
Closes #15069 from kishorvpatil/SPARK-17511.
(cherry picked from commit ff6e4cbdc80e2ad84c5d70ee07f323fad9374e3e)
Signed-off-by: Tom Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fab77dad
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fab77dad
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fab77dad
Branch: refs/heads/branch-2.0
Commit: fab77dadf70d011cec8976acfe8c851816f82426
Parents: 6fe5972
Author: Kishor Patil
Authored: Wed Sep 14 14:19:35 2016 -0500
Committer: Tom Graves
Committed: Wed Sep 14 14:33:40 2016 -0500
--
.../spark/deploy/yarn/YarnAllocator.scala | 68 +++-
.../spark/deploy/yarn/YarnAllocatorSuite.scala | 19 ++
2 files changed, 55 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/fab77dad/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
--
diff --git
a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index 1b80071..b321901 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -483,7 +483,6 @@ private[yarn] class YarnAllocator(
def updateInternalState(): Unit = synchronized {
numExecutorsRunning += 1
-assert(numExecutorsRunning <= targetNumExecutors)
executorIdToContainer(executorId) = container
containerIdToExecutorId(container.getId) = executorId
@@ -493,39 +492,44 @@ private[yarn] class YarnAllocator(
allocatedContainerToHostMap.put(containerId, executorHostname)
}
- if (launchContainers) {
-logInfo("Launching ExecutorRunnable. driverUrl: %s, executorHostname:
%s".format(
- driverUrl, executorHostname))
-
-launcherPool.execute(new Runnable {
- override def run(): Unit = {
-try {
- new ExecutorRunnable(
-container,
-conf,
-sparkConf,
-driverUrl,
-executorId,
-executorHostname,
-executorMemory,
-executorCores,
-appAttemptId.getApplicationId.toString,
-securityMgr,
-localResources
- ).run()
- updateInternalState()
-} catch {
- case NonFatal(e) =>
-logError(s"Failed to launch executor $executorId on container
$containerId", e)
-// Assigned container should be released immediately to avoid
unnecessary resource
-// occupation.
-amClient.releaseAssignedContainer(containerId)
+ if (numExecutorsRunning < targetNumExecutors) {
+if (launchContainers) {
+ logInfo("Launching ExecutorRunnable. driverUrl: %s,
executorHostname: %s".format(
+driverUrl, executorHostname))
+
+ launcherPool.execute(new Runnable {
+override def run(): Unit = {
+ try {
+new ExecutorRunnable(
+ container,
+ conf,
+ sparkConf,
+ driverUrl,
+ executorId,
+ executorHostname,
+ executorMemory,
+ executorCores,
+ appAttemptId.getApplicationId.toString,
+ securityMgr,
+ localResources
+
spark git commit: [SPARK-16757] Set up Spark caller context to HDFS and YARN
Repository: spark Updated Branches: refs/heads/master 7f16affa2 -> 6a68c5d7b [SPARK-16757] Set up Spark caller context to HDFS and YARN ## What changes were proposed in this pull request? 1. Pass `jobId` to Task. 2. Invoke Hadoop APIs. * A new function `setCallerContext` is added in `Utils`. `setCallerContext` function invokes APIs of `org.apache.hadoop.ipc.CallerContext` to set up spark caller contexts, which will be written into `hdfs-audit.log` and Yarn RM audit log. * For HDFS: Spark sets up its caller context by invoking`org.apache.hadoop.ipc.CallerContext` in `Task` and Yarn `Client` and `ApplicationMaster`. * For Yarn: Spark sets up its caller context by invoking `org.apache.hadoop.ipc.CallerContext` in Yarn `Client`. ## How was this patch tested? Manual Tests against some Spark applications in Yarn client mode and Yarn cluster mode. Need to check if spark caller contexts are written into HDFS hdfs-audit.log and Yarn RM audit log successfully. For example, run SparkKmeans in Yarn client mode: ``` ./bin/spark-submit --verbose --executor-cores 3 --num-executors 1 --master yarn --deploy-mode client --class org.apache.spark.examples.SparkKMeans examples/target/original-spark-examples_2.11-2.1.0-SNAPSHOT.jar hdfs://localhost:9000/lr_big.txt 2 5 ``` **Before**: There will be no Spark caller context in records of `hdfs-audit.log` and Yarn RM audit log. **After**: Spark caller contexts will be written in records of `hdfs-audit.log` and Yarn RM audit log. These are records in `hdfs-audit.log`: ``` 2016-09-20 11:54:24,116 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_CLIENT_AppId_application_1474394339641_0005 2016-09-20 11:54:28,164 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0005_JobId_0_StageId_0_AttemptId_0_TaskId_2_AttemptNum_0 2016-09-20 11:54:28,164 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0005_JobId_0_StageId_0_AttemptId_0_TaskId_1_AttemptNum_0 2016-09-20 11:54:28,164 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0005_JobId_0_StageId_0_AttemptId_0_TaskId_0_AttemptNum_0 ``` ``` 2016-09-20 11:59:33,868 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=mkdirs src=/private/tmp/hadoop-wyang/nm-local-dir/usercache/wyang/appcache/application_1474394339641_0006/container_1474394339641_0006_01_01/spark-warehouse dst=nullperm=wyang:supergroup:rwxr-xr-x proto=rpc callerContext=SPARK_APPLICATION_MASTER_AppId_application_1474394339641_0006_AttemptId_1 2016-09-20 11:59:37,214 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_1_AttemptNum_0 2016-09-20 11:59:37,215 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_2_AttemptNum_0 2016-09-20 11:59:37,215 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_0_AttemptNum_0 2016-09-20 11:59:42,391 INFO FSNamesystem.audit: allowed=true ugi=wyang (auth:SIMPLE) ip=/127.0.0.1 cmd=opensrc=/lr_big.txt dst=null perm=null proto=rpc callerContext=SPARK_TASK_AppId_application_1474394339641_0006_AttemptId_1_JobId_0_StageId_0_AttemptId_0_TaskId_3_AttemptNum_0 ``` This is a record in Yarn RM log: ``` 2016-09-20 11:59:24,050 INFO org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger: USER=wyang IP=127.0.0.1OPERATION=Submit Application RequestTARGET=ClientRMService RESULT=SUCCESS APPID=application_1474394339641_0006 CALLERCONTEXT=SPARK_CLIENT_AppId_application_1474394339641_0006 ``` Author: Weiqing Yang Closes #14659 from Sherry302/callercontextSubmit. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a68c5d7 Tree: http://git-wip-us.apache.or
spark git commit: [SPARK-17710][HOTFIX] Fix ClassCircularityError in ReplSuite tests in Maven build: use 'Class.forName' instead of 'Utils.classForName'
Repository: spark
Updated Branches:
refs/heads/master 7d0923202 -> 7dfad4b13
[SPARK-17710][HOTFIX] Fix ClassCircularityError in ReplSuite tests in Maven
build: use 'Class.forName' instead of 'Utils.classForName'
## What changes were proposed in this pull request?
Fix ClassCircularityError in ReplSuite tests when Spark is built by Maven build.
## How was this patch tested?
(1)
```
build/mvn -DskipTests -Phadoop-2.3 -Pyarn -Phive -Phive-thriftserver
-Pkinesis-asl -Pmesos clean package
```
Then test:
```
build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.repl.ReplSuite test
```
ReplSuite tests passed
(2)
Manual Tests against some Spark applications in Yarn client mode and Yarn
cluster mode. Need to check if spark caller contexts are written into HDFS
hdfs-audit.log and Yarn RM audit log successfully.
Author: Weiqing Yang
Closes #15286 from Sherry302/SPARK-16757.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7dfad4b1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7dfad4b1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7dfad4b1
Branch: refs/heads/master
Commit: 7dfad4b132bc46263ef788ced4a935862f5c8756
Parents: 7d09232
Author: Weiqing Yang
Authored: Wed Sep 28 20:20:03 2016 -0500
Committer: Tom Graves
Committed: Wed Sep 28 20:20:03 2016 -0500
--
core/src/main/scala/org/apache/spark/util/Utils.scala | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/7dfad4b1/core/src/main/scala/org/apache/spark/util/Utils.scala
--
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index caa768c..f3493bd 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2489,8 +2489,10 @@ private[spark] class CallerContext(
def setCurrentContext(): Boolean = {
var succeed = false
try {
- val callerContext =
Utils.classForName("org.apache.hadoop.ipc.CallerContext")
- val Builder =
Utils.classForName("org.apache.hadoop.ipc.CallerContext$Builder")
+ // scalastyle:off classforname
+ val callerContext = Class.forName("org.apache.hadoop.ipc.CallerContext")
+ val Builder =
Class.forName("org.apache.hadoop.ipc.CallerContext$Builder")
+ // scalastyle:on classforname
val builderInst =
Builder.getConstructor(classOf[String]).newInstance(context)
val hdfsContext = Builder.getMethod("build").invoke(builderInst)
callerContext.getMethod("setCurrent", callerContext).invoke(null,
hdfsContext)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-25043] print master and appId from spark-sql on startup
Repository: spark
Updated Branches:
refs/heads/master 102487584 -> b81e3031f
[SPARK-25043] print master and appId from spark-sql on startup
## What changes were proposed in this pull request?
A small change to print the master and appId from spark-sql as with logging
turned down all the way
(`log4j.logger.org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver=WARN`),
we may not know this information easily. This adds the following string before
the `spark-sql>` prompt shows on the screen.
`Spark master: yarn, Application Id: application_123456789_12345`
## How was this patch tested?
I ran spark-sql locally and saw the appId displayed as expected.
Please review http://spark.apache.org/contributing.html before opening a pull
request.
Closes #22025 from abellina/SPARK-25043_print_master_and_app_id_from_sparksql.
Lead-authored-by: Alessandro Bellina
Co-authored-by: Alessandro Bellina
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b81e3031
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b81e3031
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b81e3031
Branch: refs/heads/master
Commit: b81e3031fd247dfb4b3e02e0a986fb4b19d00f7c
Parents: 1024875
Author: Alessandro Bellina
Authored: Tue Aug 14 13:15:55 2018 -0500
Committer: Thomas Graves
Committed: Tue Aug 14 13:15:55 2018 -0500
--
.../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 8
1 file changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b81e3031/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
--
diff --git
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index d9fd3eb..bb96cea 100644
---
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -258,6 +258,8 @@ private[hive] object SparkSQLCLIDriver extends Logging {
def continuedPromptWithDBSpaces: String = continuedPrompt +
ReflectionUtils.invokeStatic(
classOf[CliDriver], "spacesForString", classOf[String] -> currentDB)
+cli.printMasterAndAppId
+
var currentPrompt = promptWithCurrentDB
var line = reader.readLine(currentPrompt + "> ")
@@ -323,6 +325,12 @@ private[hive] class SparkSQLCLIDriver extends CliDriver
with Logging {
hiveVariables.asScala.foreach(kv =>
SparkSQLEnv.sqlContext.conf.setConfString(kv._1, kv._2))
}
+ def printMasterAndAppId(): Unit = {
+val master = SparkSQLEnv.sparkContext.master
+val appId = SparkSQLEnv.sparkContext.applicationId
+console.printInfo(s"Spark master: $master, Application Id: $appId")
+ }
+
override def processCmd(cmd: String): Int = {
val cmd_trimmed: String = cmd.trim()
val cmd_lower = cmd_trimmed.toLowerCase(Locale.ROOT)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24909][CORE] Always unregister pending partition on task completion.
Repository: spark
Updated Branches:
refs/heads/branch-2.3 b072717b3 -> dbf0b9340
[SPARK-24909][CORE] Always unregister pending partition on task completion.
Spark scheduler can hang when fetch failures, executor lost, task running on
lost executor, and multiple stage attempts. To fix this we change to always
unregister the pending partition on task completion.
this PR is actually reverting the change in SPARK-19263, so that it always does
shuffleStage.pendingPartitions -= task.partitionId. The change in
SPARK-23433, should fix the issue originally from SPARK-19263.
Unit tests. The condition happens on a race which I haven't reproduced on a
real customer, just see it sometimes on customers jobs in a real cluster.
I am also working on adding spark scheduler integration tests.
Closes #21976 from tgravescs/SPARK-24909.
Authored-by: Thomas Graves
Signed-off-by: Marcelo Vanzin
(cherry picked from commit ec3e9986385880adce1648eae30007eccff862ba)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dbf0b934
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dbf0b934
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dbf0b934
Branch: refs/heads/branch-2.3
Commit: dbf0b934024dfa562775967e3e3114de5a163443
Parents: b072717
Author: Thomas Graves
Authored: Wed Aug 29 16:32:02 2018 -0700
Committer: Thomas Graves
Committed: Thu Aug 30 09:10:00 2018 -0500
--
.../apache/spark/scheduler/DAGScheduler.scala | 17 +-
.../spark/scheduler/DAGSchedulerSuite.scala | 24
2 files changed, 16 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/dbf0b934/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
--
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index 7029e22..0df38f1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -1252,18 +1252,10 @@ class DAGScheduler(
case smt: ShuffleMapTask =>
val shuffleStage = stage.asInstanceOf[ShuffleMapStage]
+shuffleStage.pendingPartitions -= task.partitionId
val status = event.result.asInstanceOf[MapStatus]
val execId = status.location.executorId
logDebug("ShuffleMapTask finished on " + execId)
-if (stageIdToStage(task.stageId).latestInfo.attemptNumber ==
task.stageAttemptId) {
- // This task was for the currently running attempt of the stage.
Since the task
- // completed successfully from the perspective of the
TaskSetManager, mark it as
- // no longer pending (the TaskSetManager may consider the task
complete even
- // when the output needs to be ignored because the task's epoch
is too small below.
- // In this case, when pending partitions is empty, there will
still be missing
- // output locations, which will cause the DAGScheduler to
resubmit the stage below.)
- shuffleStage.pendingPartitions -= task.partitionId
-}
if (failedEpoch.contains(execId) && smt.epoch <=
failedEpoch(execId)) {
logInfo(s"Ignoring possibly bogus $smt completion from executor
$execId")
} else {
@@ -1272,13 +1264,6 @@ class DAGScheduler(
// available.
mapOutputTracker.registerMapOutput(
shuffleStage.shuffleDep.shuffleId, smt.partitionId, status)
- // Remove the task's partition from pending partitions. This may
have already been
- // done above, but will not have been done yet in cases where
the task attempt was
- // from an earlier attempt of the stage (i.e., not the attempt
that's currently
- // running). This allows the DAGScheduler to mark the stage as
complete when one
- // copy of each task has finished successfully, even if the
currently active stage
- // still has tasks running.
- shuffleStage.pendingPartitions -= task.partitionId
}
if (runningStages.contains(shuffleStage) &&
shuffleStage.pendingPartitions.isEmpty) {
http://git-wip-us.apache.org/repos/asf/spark/blob/dbf0b934/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 8b6ec37..c0
spark git commit: [SPARK-25231] Fix synchronization of executor heartbeat receiver in TaskSchedulerImpl
Repository: spark
Updated Branches:
refs/heads/master 925449283 -> 559b899ac
[SPARK-25231] Fix synchronization of executor heartbeat receiver in
TaskSchedulerImpl
Running a large Spark job with speculation turned on was causing executor
heartbeats to time out on the driver end after sometime and eventually, after
hitting the max number of executor failures, the job would fail.
## What changes were proposed in this pull request?
The main reason for the heartbeat timeouts was that the
heartbeat-receiver-event-loop-thread was blocked waiting on the
TaskSchedulerImpl object which was being held by one of the
dispatcher-event-loop threads executing the method dequeueSpeculativeTasks() in
TaskSetManager.scala. On further analysis of the heartbeat receiver method
executorHeartbeatReceived() in TaskSchedulerImpl class, we found out that
instead of waiting to acquire the lock on the TaskSchedulerImpl object, we can
remove that lock and make the operations to the global variables inside the
code block to be atomic. The block of code in that method only uses one global
HashMap taskIdToTaskSetManager. Making that map a ConcurrentHashMap, we are
ensuring atomicity of operations and speeding up the heartbeat receiver thread
operation.
## How was this patch tested?
Screenshots of the thread dump have been attached below:
**heartbeat-receiver-event-loop-thread:**
https://user-images.githubusercontent.com/8190/44593413-e25df780-a788-11e8-9520-176a18401a59.png";>
**dispatcher-event-loop-thread:**
https://user-images.githubusercontent.com/8190/44593484-13d6c300-a789-11e8-8d88-34b1d51d4541.png";>
Closes #1 from pgandhi999/SPARK-25231.
Authored-by: pgandhi
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/559b899a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/559b899a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/559b899a
Branch: refs/heads/master
Commit: 559b899aceb160fcec3a57109c0b60a0ae40daeb
Parents: 9254492
Author: pgandhi
Authored: Wed Sep 5 16:10:49 2018 -0500
Committer: Thomas Graves
Committed: Wed Sep 5 16:10:49 2018 -0500
--
.../org/apache/spark/scheduler/TaskSchedulerImpl.scala | 12 ++--
.../cluster/CoarseGrainedSchedulerBackend.scala | 2 +-
.../spark/scheduler/SchedulerIntegrationSuite.scala | 3 ++-
.../apache/spark/scheduler/TaskSchedulerImplSuite.scala | 6 +++---
4 files changed, 12 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/559b899a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 8992d7e..8b71170 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler
import java.nio.ByteBuffer
import java.util.{Locale, Timer, TimerTask}
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
import java.util.concurrent.atomic.AtomicLong
import scala.collection.Set
@@ -91,7 +91,7 @@ private[spark] class TaskSchedulerImpl(
private val taskSetsByStageIdAndAttempt = new HashMap[Int, HashMap[Int,
TaskSetManager]]
// Protected by `this`
- private[scheduler] val taskIdToTaskSetManager = new HashMap[Long,
TaskSetManager]
+ private[scheduler] val taskIdToTaskSetManager = new ConcurrentHashMap[Long,
TaskSetManager]
val taskIdToExecutorId = new HashMap[Long, String]
@volatile private var hasReceivedTask = false
@@ -315,7 +315,7 @@ private[spark] class TaskSchedulerImpl(
for (task <- taskSet.resourceOffer(execId, host, maxLocality)) {
tasks(i) += task
val tid = task.taskId
-taskIdToTaskSetManager(tid) = taskSet
+taskIdToTaskSetManager.put(tid, taskSet)
taskIdToExecutorId(tid) = execId
executorIdToRunningTaskIds(execId).add(tid)
availableCpus(i) -= CPUS_PER_TASK
@@ -465,7 +465,7 @@ private[spark] class TaskSchedulerImpl(
var reason: Option[ExecutorLossReason] = None
synchronized {
try {
-taskIdToTaskSetManager.get(tid) match {
+Option(taskIdToTaskSetManager.get(tid)) match {
case Some(taskSet) =>
if (state == TaskState.LOST) {
// TaskState.LOST is only used by the deprecated Mesos
fine-grained scheduling mode,
@@ -517,10 +517,10 @@ private[spark] class TaskSchedulerImpl(
accumUpdates: Array[(Long, Seq[AccumulatorV2[_,
spark git commit: [SPARK-25231] Fix synchronization of executor heartbeat receiver in TaskSchedulerImpl
Repository: spark
Updated Branches:
refs/heads/branch-2.3 dbf0b9340 -> 31e46ec60
[SPARK-25231] Fix synchronization of executor heartbeat receiver in
TaskSchedulerImpl
Running a large Spark job with speculation turned on was causing executor
heartbeats to time out on the driver end after sometime and eventually, after
hitting the max number of executor failures, the job would fail.
## What changes were proposed in this pull request?
The main reason for the heartbeat timeouts was that the
heartbeat-receiver-event-loop-thread was blocked waiting on the
TaskSchedulerImpl object which was being held by one of the
dispatcher-event-loop threads executing the method dequeueSpeculativeTasks() in
TaskSetManager.scala. On further analysis of the heartbeat receiver method
executorHeartbeatReceived() in TaskSchedulerImpl class, we found out that
instead of waiting to acquire the lock on the TaskSchedulerImpl object, we can
remove that lock and make the operations to the global variables inside the
code block to be atomic. The block of code in that method only uses one global
HashMap taskIdToTaskSetManager. Making that map a ConcurrentHashMap, we are
ensuring atomicity of operations and speeding up the heartbeat receiver thread
operation.
## How was this patch tested?
Screenshots of the thread dump have been attached below:
**heartbeat-receiver-event-loop-thread:**
https://user-images.githubusercontent.com/8190/44593413-e25df780-a788-11e8-9520-176a18401a59.png";>
**dispatcher-event-loop-thread:**
https://user-images.githubusercontent.com/8190/44593484-13d6c300-a789-11e8-8d88-34b1d51d4541.png";>
Closes #1 from pgandhi999/SPARK-25231.
Authored-by: pgandhi
Signed-off-by: Thomas Graves
(cherry picked from commit 559b899aceb160fcec3a57109c0b60a0ae40daeb)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/31e46ec6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/31e46ec6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/31e46ec6
Branch: refs/heads/branch-2.3
Commit: 31e46ec60849d315a4e83e0a332606a4405907ad
Parents: dbf0b93
Author: pgandhi
Authored: Wed Sep 5 16:10:49 2018 -0500
Committer: Thomas Graves
Committed: Wed Sep 5 16:11:08 2018 -0500
--
.../org/apache/spark/scheduler/TaskSchedulerImpl.scala | 12 ++--
.../cluster/CoarseGrainedSchedulerBackend.scala | 2 +-
.../spark/scheduler/SchedulerIntegrationSuite.scala | 3 ++-
.../apache/spark/scheduler/TaskSchedulerImplSuite.scala | 6 +++---
4 files changed, 12 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/31e46ec6/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 56c0bf6..4edc6b2 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler
import java.nio.ByteBuffer
import java.util.{Locale, Timer, TimerTask}
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
import java.util.concurrent.atomic.AtomicLong
import scala.collection.Set
@@ -90,7 +90,7 @@ private[spark] class TaskSchedulerImpl(
private val taskSetsByStageIdAndAttempt = new HashMap[Int, HashMap[Int,
TaskSetManager]]
// Protected by `this`
- private[scheduler] val taskIdToTaskSetManager = new HashMap[Long,
TaskSetManager]
+ private[scheduler] val taskIdToTaskSetManager = new ConcurrentHashMap[Long,
TaskSetManager]
val taskIdToExecutorId = new HashMap[Long, String]
@volatile private var hasReceivedTask = false
@@ -286,7 +286,7 @@ private[spark] class TaskSchedulerImpl(
for (task <- taskSet.resourceOffer(execId, host, maxLocality)) {
tasks(i) += task
val tid = task.taskId
-taskIdToTaskSetManager(tid) = taskSet
+taskIdToTaskSetManager.put(tid, taskSet)
taskIdToExecutorId(tid) = execId
executorIdToRunningTaskIds(execId).add(tid)
availableCpus(i) -= CPUS_PER_TASK
@@ -392,7 +392,7 @@ private[spark] class TaskSchedulerImpl(
var reason: Option[ExecutorLossReason] = None
synchronized {
try {
-taskIdToTaskSetManager.get(tid) match {
+Option(taskIdToTaskSetManager.get(tid)) match {
case Some(taskSet) =>
if (state == TaskState.LOST) {
// TaskState.LOST is only used by the deprecated Mesos
fine-grained scheduling mode,
@@ -444,10
spark git commit: [SPARK-24415][CORE] Fixed the aggregated stage metrics by retaining stage objects in liveStages until all tasks are complete
Repository: spark
Updated Branches:
refs/heads/branch-2.3 84922e506 -> 5b8b6b4e9
[SPARK-24415][CORE] Fixed the aggregated stage metrics by retaining stage
objects in liveStages until all tasks are complete
The problem occurs because stage object is removed from liveStages in
AppStatusListener onStageCompletion. Because of this any onTaskEnd event
received after onStageCompletion event do not update stage metrics.
The fix is to retain stage objects in liveStages until all tasks are complete.
1. Fixed the reproducible example posted in the JIRA
2. Added unit test
Closes #22209 from ankuriitg/ankurgupta/SPARK-24415.
Authored-by: ankurgupta
Signed-off-by: Marcelo Vanzin
(cherry picked from commit 39a02d8f75def7191c66d388729ba1721c92188d)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5b8b6b4e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5b8b6b4e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5b8b6b4e
Branch: refs/heads/branch-2.3
Commit: 5b8b6b4e9e36228e993a15cab19c80e7fad43786
Parents: 84922e5
Author: ankurgupta
Authored: Wed Sep 5 09:41:05 2018 -0700
Committer: Thomas Graves
Committed: Fri Sep 7 08:48:39 2018 -0500
--
.../apache/spark/status/AppStatusListener.scala | 61 +++-
.../spark/status/AppStatusListenerSuite.scala | 55 ++
.../spark/streaming/UISeleniumSuite.scala | 9 ++-
3 files changed, 108 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5b8b6b4e/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 496165c..9375ae9 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -310,11 +310,20 @@ private[spark] class AppStatusListener(
val e = it.next()
if (job.stageIds.contains(e.getKey()._1)) {
val stage = e.getValue()
- stage.status = v1.StageStatus.SKIPPED
- job.skippedStages += stage.info.stageId
- job.skippedTasks += stage.info.numTasks
- it.remove()
- update(stage, now)
+ if (v1.StageStatus.PENDING.equals(stage.status)) {
+stage.status = v1.StageStatus.SKIPPED
+job.skippedStages += stage.info.stageId
+job.skippedTasks += stage.info.numTasks
+job.activeStages -= 1
+
+pools.get(stage.schedulingPool).foreach { pool =>
+ pool.stageIds = pool.stageIds - stage.info.stageId
+ update(pool, now)
+}
+
+it.remove()
+update(stage, now, last = true)
+ }
}
}
@@ -466,7 +475,16 @@ private[spark] class AppStatusListener(
if (killedDelta > 0) {
stage.killedSummary = killedTasksSummary(event.reason,
stage.killedSummary)
}
- maybeUpdate(stage, now)
+ // [SPARK-24415] Wait for all tasks to finish before removing stage from
live list
+ val removeStage =
+stage.activeTasks == 0 &&
+ (v1.StageStatus.COMPLETE.equals(stage.status) ||
+v1.StageStatus.FAILED.equals(stage.status))
+ if (removeStage) {
+update(stage, now, last = true)
+ } else {
+maybeUpdate(stage, now)
+ }
// Store both stage ID and task index in a single long variable for
tracking at job level.
val taskIndex = (event.stageId.toLong << Integer.SIZE) |
event.taskInfo.index
@@ -481,7 +499,7 @@ private[spark] class AppStatusListener(
if (killedDelta > 0) {
job.killedSummary = killedTasksSummary(event.reason,
job.killedSummary)
}
-maybeUpdate(job, now)
+conditionalLiveUpdate(job, now, removeStage)
}
val esummary = stage.executorSummary(event.taskInfo.executorId)
@@ -492,7 +510,7 @@ private[spark] class AppStatusListener(
if (metricsDelta != null) {
esummary.metrics = LiveEntityHelpers.addMetrics(esummary.metrics,
metricsDelta)
}
- maybeUpdate(esummary, now)
+ conditionalLiveUpdate(esummary, now, removeStage)
if (!stage.cleaning && stage.savedTasks.get() > maxTasksPerStage) {
stage.cleaning = true
@@ -500,6 +518,9 @@ private[spark] class AppStatusListener(
cleanupTasks(stage)
}
}
+ if (removeStage) {
+liveStages.remove((event.stageId, event.stageAttemptId))
+ }
}
liveExecutors.get(event.taskInfo.executorId).foreach { exec =>
@@ -524,17 +545,13 @@ private[spark] clas
spark git commit: [SPARK-24355] Spark external shuffle server improvement to better handle block fetch requests.
Repository: spark Updated Branches: refs/heads/master 2c9d8f56c -> ff601cf71 [SPARK-24355] Spark external shuffle server improvement to better handle block fetch requests. ## What changes were proposed in this pull request? Description: Right now, the default server side netty handler threads is 2 * # cores, and can be further configured with parameter spark.shuffle.io.serverThreads. In order to process a client request, it would require one available server netty handler thread. However, when the server netty handler threads start to process ChunkFetchRequests, they will be blocked on disk I/O, mostly due to disk contentions from the random read operations initiated by all the ChunkFetchRequests received from clients. As a result, when the shuffle server is serving many concurrent ChunkFetchRequests, the server side netty handler threads could all be blocked on reading shuffle files, thus leaving no handler thread available to process other types of requests which should all be very quick to process. This issue could potentially be fixed by limiting the number of netty handler threads that could get blocked when processing ChunkFetchRequest. We have a patch to do this by using a separate EventLoopGroup with a dedicated ChannelHandler to process ChunkFetchRequest. This enables shuffle server to reserve netty handler threads for non-ChunkFetchRequest, thus enabling consistent processing time for these requests which are fast to process. After deploying the patch in our infrastructure, we no longer see timeout issues with either executor registration with local shuffle server or shuffle client establishing connection with remote shuffle server. (Please fill in changes proposed in this fix) For Original PR please refer here https://github.com/apache/spark/pull/21402 ## How was this patch tested? Unit tests and stress testing. (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) Please review http://spark.apache.org/contributing.html before opening a pull request. Closes #22173 from redsanket/SPARK-24335. Authored-by: Sanket Chintapalli Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ff601cf7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ff601cf7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ff601cf7 Branch: refs/heads/master Commit: ff601cf71d226082e156c4ff9a8f5593aa7a2085 Parents: 2c9d8f5 Author: Sanket Chintapalli Authored: Fri Sep 21 09:05:56 2018 -0500 Committer: Thomas Graves Committed: Fri Sep 21 09:05:56 2018 -0500 -- .../apache/spark/network/TransportContext.java | 66 - .../server/ChunkFetchRequestHandler.java| 135 +++ .../network/server/TransportChannelHandler.java | 21 ++- .../network/server/TransportRequestHandler.java | 35 + .../spark/network/util/TransportConf.java | 28 .../network/ChunkFetchRequestHandlerSuite.java | 102 ++ .../spark/network/ExtendedChannelPromise.java | 69 ++ .../network/TransportRequestHandlerSuite.java | 55 +--- .../network/shuffle/ExternalShuffleClient.java | 2 +- 9 files changed, 425 insertions(+), 88 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/ff601cf7/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java -- diff --git a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java index ae91bc9..480b526 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java +++ b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; import io.netty.channel.Channel; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.handler.timeout.IdleStateHandler; import org.slf4j.Logger; @@ -32,11 +34,13 @@ import org.apache.spark.network.client.TransportClientFactory; import org.apache.spark.network.client.TransportResponseHandler; import org.apache.spark.network.protocol.MessageDecoder; import org.apache.spark.network.protocol.MessageEncoder; +import org.apache.spark.network.server.ChunkFetchRequestHandler; import org.apache.spark.network.server.RpcHandler; import org.apache.spark.network.server.TransportChannelHandler; import org.apache.spark.network.server.TransportRequestHandler; import org.apache.spark.network.serve
spark git commit: [SPARK-18364][YARN] Expose metrics for YarnShuffleService
Repository: spark
Updated Branches:
refs/heads/master b96fd44f0 -> a802c69b1
[SPARK-18364][YARN] Expose metrics for YarnShuffleService
## What changes were proposed in this pull request?
This PR is follow-up of closed https://github.com/apache/spark/pull/17401
which only ended due to of inactivity, but its still nice feature to have.
Given review by jerryshao taken in consideration and edited:
- VisibleForTesting deleted because of dependency conflicts
- removed unnecessary reflection for `MetricsSystemImpl`
- added more available types for gauge
## How was this patch tested?
Manual deploy of new yarn-shuffle jar into a Node Manager and verifying that
the metrics appear in the Node Manager-standard location. This is JMX with an
query endpoint running on `hostname:port`
Resulting metrics look like this:
```
curl -sk -XGET hostname:port | grep -v '#' | grep 'shuffleService'
hadoop_nodemanager_openblockrequestlatencymillis_rate15{name="shuffleService",}
0.31428910657834713
hadoop_nodemanager_blocktransferratebytes_rate15{name="shuffleService",}
566144.9983653595
hadoop_nodemanager_blocktransferratebytes_ratemean{name="shuffleService",}
2464409.9678099006
hadoop_nodemanager_openblockrequestlatencymillis_rate1{name="shuffleService",}
1.2893844732240272
hadoop_nodemanager_registeredexecutorssize{name="shuffleService",} 2.0
hadoop_nodemanager_openblockrequestlatencymillis_ratemean{name="shuffleService",}
1.255574678369966
hadoop_nodemanager_openblockrequestlatencymillis_count{name="shuffleService",}
315.0
hadoop_nodemanager_openblockrequestlatencymillis_rate5{name="shuffleService",}
0.7661929192569739
hadoop_nodemanager_registerexecutorrequestlatencymillis_ratemean{name="shuffleService",}
0.0
hadoop_nodemanager_registerexecutorrequestlatencymillis_count{name="shuffleService",}
0.0
hadoop_nodemanager_registerexecutorrequestlatencymillis_rate1{name="shuffleService",}
0.0
hadoop_nodemanager_registerexecutorrequestlatencymillis_rate5{name="shuffleService",}
0.0
hadoop_nodemanager_blocktransferratebytes_count{name="shuffleService",}
6.18271213E8
hadoop_nodemanager_registerexecutorrequestlatencymillis_rate15{name="shuffleService",}
0.0
hadoop_nodemanager_blocktransferratebytes_rate5{name="shuffleService",}
1154114.4881816586
hadoop_nodemanager_blocktransferratebytes_rate1{name="shuffleService",}
574745.0749848988
```
Closes #22485 from mareksimunek/SPARK-18364.
Lead-authored-by: marek.simunek
Co-authored-by: Andrew Ash
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a802c69b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a802c69b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a802c69b
Branch: refs/heads/master
Commit: a802c69b130b69a35b372ffe1b01289577f6fafb
Parents: b96fd44
Author: marek.simunek
Authored: Mon Oct 1 11:04:37 2018 -0500
Committer: Thomas Graves
Committed: Mon Oct 1 11:04:37 2018 -0500
--
.../spark/network/yarn/YarnShuffleService.java | 11 ++
.../network/yarn/YarnShuffleServiceMetrics.java | 137 +++
.../yarn/YarnShuffleServiceMetricsSuite.scala | 73 ++
3 files changed, 221 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a802c69b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
--
diff --git
a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
index d8b2ed6..72ae1a1 100644
---
a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
+++
b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.api.*;
import org.apache.spark.network.util.LevelDBProvider;
@@ -168,6 +170,15 @@ public class YarnShuffleService extends AuxiliaryService {
TransportConf transportConf = new TransportConf("shuffle", new
HadoopConfigProvider(conf));
blockHandler = new ExternalShuffleBlockHandler(transportConf,
registeredExecutorFile);
+ // register metrics on the block handler into the Node Manager's metrics
system.
+ YarnShuffleServiceMetrics serviceMetrics =
+new YarnShuffleServiceMetrics(blockHandler.getAllMetrics());
+
+ MetricsSystemImpl metricsSy
spark git commit: [SPARK-25641] Change the spark.shuffle.server.chunkFetchHandlerThreadsPercent default to 100
Repository: spark
Updated Branches:
refs/heads/master 1a2862535 -> 6353425af
[SPARK-25641] Change the spark.shuffle.server.chunkFetchHandlerThreadsPercent
default to 100
## What changes were proposed in this pull request?
We want to change the default percentage to 100 for
spark.shuffle.server.chunkFetchHandlerThreadsPercent. The reason being
currently this is set to 0. Which means currently if server.ioThreads > 0, the
default number of threads would be 2 * #cores instead of server.io.Threads. We
want the default to server.io.Threads in case this is not set at all. Also here
a default of 0 would also mean 2 * #cores
## How was this patch tested?
Manual
(Please explain how this patch was tested. E.g. unit tests, integration tests,
manual tests)
(If this patch involves UI changes, please attach a screenshot; otherwise,
remove this)
Please review http://spark.apache.org/contributing.html before opening a pull
request.
Closes #22628 from redsanket/SPARK-25641.
Lead-authored-by: Sanket Chintapalli
Co-authored-by: Sanket Chintapalli
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6353425a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6353425a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6353425a
Branch: refs/heads/master
Commit: 6353425af76f9cc9de7ee4094f41df7a7390d898
Parents: 1a28625
Author: Sanket Chintapalli
Authored: Mon Oct 8 13:19:34 2018 -0500
Committer: Thomas Graves
Committed: Mon Oct 8 13:19:34 2018 -0500
--
.../apache/spark/network/util/TransportConf.java| 16 ++--
1 file changed, 10 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6353425a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
--
diff --git
a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index 6d5cccd..43a6bc7 100644
---
a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++
b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -296,17 +296,21 @@ public class TransportConf {
* and could take long time to process due to disk contentions. By
configuring a slightly
* higher number of shuffler server threads, we are able to reserve some
threads for
* handling other RPC messages, thus making the Client less likely to
experience timeout
- * when sending RPC messages to the shuffle server. Default to 0, which is
2*#cores
- * or io.serverThreads. 90 would mean 90% of 2*#cores or 90% of
io.serverThreads
- * which equals 0.9 * 2*#cores or 0.9 * io.serverThreads.
+ * when sending RPC messages to the shuffle server. The number of threads
used for handling
+ * chunked fetch requests are percentage of io.serverThreads (if defined)
else it is a percentage
+ * of 2 * #cores. However, a percentage of 0 means netty default number of
threads which
+ * is 2 * #cores ignoring io.serverThreads. The percentage here is
configured via
+ * spark.shuffle.server.chunkFetchHandlerThreadsPercent. The returned value
is rounded off to
+ * ceiling of the nearest integer.
*/
public int chunkFetchHandlerThreads() {
if (!this.getModuleName().equalsIgnoreCase("shuffle")) {
return 0;
}
int chunkFetchHandlerThreadsPercent =
- conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 0);
-return this.serverThreads() > 0 ? (this.serverThreads() *
chunkFetchHandlerThreadsPercent)/100:
- (2 * NettyRuntime.availableProcessors() *
chunkFetchHandlerThreadsPercent)/100;
+ conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 100);
+return (int)Math.ceil(
+ (this.serverThreads() > 0 ? this.serverThreads() : 2 *
NettyRuntime.availableProcessors()) *
+ chunkFetchHandlerThreadsPercent/(double)100);
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24851][UI] Map a Stage ID to it's Associated Job ID
Repository: spark
Updated Branches:
refs/heads/master e3133f4ab -> deb9588b2
[SPARK-24851][UI] Map a Stage ID to it's Associated Job ID
It would be nice to have a field in Stage Page UI which would show mapping of
the current stage id to the job id's to which that stage belongs to.
## What changes were proposed in this pull request?
Added a field in Stage UI to display the corresponding job id for that
particular stage.
## How was this patch tested?
https://user-images.githubusercontent.com/8190/43220447-a8e94f80-900f-11e8-8a20-a235bbd5a369.png";>
Closes #21809 from pgandhi999/SPARK-24851.
Authored-by: pgandhi
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/deb9588b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/deb9588b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/deb9588b
Branch: refs/heads/master
Commit: deb9588b2ab6596b30ab17f56c59951cabf57162
Parents: e3133f4
Author: pgandhi
Authored: Tue Oct 9 08:59:21 2018 -0500
Committer: Thomas Graves
Committed: Tue Oct 9 08:59:21 2018 -0500
--
.../scala/org/apache/spark/status/AppStatusStore.scala | 8 +---
.../apache/spark/status/api/v1/StagesResource.scala| 2 +-
.../scala/org/apache/spark/ui/jobs/StagePage.scala | 13 +++--
3 files changed, 17 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/deb9588b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
--
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index e237281..9839cbb 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -112,10 +112,12 @@ private[spark] class AppStatusStore(
}
}
- def stageAttempt(stageId: Int, stageAttemptId: Int, details: Boolean =
false): v1.StageData = {
+ def stageAttempt(stageId: Int, stageAttemptId: Int,
+ details: Boolean = false): (v1.StageData, Seq[Int]) = {
val stageKey = Array(stageId, stageAttemptId)
-val stage = store.read(classOf[StageDataWrapper], stageKey).info
-if (details) stageWithDetails(stage) else stage
+val stageDataWrapper = store.read(classOf[StageDataWrapper], stageKey)
+val stage = if (details) stageWithDetails(stageDataWrapper.info) else
stageDataWrapper.info
+(stage, stageDataWrapper.jobIds.toSeq)
}
def taskCount(stageId: Int, stageAttemptId: Int): Long = {
http://git-wip-us.apache.org/repos/asf/spark/blob/deb9588b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
index 96249e4..30d52b9 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
@@ -56,7 +56,7 @@ private[v1] class StagesResource extends BaseAppResource {
@PathParam("stageAttemptId") stageAttemptId: Int,
@QueryParam("details") @DefaultValue("true") details: Boolean):
StageData = withUI { ui =>
try {
- ui.store.stageAttempt(stageId, stageAttemptId, details = details)
+ ui.store.stageAttempt(stageId, stageAttemptId, details = details)._1
} catch {
case _: NoSuchElementException =>
// Change the message depending on whether there are any attempts for
the requested stage.
http://git-wip-us.apache.org/repos/asf/spark/blob/deb9588b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
--
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 7428bbe..0f74b07 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -105,7 +105,7 @@ private[ui] class StagePage(parent: StagesTab, store:
AppStatusStore) extends We
val stageAttemptId = parameterAttempt.toInt
val stageHeader = s"Details for Stage $stageId (Attempt $stageAttemptId)"
-val stageData = parent.store
+val (stageData, stageJobIds) = parent.store
.asOption(parent.store.stageAttempt(stageId, stageAttemptId, details =
false))
.getOrElse {
val content =
@@ -183,6 +183,15 @@ private[ui] class StagePage(parent: StagesTab, store:
AppStatusStore) extends We
{Utils.bytesToString(stageDat
spark git commit: [SPARK-25753][CORE] fix reading small files via BinaryFileRDD
Repository: spark
Updated Branches:
refs/heads/master fc64e83f9 -> 81a305dd0
[SPARK-25753][CORE] fix reading small files via BinaryFileRDD
## What changes were proposed in this pull request?
This is a follow up of #21601, `StreamFileInputFormat` and
`WholeTextFileInputFormat` have the same problem.
`Minimum split size pernode 5123456 cannot be larger than maximum split size
4194304
java.io.IOException: Minimum split size pernode 5123456 cannot be larger than
maximum split size 4194304
at
org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.getSplits(CombineFileInputFormat.java:
201)
at
org.apache.spark.rdd.BinaryFileRDD.getPartitions(BinaryFileRDD.scala:52)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:254)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:252)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2138)`
## How was this patch tested?
Added a unit test
Closes #22725 from 10110346/maxSplitSize_node_rack.
Authored-by: liuxian
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/81a305dd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/81a305dd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/81a305dd
Branch: refs/heads/master
Commit: 81a305dd0418f6e0136b4e38ffe91e0b76c8806e
Parents: fc64e83
Author: liuxian
Authored: Mon Oct 22 08:53:18 2018 -0500
Committer: Thomas Graves
Committed: Mon Oct 22 08:53:18 2018 -0500
--
.../org/apache/spark/input/PortableDataStream.scala| 12
core/src/test/scala/org/apache/spark/FileSuite.scala | 13 +
2 files changed, 25 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/81a305dd/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
index ab020aa..5b33c11 100644
--- a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
+++ b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
@@ -52,6 +52,18 @@ private[spark] abstract class StreamFileInputFormat[T]
val totalBytes = files.filterNot(_.isDirectory).map(_.getLen +
openCostInBytes).sum
val bytesPerCore = totalBytes / defaultParallelism
val maxSplitSize = Math.min(defaultMaxSplitBytes,
Math.max(openCostInBytes, bytesPerCore))
+
+// For small files we need to ensure the min split size per node & rack <=
maxSplitSize
+val jobConfig = context.getConfiguration
+val minSplitSizePerNode =
jobConfig.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERNODE, 0L)
+val minSplitSizePerRack =
jobConfig.getLong(CombineFileInputFormat.SPLIT_MINSIZE_PERRACK, 0L)
+
+if (maxSplitSize < minSplitSizePerNode) {
+ super.setMinSplitSizeNode(maxSplitSize)
+}
+if (maxSplitSize < minSplitSizePerRack) {
+ super.setMinSplitSizeRack(maxSplitSize)
+}
super.setMaxSplitSize(maxSplitSize)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/81a305dd/core/src/test/scala/org/apache/spark/FileSuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/FileSuite.scala
b/core/src/test/scala/org/apache/spark/FileSuite.scala
index 81b18c7..34efcdf 100644
--- a/core/src/test/scala/org/apache/spark/FileSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FileSuite.scala
@@ -320,6 +320,19 @@ class FileSuite extends SparkFunSuite with
LocalSparkContext {
}
}
+ test("minimum split size per node and per rack should be less than or equal
to maxSplitSize") {
+sc = new SparkContext("local", "test")
+val testOutput = Array[Byte](1, 2, 3, 4, 5)
+val outFile = writeBinaryData(testOutput, 1)
+sc.hadoopConfiguration.setLong(
+ "mapreduce.input.fileinputformat.split.minsize.per.node", 5123456)
+sc.hadoopConfiguration.setLong(
+ "mapreduce.input.fileinputformat.split.minsize.per.rack", 5123456)
+
+val (_, data) = sc.binaryFiles(outFile.getAbsolutePath).collect().head
+assert(data.toArray === testOutput)
+ }
+
test("fixed record length binary file as byte array") {
sc = new SparkContext("local", "test")
val testOutput = Array[Byte](1, 2, 3, 4, 5, 6)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-21501] Change CacheLoader to limit entries based on memory footprint
Repository: spark
Updated Branches:
refs/heads/master d6b30edd4 -> 1662e9311
[SPARK-21501] Change CacheLoader to limit entries based on memory footprint
Right now the spark shuffle service has a cache for index files. It is based on
a # of files cached (spark.shuffle.service.index.cache.entries). This can cause
issues if people have a lot of reducers because the size of each entry can
fluctuate based on the # of reducers.
We saw an issues with a job that had 17 reducers and it caused NM with
spark shuffle service to use 700-800MB or memory in NM by itself.
We should change this cache to be memory based and only allow a certain memory
size used. When I say memory based I mean the cache should have a limit of say
100MB.
https://issues.apache.org/jira/browse/SPARK-21501
Manual Testing with 17 reducers has been performed with cache loaded up to
max 100MB default limit, with each shuffle index file of size 1.3MB. Eviction
takes place as soon as the total cache size reaches the 100MB limit and the
objects will be ready for garbage collection there by avoiding NM to crash. No
notable difference in runtime has been observed.
Author: Sanket Chintapalli
Closes #18940 from redsanket/SPARK-21501.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1662e931
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1662e931
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1662e931
Branch: refs/heads/master
Commit: 1662e93119d68498942386906de309d35f4a135f
Parents: d6b30ed
Author: Sanket Chintapalli
Authored: Wed Aug 23 11:51:11 2017 -0500
Committer: Tom Graves
Committed: Wed Aug 23 11:51:11 2017 -0500
--
.../org/apache/spark/network/util/TransportConf.java | 4
.../network/shuffle/ExternalShuffleBlockResolver.java| 11 +--
.../spark/network/shuffle/ShuffleIndexInformation.java | 11 ++-
core/src/main/scala/org/apache/spark/SparkConf.scala | 4 +++-
docs/configuration.md| 6 +++---
5 files changed, 29 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1662e931/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
--
diff --git
a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index 88256b8..fa2ff42 100644
---
a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++
b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -67,6 +67,10 @@ public class TransportConf {
return conf.getInt(name, defaultValue);
}
+ public String get(String name, String defaultValue) {
+return conf.get(name, defaultValue);
+ }
+
private String getConfKey(String suffix) {
return "spark." + module + "." + suffix;
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1662e931/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
--
diff --git
a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
index d7ec0e2..e639989 100644
---
a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
+++
b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java
@@ -33,6 +33,7 @@ import com.google.common.base.Objects;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.cache.Weigher;
import com.google.common.collect.Maps;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBIterator;
@@ -104,7 +105,7 @@ public class ExternalShuffleBlockResolver {
Executor directoryCleaner) throws IOException {
this.conf = conf;
this.registeredExecutorFile = registeredExecutorFile;
-int indexCacheEntries =
conf.getInt("spark.shuffle.service.index.cache.entries", 1024);
+String indexCacheSize = conf.get("spark.shuffle.service.index.cache.size",
"100m");
CacheLoader indexCacheLoader =
new CacheLoader() {
public ShuffleIndexInformation load(File file) throws IOException {
@@ -112,7 +113,13 @@ public class ExternalShuffleBlockResolver {
}
};
shuffleIndexCache = CacheBuilder.newBuilder()
-
.maximumSize(indexCacheEntries).build(indexCac
spark git commit: [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server
Repository: spark Updated Branches: refs/heads/master 0456b4050 -> 24e6c187f [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server History Server Launch uses SparkClassCommandBuilder for launching the server. It is observed that SPARK_CLASSPATH has been removed and deprecated. For spark-submit this takes a different route and spark.driver.extraClasspath takes care of specifying additional jars in the classpath that were previously specified in the SPARK_CLASSPATH. Right now the only way specify the additional jars for launching daemons such as history server is using SPARK_DIST_CLASSPATH (https://spark.apache.org/docs/latest/hadoop-provided.html) but this I presume is a distribution classpath. It would be nice to have a similar config like spark.driver.extraClasspath for launching daemons similar to history server. Added new environment variable SPARK_DAEMON_CLASSPATH to set classpath for launching daemons. Tested and verified for History Server and Standalone Mode. ## How was this patch tested? Initially, history server start script would fail for the reason being that it could not find the required jars for launching the server in the java classpath. Same was true for running Master and Worker in standalone mode. By adding the environment variable SPARK_DAEMON_CLASSPATH to the java classpath, both the daemons(History Server, Standalone daemons) are starting up and running. Author: pgandhi Author: pgandhi999 Closes #19047 from pgandhi999/master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/24e6c187 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/24e6c187 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/24e6c187 Branch: refs/heads/master Commit: 24e6c187fbaa6874eedbdda6b3b5dc6ff9e1de36 Parents: 0456b40 Author: pgandhi Authored: Mon Aug 28 08:51:22 2017 -0500 Committer: Tom Graves Committed: Mon Aug 28 08:51:22 2017 -0500 -- conf/spark-env.sh.template | 1 + docs/monitoring.md | 4 docs/running-on-mesos.md| 2 ++ docs/spark-standalone.md| 4 .../org/apache/spark/launcher/SparkClassCommandBuilder.java | 5 + 5 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/24e6c187/conf/spark-env.sh.template -- diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index 1663019..f8c895f 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -52,6 +52,7 @@ # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") +# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers # Generic options for the daemons used in the standalone deploy mode http://git-wip-us.apache.org/repos/asf/spark/blob/24e6c187/docs/monitoring.md -- diff --git a/docs/monitoring.md b/docs/monitoring.md index 3e577c5..d22cd94 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -62,6 +62,10 @@ The history server can be configured as follows: JVM options for the history server (default: none). +SPARK_DAEMON_CLASSPATH +Classpath for the history server (default: none). + + SPARK_PUBLIC_DNS The public address for the history server. If this is not set, links to application history http://git-wip-us.apache.org/repos/asf/spark/blob/24e6c187/docs/running-on-mesos.md -- diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 0e5a20c..c12b858 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -160,6 +160,8 @@ If you like to run the `MesosClusterDispatcher` with Marathon, you need to run t The `MesosClusterDispatcher` also supports writing recovery state into Zookeeper. This will allow the `MesosClusterDispatcher` to be able to recover all submitted and running containers on relaunch. In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spark-env by configuring `spark.deploy.recoveryMode` and related spark.deploy.zookeeper.* configurations. For more information about these configurations please refer to the configurations [doc](configurations.html#deploy). +You can also specify any
spark git commit: [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server
Repository: spark Updated Branches: refs/heads/branch-2.2 0d4ef2f69 -> 59bb7ebfb [SPARK-21798] No config to replace deprecated SPARK_CLASSPATH config for launching daemons like History Server History Server Launch uses SparkClassCommandBuilder for launching the server. It is observed that SPARK_CLASSPATH has been removed and deprecated. For spark-submit this takes a different route and spark.driver.extraClasspath takes care of specifying additional jars in the classpath that were previously specified in the SPARK_CLASSPATH. Right now the only way specify the additional jars for launching daemons such as history server is using SPARK_DIST_CLASSPATH (https://spark.apache.org/docs/latest/hadoop-provided.html) but this I presume is a distribution classpath. It would be nice to have a similar config like spark.driver.extraClasspath for launching daemons similar to history server. Added new environment variable SPARK_DAEMON_CLASSPATH to set classpath for launching daemons. Tested and verified for History Server and Standalone Mode. ## How was this patch tested? Initially, history server start script would fail for the reason being that it could not find the required jars for launching the server in the java classpath. Same was true for running Master and Worker in standalone mode. By adding the environment variable SPARK_DAEMON_CLASSPATH to the java classpath, both the daemons(History Server, Standalone daemons) are starting up and running. Author: pgandhi Author: pgandhi999 Closes #19047 from pgandhi999/master. (cherry picked from commit 24e6c187fbaa6874eedbdda6b3b5dc6ff9e1de36) Signed-off-by: Tom Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/59bb7ebf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/59bb7ebf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/59bb7ebf Branch: refs/heads/branch-2.2 Commit: 59bb7ebfb83c292cea853d6cd6fdf9748baa6ce2 Parents: 0d4ef2f Author: pgandhi Authored: Mon Aug 28 08:51:22 2017 -0500 Committer: Tom Graves Committed: Mon Aug 28 08:51:49 2017 -0500 -- conf/spark-env.sh.template | 1 + docs/monitoring.md | 4 docs/running-on-mesos.md| 2 ++ docs/spark-standalone.md| 4 .../org/apache/spark/launcher/SparkClassCommandBuilder.java | 5 + 5 files changed, 16 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/59bb7ebf/conf/spark-env.sh.template -- diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index b7c985a..0f9150b 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -51,6 +51,7 @@ # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") +# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers # Generic options for the daemons used in the standalone deploy mode http://git-wip-us.apache.org/repos/asf/spark/blob/59bb7ebf/docs/monitoring.md -- diff --git a/docs/monitoring.md b/docs/monitoring.md index 3e577c5..d22cd94 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -62,6 +62,10 @@ The history server can be configured as follows: JVM options for the history server (default: none). +SPARK_DAEMON_CLASSPATH +Classpath for the history server (default: none). + + SPARK_PUBLIC_DNS The public address for the history server. If this is not set, links to application history http://git-wip-us.apache.org/repos/asf/spark/blob/59bb7ebf/docs/running-on-mesos.md -- diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 847a659..6b69bfc 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -158,6 +158,8 @@ If you like to run the `MesosClusterDispatcher` with Marathon, you need to run t The `MesosClusterDispatcher` also supports writing recovery state into Zookeeper. This will allow the `MesosClusterDispatcher` to be able to recover all submitted and running containers on relaunch. In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spark-env by configuring `spark.deploy.recoveryMode` and related spark.deploy.zookeeper.* configurations. For more information about these configur
spark git commit: [SPARK-21890] Credentials not being passed to add the tokens
Repository: spark
Updated Branches:
refs/heads/master eea2b877c -> b9ab791a9
[SPARK-21890] Credentials not being passed to add the tokens
I observed this while running a oozie job trying to connect to hbase via spark.
It look like the creds are not being passed in
thehttps://github.com/apache/spark/blob/branch-2.2/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/HadoopFSCredentialProvider.scala#L53
for 2.2 release.
More Info as to why it fails on secure grid:
Oozie client gets the necessary tokens the application needs before launching.
It passes those tokens along to the oozie launcher job (MR job) which will then
actually call the Spark client to launch the spark app and pass the tokens
along.
The oozie launcher job cannot get anymore tokens because all it has is tokens (
you can't get tokens with tokens, you need tgt or keytab).
The error here is because the launcher job runs the Spark Client to submit the
spark job but the spark client doesn't see that it already has the hdfs tokens
so it tries to get more, which ends with the exception.
There was a change with SPARK-19021 to generalize the hdfs credentials provider
that changed it so we don't pass the existing credentials into the call to get
tokens so it doesn't realize it already has the necessary tokens.
https://issues.apache.org/jira/browse/SPARK-21890
Modified to pass creds to get delegation tokens
Author: Sanket Chintapalli
Closes #19140 from redsanket/SPARK-21890-master.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b9ab791a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b9ab791a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b9ab791a
Branch: refs/heads/master
Commit: b9ab791a9efb0dc165ba283c91acf831fa6be5d8
Parents: eea2b87
Author: Sanket Chintapalli
Authored: Thu Sep 7 11:25:24 2017 -0500
Committer: Tom Graves
Committed: Thu Sep 7 11:25:24 2017 -0500
--
.../security/HBaseDelegationTokenProvider.scala | 2 +
.../security/HadoopDelegationTokenManager.scala | 2 +-
.../HadoopDelegationTokenProvider.scala | 3 ++
.../HadoopFSDelegationTokenProvider.scala | 50 ++--
.../security/HiveDelegationTokenProvider.scala | 2 +
.../HadoopDelegationTokenManagerSuite.scala | 4 +-
6 files changed, 35 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b9ab791a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
index 35621da..78b0e6b 100644
---
a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.security.Credentials
import org.apache.hadoop.security.token.{Token, TokenIdentifier}
+import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
@@ -34,6 +35,7 @@ private[security] class HBaseDelegationTokenProvider
override def obtainDelegationTokens(
hadoopConf: Configuration,
+ sparkConf: SparkConf,
creds: Credentials): Option[Long] = {
try {
val mirror = universe.runtimeMirror(Utils.getContextOrSparkClassLoader)
http://git-wip-us.apache.org/repos/asf/spark/blob/b9ab791a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
index c317c4f..c134b7e 100644
---
a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
@@ -116,7 +116,7 @@ private[spark] class HadoopDelegationTokenManager(
creds: Credentials): Long = {
delegationTokenProviders.values.flatMap { provider =>
if (provider.delegationTokensRequired(hadoopConf)) {
-provider.obtainDelegationTokens(hadoopConf, creds)
+provider.obtainDelegationTokens(hadoopConf, sparkConf, creds)
} else {
logDebug(s"Service ${provider.serviceName} does not require a token." +
s" Check your configuration to see if security is disabled or not.")
http://git-wip-us.apache.org/repos/asf/spark/blob/b9
spark git commit: [SPARK-22683][CORE] Add a executorAllocationRatio parameter to throttle the parallelism of the dynamic allocation
Repository: spark
Updated Branches:
refs/heads/master 4926a7c2f -> 55c4ca88a
[SPARK-22683][CORE] Add a executorAllocationRatio parameter to throttle the
parallelism of the dynamic allocation
## What changes were proposed in this pull request?
By default, the dynamic allocation will request enough executors to maximize the
parallelism according to the number of tasks to process. While this minimizes
the
latency of the job, with small tasks this setting can waste a lot of resources
due to
executor allocation overhead, as some executor might not even do any work.
This setting allows to set a ratio that will be used to reduce the number of
target executors w.r.t. full parallelism.
The number of executors computed with this setting is still fenced by
`spark.dynamicAllocation.maxExecutors` and
`spark.dynamicAllocation.minExecutors`
## How was this patch tested?
Units tests and runs on various actual workloads on a Yarn Cluster
Author: Julien Cuquemelle
Closes #19881 from jcuquemelle/AddTaskPerExecutorSlot.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/55c4ca88
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/55c4ca88
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/55c4ca88
Branch: refs/heads/master
Commit: 55c4ca88a3b093ee197a8689631be8d1fac1f10f
Parents: 4926a7c
Author: Julien Cuquemelle
Authored: Tue Apr 24 10:56:55 2018 -0500
Committer: Thomas Graves
Committed: Tue Apr 24 10:56:55 2018 -0500
--
.../spark/ExecutorAllocationManager.scala | 24 +++---
.../apache/spark/internal/config/package.scala | 4 +++
.../spark/ExecutorAllocationManagerSuite.scala | 33
docs/configuration.md | 18 +++
4 files changed, 74 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/55c4ca88/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index 189d913..aa363ee 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -26,7 +26,7 @@ import scala.util.control.{ControlThrowable, NonFatal}
import com.codahale.metrics.{Gauge, MetricRegistry}
import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config.{DYN_ALLOCATION_MAX_EXECUTORS,
DYN_ALLOCATION_MIN_EXECUTORS}
+import org.apache.spark.internal.config._
import org.apache.spark.metrics.source.Source
import org.apache.spark.scheduler._
import org.apache.spark.storage.BlockManagerMaster
@@ -69,6 +69,10 @@ import org.apache.spark.util.{Clock, SystemClock,
ThreadUtils, Utils}
* spark.dynamicAllocation.maxExecutors - Upper bound on the number of
executors
* spark.dynamicAllocation.initialExecutors - Number of executors to start
with
*
+ * spark.dynamicAllocation.executorAllocationRatio -
+ * This is used to reduce the parallelism of the dynamic allocation that
can waste
+ * resources when tasks are small
+ *
* spark.dynamicAllocation.schedulerBacklogTimeout (M) -
* If there are backlogged tasks for this duration, add new executors
*
@@ -116,9 +120,12 @@ private[spark] class ExecutorAllocationManager(
// TODO: The default value of 1 for spark.executor.cores works right now
because dynamic
// allocation is only supported for YARN and the default number of cores per
executor in YARN is
// 1, but it might need to be attained differently for different cluster
managers
- private val tasksPerExecutor =
+ private val tasksPerExecutorForFullParallelism =
conf.getInt("spark.executor.cores", 1) / conf.getInt("spark.task.cpus", 1)
+ private val executorAllocationRatio =
+conf.get(DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO)
+
validateSettings()
// Number of executors to add in the next round
@@ -209,8 +216,13 @@ private[spark] class ExecutorAllocationManager(
throw new SparkException("Dynamic allocation of executors requires the
external " +
"shuffle service. You may enable this through
spark.shuffle.service.enabled.")
}
-if (tasksPerExecutor == 0) {
- throw new SparkException("spark.executor.cores must not be less than
spark.task.cpus.")
+if (tasksPerExecutorForFullParallelism == 0) {
+ throw new SparkException("spark.executor.cores must not be <
spark.task.cpus.")
+}
+
+if (executorAllocationRatio > 1.0 || executorAllocationRatio <= 0.0) {
+ throw new SparkException(
+"spark.dynamicAllocation.executorAllocationRatio must be > 0 and <=
1.0")
}
}
@@ -2
git commit: SPARK-1291: Link the spark UI to RM ui in yarn-client mode
Repository: spark
Updated Branches:
refs/heads/master 9dd635eb5 -> 72ea56da8
SPARK-1291: Link the spark UI to RM ui in yarn-client mode
Author: witgo
Closes #1112 from witgo/SPARK-1291 and squashes the following commits:
6022bcd [witgo] review commit
1fbb925 [witgo] add addAmIpFilter to yarn alpha
210299c [witgo] review commit
1b92a07 [witgo] review commit
6896586 [witgo] Add comments to addWebUIFilter
3e9630b [witgo] review commit
142ee29 [witgo] review commit
1fe7710 [witgo] Link the spark UI to RM ui in yarn-client mode
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/72ea56da
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/72ea56da
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/72ea56da
Branch: refs/heads/master
Commit: 72ea56da8e383c61c6f18eeefef03b9af00f5158
Parents: 9dd635e
Author: witgo
Authored: Tue Jul 15 13:52:56 2014 -0500
Committer: Thomas Graves
Committed: Tue Jul 15 13:52:56 2014 -0500
--
.../cluster/CoarseGrainedClusterMessage.scala | 3 +++
.../cluster/CoarseGrainedSchedulerBackend.scala | 18 +++
.../scala/org/apache/spark/ui/UIUtils.scala | 11 +-
.../spark/deploy/yarn/ExecutorLauncher.scala| 22 ---
.../cluster/YarnClientSchedulerBackend.scala| 1 +
.../spark/deploy/yarn/ExecutorLauncher.scala| 23 +---
6 files changed, 71 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/72ea56da/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
index 318e165..6abf6d9 100644
---
a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
+++
b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala
@@ -66,4 +66,7 @@ private[spark] object CoarseGrainedClusterMessages {
case class RemoveExecutor(executorId: String, reason: String) extends
CoarseGrainedClusterMessage
+ case class AddWebUIFilter(filterName:String, filterParams: String, proxyBase
:String)
+extends CoarseGrainedClusterMessage
+
}
http://git-wip-us.apache.org/repos/asf/spark/blob/72ea56da/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 0f5545e..9f085ee 100644
---
a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++
b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -31,6 +31,7 @@ import org.apache.spark.{SparkEnv, Logging, SparkException,
TaskState}
import org.apache.spark.scheduler.{SchedulerBackend, SlaveLost,
TaskDescription, TaskSchedulerImpl, WorkerOffer}
import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._
import org.apache.spark.util.{SerializableBuffer, AkkaUtils, Utils}
+import org.apache.spark.ui.JettyUtils
/**
* A scheduler backend that waits for coarse grained executors to connect to
it through Akka.
@@ -136,6 +137,9 @@ class CoarseGrainedSchedulerBackend(scheduler:
TaskSchedulerImpl, actorSystem: A
removeExecutor(executorId, reason)
sender ! true
+ case AddWebUIFilter(filterName, filterParams, proxyBase) =>
+addWebUIFilter(filterName, filterParams, proxyBase)
+sender ! true
case DisassociatedEvent(_, address, _) =>
addressToExecutorId.get(address).foreach(removeExecutor(_,
"remote Akka client disassociated"))
@@ -276,6 +280,20 @@ class CoarseGrainedSchedulerBackend(scheduler:
TaskSchedulerImpl, actorSystem: A
}
false
}
+
+ // Add filters to the SparkUI
+ def addWebUIFilter(filterName: String, filterParams: String, proxyBase:
String) {
+if (proxyBase != null && proxyBase.nonEmpty) {
+ System.setProperty("spark.ui.proxyBase", proxyBase)
+}
+
+if (Seq(filterName, filterParams).forall(t => t != null && t.nonEmpty)) {
+ logInfo(s"Add WebUI Filter. $filterName, $filterParams, $proxyBase")
+ conf.set("spark.ui.filters", filterName)
+ conf.set(s"spark.$filterName.params", filterParams)
+ JettyUtils.addFilters(scheduler.sc.ui.getHandlers, conf)
+}
+ }
}
private[spark] object CoarseGrainedSchedulerBackend {
http://git-wip-us.apache.org/repos/asf/spark/blob/
git commit: SPARK-1707. Remove unnecessary 3 second sleep in YarnClusterScheduler
Repository: spark
Updated Branches:
refs/heads/master cd273a238 -> f89cf65d7
SPARK-1707. Remove unnecessary 3 second sleep in YarnClusterScheduler
Author: Sandy Ryza
Closes #634 from sryza/sandy-spark-1707 and squashes the following commits:
2f6e358 [Sandy Ryza] Default min registered executors ratio to .8 for YARN
354c630 [Sandy Ryza] Remove outdated comments
c744ef3 [Sandy Ryza] Take out waitForInitialAllocations
2a4329b [Sandy Ryza] SPARK-1707. Remove unnecessary 3 second sleep in
YarnClusterScheduler
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f89cf65d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f89cf65d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f89cf65d
Branch: refs/heads/master
Commit: f89cf65d7aced0bb387c05586f9f51cb29865022
Parents: cd273a2
Author: Sandy Ryza
Authored: Mon Jul 21 13:15:46 2014 -0500
Committer: Thomas Graves
Committed: Mon Jul 21 13:15:46 2014 -0500
--
.../spark/deploy/yarn/ApplicationMaster.scala | 39 --
.../cluster/YarnClientClusterScheduler.scala| 10 -
.../cluster/YarnClientSchedulerBackend.scala| 5 +++
.../cluster/YarnClusterScheduler.scala | 8 +---
.../cluster/YarnClusterSchedulerBackend.scala | 5 +++
.../spark/deploy/yarn/ApplicationMaster.scala | 43
6 files changed, 11 insertions(+), 99 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f89cf65d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 062f946..3ec3648 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -255,10 +255,6 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
sparkContext.getConf)
}
}
-} finally {
- // in case of exceptions, etc - ensure that count is atleast
ALLOCATOR_LOOP_WAIT_COUNT :
- // so that the loop (in ApplicationMaster.sparkContextInitialized) breaks
-
ApplicationMaster.incrementAllocatorLoop(ApplicationMaster.ALLOCATOR_LOOP_WAIT_COUNT)
}
}
@@ -277,13 +273,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
}
yarnAllocator.allocateContainers(
math.max(args.numExecutors - yarnAllocator.getNumExecutorsRunning,
0))
-ApplicationMaster.incrementAllocatorLoop(1)
Thread.sleep(ApplicationMaster.ALLOCATE_HEARTBEAT_INTERVAL)
}
-} finally {
- // In case of exceptions, etc - ensure that count is at least
ALLOCATOR_LOOP_WAIT_COUNT,
- // so that the loop in ApplicationMaster#sparkContextInitialized()
breaks.
-
ApplicationMaster.incrementAllocatorLoop(ApplicationMaster.ALLOCATOR_LOOP_WAIT_COUNT)
}
logInfo("All executors have launched.")
@@ -411,24 +402,10 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
}
object ApplicationMaster extends Logging {
- // Number of times to wait for the allocator loop to complete.
- // Each loop iteration waits for 100ms, so maximum of 3 seconds.
- // This is to ensure that we have reasonable number of containers before we
start
// TODO: Currently, task to container is computed once (TaskSetManager) -
which need not be
// optimal as more containers are available. Might need to handle this
better.
- private val ALLOCATOR_LOOP_WAIT_COUNT = 30
private val ALLOCATE_HEARTBEAT_INTERVAL = 100
- def incrementAllocatorLoop(by: Int) {
-val count = yarnAllocatorLoop.getAndAdd(by)
-if (count >= ALLOCATOR_LOOP_WAIT_COUNT) {
- yarnAllocatorLoop.synchronized {
-// to wake threads off wait ...
-yarnAllocatorLoop.notifyAll()
- }
-}
- }
-
private val applicationMasters = new
CopyOnWriteArrayList[ApplicationMaster]()
def register(master: ApplicationMaster) {
@@ -437,7 +414,6 @@ object ApplicationMaster extends Logging {
val sparkContextRef: AtomicReference[SparkContext] =
new AtomicReference[SparkContext](null /* initialValue */)
- val yarnAllocatorLoop: AtomicInteger = new AtomicInteger(0)
def sparkContextInitialized(sc: SparkContext): Boolean = {
var modified = false
@@ -472,21 +448,6 @@ object ApplicationMaster extends Logging {
modified
}
-
- /**
- * Returns when we've either
- * 1) received all the requested executors,
- * 2) waited ALLOCATOR_LOOP_WAIT_COUNT * ALLOCATE_HEARTBEAT_INTERVAL ms,
- * 3) hi
git commit: [YARN][SPARK-2606]:In some cases, the spark UI pages display incorrect
Repository: spark
Updated Branches:
refs/heads/master 5f7b99168 -> ddadf1b00
[YARN][SPARK-2606]:In some cases,the spark UI pages display incorrect
The issue is caused by #1112 .
Author: GuoQiang Li
Closes #1501 from witgo/webui_style and squashes the following commits:
4b34998 [GuoQiang Li] In some cases, pages display incorrect in WebUI
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ddadf1b0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ddadf1b0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ddadf1b0
Branch: refs/heads/master
Commit: ddadf1b00470b9d7bf7386dacf198d41407a0a2b
Parents: 5f7b991
Author: GuoQiang Li
Authored: Tue Jul 22 20:34:40 2014 -0500
Committer: Thomas Graves
Committed: Tue Jul 22 20:34:40 2014 -0500
--
core/src/main/scala/org/apache/spark/ui/UIUtils.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ddadf1b0/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
--
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index e07aa2e..715cc2f 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -149,7 +149,7 @@ private[spark] object UIUtils extends Logging {
def prependBaseUri(basePath: String = "", resource: String = "") = uiRoot +
basePath + resource
- val commonHeaderNodes = {
+ def commonHeaderNodes = {
git commit: [YARN] SPARK-2577: File upload to viewfs is broken due to mount point re...
Repository: spark
Updated Branches:
refs/heads/master ddadf1b00 -> 02e457294
[YARN] SPARK-2577: File upload to viewfs is broken due to mount point re...
Opting to the option 2 defined in SPARK-2577, i.e., retrieve and pass the
correct file system object to addResource.
Author: Gera Shegalov
Closes #1483 from gerashegalov/master and squashes the following commits:
90c9087 [Gera Shegalov] [YARN] SPARK-2577: File upload to viewfs is broken due
to mount point resolution
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/02e45729
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/02e45729
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/02e45729
Branch: refs/heads/master
Commit: 02e45729472a22a31629cdd17dc3836ba6810189
Parents: ddadf1b
Author: Gera Shegalov
Authored: Tue Jul 22 21:05:12 2014 -0500
Committer: Thomas Graves
Committed: Tue Jul 22 21:05:12 2014 -0500
--
.../src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/02e45729/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 556f493..a1298e8 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -232,7 +232,8 @@ trait ClientBase extends Logging {
if (!ClientBase.LOCAL_SCHEME.equals(localURI.getScheme())) {
val setPermissions = if (destName.equals(ClientBase.APP_JAR)) true
else false
val destPath = copyRemoteFile(dst, qualifyForLocal(localURI),
replication, setPermissions)
- distCacheMgr.addResource(fs, conf, destPath, localResources,
LocalResourceType.FILE,
+ val destFs = FileSystem.get(destPath.toUri(), conf)
+ distCacheMgr.addResource(destFs, conf, destPath, localResources,
LocalResourceType.FILE,
destName, statCache)
} else if (confKey != null) {
sparkConf.set(confKey, localPath)
git commit: SPARK-2150: Provide direct link to finished application UI in yarn resou...
Repository: spark
Updated Branches:
refs/heads/master 42dfab7d3 -> 46e224aaa
SPARK-2150: Provide direct link to finished application UI in yarn resou...
...rce manager UI
Use the event logger directory to provide a direct link to finished
application UI in yarn resourcemanager UI.
Author: Rahul Singhal
Closes #1094 from rahulsinghaliitd/SPARK-2150 and squashes the following
commits:
95f230c [Rahul Singhal] SPARK-2150: Provide direct link to finished application
UI in yarn resource manager UI
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/46e224aa
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/46e224aa
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/46e224aa
Branch: refs/heads/master
Commit: 46e224aaa26df4b232c5176e98472a902862b76c
Parents: 42dfab7
Author: Rahul Singhal
Authored: Thu Jul 24 09:31:04 2014 -0500
Committer: Thomas Graves
Committed: Thu Jul 24 09:31:04 2014 -0500
--
.../spark/deploy/history/FsHistoryProvider.scala| 3 ++-
.../apache/spark/deploy/history/HistoryPage.scala | 2 +-
.../apache/spark/deploy/history/HistoryServer.scala | 4 +++-
.../org/apache/spark/deploy/master/Master.scala | 11 +++
.../spark/scheduler/EventLoggingListener.scala | 7 +++
.../spark/deploy/yarn/ApplicationMaster.scala | 4 +++-
.../apache/spark/deploy/yarn/ExecutorLauncher.scala | 2 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 16
.../cluster/YarnClientSchedulerBackend.scala| 3 ++-
.../spark/deploy/yarn/ApplicationMaster.scala | 5 +++--
.../apache/spark/deploy/yarn/ExecutorLauncher.scala | 2 +-
11 files changed, 46 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/46e224aa/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index a8c9ac0..01e7065 100644
---
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -169,7 +169,8 @@ private[history] class FsHistoryProvider(conf: SparkConf)
extends ApplicationHis
val ui: SparkUI = if (renderUI) {
val conf = this.conf.clone()
val appSecManager = new SecurityManager(conf)
-new SparkUI(conf, appSecManager, replayBus, appId, "/history/" + appId)
+new SparkUI(conf, appSecManager, replayBus, appId,
+ HistoryServer.UI_PATH_PREFIX + s"/$appId")
// Do not call ui.bind() to avoid creating a new server for each
application
} else {
null
http://git-wip-us.apache.org/repos/asf/spark/blob/46e224aa/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
index a958c83..d7a3e3f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -75,7 +75,7 @@ private[spark] class HistoryPage(parent: HistoryServer)
extends WebUIPage("") {
"Last Updated")
private def appRow(info: ApplicationHistoryInfo): Seq[Node] = {
-val uiAddress = "/history/" + info.id
+val uiAddress = HistoryServer.UI_PATH_PREFIX + s"/${info.id}"
val startTime = UIUtils.formatDate(info.startTime)
val endTime = UIUtils.formatDate(info.endTime)
val duration = UIUtils.formatDuration(info.endTime - info.startTime)
http://git-wip-us.apache.org/repos/asf/spark/blob/46e224aa/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index 56b38dd..cacb9da 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -114,7 +114,7 @@ class HistoryServer(
attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
val contextHandler = new ServletContextHandler
-contextHandler.setContextPath("/history")
+contextHandler.setContextPath(HistoryServer.UI_PATH_PREFIX)
contextHandler.addServlet(new ServletHolder(loaderServlet), "/*")
attachHandler(contextHandler)
}
@@ -172,6 +17
git commit: [SPARK-2037]: yarn client mode doesn't support spark.yarn.max.executor.failures
Repository: spark
Updated Branches:
refs/heads/master c960b5051 -> 323a83c52
[SPARK-2037]: yarn client mode doesn't support spark.yarn.max.executor.failures
Author: GuoQiang Li
Closes #1180 from witgo/SPARK-2037 and squashes the following commits:
3d52411 [GuoQiang Li] review commit
7058f4d [GuoQiang Li] Correctly stop SparkContext
6d0561f [GuoQiang Li] Fix: yarn client mode doesn't support
spark.yarn.max.executor.failures
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/323a83c5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/323a83c5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/323a83c5
Branch: refs/heads/master
Commit: 323a83c5235f9289cd9526491d62365df96a429b
Parents: c960b50
Author: GuoQiang Li
Authored: Thu Jul 24 14:46:10 2014 -0500
Committer: Thomas Graves
Committed: Thu Jul 24 14:46:10 2014 -0500
--
.../spark/deploy/yarn/ExecutorLauncher.scala| 80 +---
.../cluster/YarnClientSchedulerBackend.scala| 28 +++
.../spark/deploy/yarn/ExecutorLauncher.scala| 45 ---
3 files changed, 115 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/323a83c5/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
index d232c18..184e2ad 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
@@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
import akka.actor._
import akka.remote._
-import akka.actor.Terminated
import org.apache.spark.{Logging, SecurityManager, SparkConf}
import org.apache.spark.util.{Utils, AkkaUtils}
import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
@@ -57,10 +56,17 @@ class ExecutorLauncher(args: ApplicationMasterArguments,
conf: Configuration, sp
private val yarnConf: YarnConfiguration = new YarnConfiguration(conf)
private var yarnAllocator: YarnAllocationHandler = _
- private var driverClosed:Boolean = false
+
+ private var driverClosed: Boolean = false
+ private var isFinished: Boolean = false
+ private var registered: Boolean = false
+
+ // Default to numExecutors * 2, with minimum of 3
+ private val maxNumExecutorFailures =
sparkConf.getInt("spark.yarn.max.executor.failures",
+sparkConf.getInt("spark.yarn.max.worker.failures",
math.max(args.numExecutors * 2, 3)))
val securityManager = new SecurityManager(sparkConf)
- val actorSystem : ActorSystem = AkkaUtils.createActorSystem("sparkYarnAM",
Utils.localHostName, 0,
+ val actorSystem: ActorSystem = AkkaUtils.createActorSystem("sparkYarnAM",
Utils.localHostName, 0,
conf = sparkConf, securityManager = securityManager)._1
var actor: ActorRef = _
@@ -97,23 +103,26 @@ class ExecutorLauncher(args: ApplicationMasterArguments,
conf: Configuration, sp
appAttemptId = getApplicationAttemptId()
resourceManager = registerWithResourceManager()
-val appMasterResponse: RegisterApplicationMasterResponse =
registerApplicationMaster()
-
-// Compute number of threads for akka
-val minimumMemory =
appMasterResponse.getMinimumResourceCapability().getMemory()
-
-if (minimumMemory > 0) {
- val mem = args.executorMemory +
sparkConf.getInt("spark.yarn.executor.memoryOverhead",
-YarnAllocationHandler.MEMORY_OVERHEAD)
- val numCore = (mem / minimumMemory) + (if (0 != (mem % minimumMemory))
1 else 0)
-
- if (numCore > 0) {
-// do not override - hits
https://issues.apache.org/jira/browse/HADOOP-8406
-// TODO: Uncomment when hadoop is on a version which has this fixed.
-// args.workerCores = numCore
+synchronized {
+ if (!isFinished) {
+val appMasterResponse: RegisterApplicationMasterResponse =
registerApplicationMaster()
+// Compute number of threads for akka
+val minimumMemory =
appMasterResponse.getMinimumResourceCapability().getMemory()
+
+if (minimumMemory > 0) {
+ val mem = args.executorMemory +
sparkConf.getInt("spark.yarn.executor.memoryOverhead",
+YarnAllocationHandler.MEMORY_OVERHEAD)
+ val numCore = (mem / minimumMemory) + (if (0 != (mem %
minimumMemory)) 1 else 0)
+
+ if (numCore > 0) {
+// do not override - hits
https://issues.apache.org/jira/browse/HADOOP-8406
+// TODO: Uncomment when hadoop is on a version which has this
fixed.
+// args
git commit: SPARK-1528 - spark on yarn, add support for accessing remote HDFS
Repository: spark
Updated Branches:
refs/heads/master e87075df9 -> 2c0f705e2
SPARK-1528 - spark on yarn, add support for accessing remote HDFS
Add a config (spark.yarn.access.namenodes) to allow applications running on
yarn to access other secure HDFS cluster. User just specifies the namenodes of
the other clusters and we get Tokens for those and ship them with the spark
application.
Author: Thomas Graves
Closes #1159 from tgravescs/spark-1528 and squashes the following commits:
ddbcd16 [Thomas Graves] review comments
0ac8501 [Thomas Graves] SPARK-1528 - add support for accessing remote HDFS
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2c0f705e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2c0f705e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2c0f705e
Branch: refs/heads/master
Commit: 2c0f705e26ca3dfc43a1e9a0722c0e57f67c970a
Parents: e87075d
Author: Thomas Graves
Authored: Tue Aug 5 12:48:26 2014 -0500
Committer: Thomas Graves
Committed: Tue Aug 5 12:48:26 2014 -0500
--
docs/running-on-yarn.md | 7 +++
.../apache/spark/deploy/yarn/ClientBase.scala | 56 ++--
.../spark/deploy/yarn/ClientBaseSuite.scala | 55 ++-
3 files changed, 101 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2c0f705e/docs/running-on-yarn.md
--
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 0362f5a..573930d 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -106,6 +106,13 @@ Most of the configs are the same for Spark on YARN as for
other deployment modes
set this configuration to "hdfs:///some/path".
+
+ spark.yarn.access.namenodes
+ (none)
+
+A list of secure HDFS namenodes your Spark application is going to access.
For example,
`spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The
Spark application must have acess to the namenodes listed and Kerberos must be
properly configured to be able to access them (either in the same realm or in a
trusted realm). Spark acquires security tokens for each of the namenodes so
that the Spark application can access those remote HDFS clusters.
+
+
# Launching Spark on YARN
http://git-wip-us.apache.org/repos/asf/spark/blob/2c0f705e/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index b7e8636..ed8f56a 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs._
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.mapred.Master
import org.apache.hadoop.mapreduce.MRJobConfig
-import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.util.StringUtils
import org.apache.hadoop.yarn.api._
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
@@ -191,23 +191,11 @@ trait ClientBase extends Logging {
// Upload Spark and the application JAR to the remote file system if
necessary. Add them as
// local resources to the application master.
val fs = FileSystem.get(conf)
-
-val delegTokenRenewer = Master.getMasterPrincipal(conf)
-if (UserGroupInformation.isSecurityEnabled()) {
- if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
-val errorMessage = "Can't get Master Kerberos principal for use as
renewer"
-logError(errorMessage)
-throw new SparkException(errorMessage)
- }
-}
val dst = new Path(fs.getHomeDirectory(), appStagingDir)
-val replication = sparkConf.getInt("spark.yarn.submit.file.replication",
3).toShort
-
-if (UserGroupInformation.isSecurityEnabled()) {
- val dstFs = dst.getFileSystem(conf)
- dstFs.addDelegationTokens(delegTokenRenewer, credentials)
-}
+val nns = ClientBase.getNameNodesToAccess(sparkConf) + dst
+ClientBase.obtainTokensForNamenodes(nns, conf, credentials)
+val replication = sparkConf.getInt("spark.yarn.submit.file.replication",
3).toShort
val localResources = HashMap[String, LocalResource]()
FileSystem.mkdirs(fs, dst, new FsPermission(STAGING_DIR_PERMISSION))
@@ -614,4 +602,40 @@ object ClientBase extends Logging {
YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, path,
File.pathS
git commit: SPARK-1528 - spark on yarn, add support for accessing remote HDFS
Repository: spark
Updated Branches:
refs/heads/branch-1.1 b92a45058 -> 6c0c65fc8
SPARK-1528 - spark on yarn, add support for accessing remote HDFS
Add a config (spark.yarn.access.namenodes) to allow applications running on
yarn to access other secure HDFS cluster. User just specifies the namenodes of
the other clusters and we get Tokens for those and ship them with the spark
application.
Author: Thomas Graves
Closes #1159 from tgravescs/spark-1528 and squashes the following commits:
ddbcd16 [Thomas Graves] review comments
0ac8501 [Thomas Graves] SPARK-1528 - add support for accessing remote HDFS
(cherry picked from commit 2c0f705e26ca3dfc43a1e9a0722c0e57f67c970a)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c0c65fc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c0c65fc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c0c65fc
Branch: refs/heads/branch-1.1
Commit: 6c0c65fc85677ab2cae819a546ea50ed660994c3
Parents: b92a450
Author: Thomas Graves
Authored: Tue Aug 5 12:48:26 2014 -0500
Committer: Thomas Graves
Committed: Tue Aug 5 12:48:51 2014 -0500
--
docs/running-on-yarn.md | 7 +++
.../apache/spark/deploy/yarn/ClientBase.scala | 56 ++--
.../spark/deploy/yarn/ClientBaseSuite.scala | 55 ++-
3 files changed, 101 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6c0c65fc/docs/running-on-yarn.md
--
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 0362f5a..573930d 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -106,6 +106,13 @@ Most of the configs are the same for Spark on YARN as for
other deployment modes
set this configuration to "hdfs:///some/path".
+
+ spark.yarn.access.namenodes
+ (none)
+
+A list of secure HDFS namenodes your Spark application is going to access.
For example,
`spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The
Spark application must have acess to the namenodes listed and Kerberos must be
properly configured to be able to access them (either in the same realm or in a
trusted realm). Spark acquires security tokens for each of the namenodes so
that the Spark application can access those remote HDFS clusters.
+
+
# Launching Spark on YARN
http://git-wip-us.apache.org/repos/asf/spark/blob/6c0c65fc/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index b7e8636..ed8f56a 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs._
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.mapred.Master
import org.apache.hadoop.mapreduce.MRJobConfig
-import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.util.StringUtils
import org.apache.hadoop.yarn.api._
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
@@ -191,23 +191,11 @@ trait ClientBase extends Logging {
// Upload Spark and the application JAR to the remote file system if
necessary. Add them as
// local resources to the application master.
val fs = FileSystem.get(conf)
-
-val delegTokenRenewer = Master.getMasterPrincipal(conf)
-if (UserGroupInformation.isSecurityEnabled()) {
- if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
-val errorMessage = "Can't get Master Kerberos principal for use as
renewer"
-logError(errorMessage)
-throw new SparkException(errorMessage)
- }
-}
val dst = new Path(fs.getHomeDirectory(), appStagingDir)
-val replication = sparkConf.getInt("spark.yarn.submit.file.replication",
3).toShort
-
-if (UserGroupInformation.isSecurityEnabled()) {
- val dstFs = dst.getFileSystem(conf)
- dstFs.addDelegationTokens(delegTokenRenewer, credentials)
-}
+val nns = ClientBase.getNameNodesToAccess(sparkConf) + dst
+ClientBase.obtainTokensForNamenodes(nns, conf, credentials)
+val replication = sparkConf.getInt("spark.yarn.submit.file.replication",
3).toShort
val localResources = HashMap[String, LocalResource]()
FileSystem.mkdirs(fs, dst, new FsPermission(STAGING_DIR_PERMISSION))
@@ -614,4 +602,40 @@ object ClientBase extends Loggin
git commit: SPARK-1890 and SPARK-1891- add admin and modify acls
Repository: spark
Updated Branches:
refs/heads/master 2c0f705e2 -> 1ca23
SPARK-1890 and SPARK-1891- add admin and modify acls
It was easier to combine these 2 jira since they touch many of the same places.
This pr adds the following:
- adds modify acls
- adds admin acls (list of admins/users that get added to both view and modify
acls)
- modify Kill button on UI to take modify acls into account
- changes config name of spark.ui.acls.enable to spark.acls.enable since I
choose poorly in original name. We keep backwards compatibility so people can
still use spark.ui.acls.enable. The acls should apply to any web ui as well as
any CLI interfaces.
- send view and modify acls information on to YARN so that YARN interfaces can
use (yarn cli for killing applications for example).
Author: Thomas Graves
Closes #1196 from tgravescs/SPARK-1890 and squashes the following commits:
8292eb1 [Thomas Graves] review comments
b92ec89 [Thomas Graves] remove unneeded variable from applistener
4c765f4 [Thomas Graves] Add in admin acls
72eb0ac [Thomas Graves] Add modify acls
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1ca2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1ca2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1ca2
Branch: refs/heads/master
Commit: 1ca23d3aa40423d658cfbf2c956ad415a6b1
Parents: 2c0f705
Author: Thomas Graves
Authored: Tue Aug 5 12:52:52 2014 -0500
Committer: Thomas Graves
Committed: Tue Aug 5 12:52:52 2014 -0500
--
.../org/apache/spark/SecurityManager.scala | 107 ---
.../deploy/history/FsHistoryProvider.scala | 4 +-
.../scheduler/ApplicationEventListener.scala| 4 +-
.../apache/spark/ui/jobs/JobProgressTab.scala | 2 +-
.../org/apache/spark/SecurityManagerSuite.scala | 83 --
docs/configuration.md | 27 -
docs/security.md| 7 +-
.../apache/spark/deploy/yarn/ClientBase.scala | 9 +-
8 files changed, 206 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1ca2/core/src/main/scala/org/apache/spark/SecurityManager.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala
b/core/src/main/scala/org/apache/spark/SecurityManager.scala
index 74aa441..25c2c9f 100644
--- a/core/src/main/scala/org/apache/spark/SecurityManager.scala
+++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala
@@ -41,10 +41,19 @@ import org.apache.spark.deploy.SparkHadoopUtil
* secure the UI if it has data that other users should not be allowed to see.
The javax
* servlet filter specified by the user can authenticate the user and then
once the user
* is logged in, Spark can compare that user versus the view acls to make sure
they are
- * authorized to view the UI. The configs 'spark.ui.acls.enable' and
'spark.ui.view.acls'
+ * authorized to view the UI. The configs 'spark.acls.enable' and
'spark.ui.view.acls'
* control the behavior of the acls. Note that the person who started the
application
* always has view access to the UI.
*
+ * Spark has a set of modify acls (`spark.modify.acls`) that controls which
users have permission
+ * to modify a single application. This would include things like killing the
application. By
+ * default the person who started the application has modify access. For
modify access through
+ * the UI, you must have a filter that does authentication in place for the
modify acls to work
+ * properly.
+ *
+ * Spark also has a set of admin acls (`spark.admin.acls`) which is a set of
users/administrators
+ * who always have permission to view or modify the Spark application.
+ *
* Spark does not currently support encryption after authentication.
*
* At this point spark has multiple communication protocols that need to be
secured and
@@ -137,18 +146,32 @@ private[spark] class SecurityManager(sparkConf:
SparkConf) extends Logging {
private val sparkSecretLookupKey = "sparkCookie"
private val authOn = sparkConf.getBoolean("spark.authenticate", false)
- private var uiAclsOn = sparkConf.getBoolean("spark.ui.acls.enable", false)
+ // keep spark.ui.acls.enable for backwards compatibility with 1.0
+ private var aclsOn = sparkConf.getOption("spark.acls.enable").getOrElse(
+sparkConf.get("spark.ui.acls.enable", "false")).toBoolean
+
+ // admin acls should be set before view or modify acls
+ private var adminAcls: Set[String] =
+stringToSet(sparkConf.get("spark.admin.acls", ""))
private var viewAcls: Set[String] = _
+
+ // list of users who have permission to modify the application. This should
+ // apply to both UI and CLI for things like killing th
git commit: SPARK-1890 and SPARK-1891- add admin and modify acls
Repository: spark
Updated Branches:
refs/heads/branch-1.1 6c0c65fc8 -> e3fe6571d
SPARK-1890 and SPARK-1891- add admin and modify acls
It was easier to combine these 2 jira since they touch many of the same places.
This pr adds the following:
- adds modify acls
- adds admin acls (list of admins/users that get added to both view and modify
acls)
- modify Kill button on UI to take modify acls into account
- changes config name of spark.ui.acls.enable to spark.acls.enable since I
choose poorly in original name. We keep backwards compatibility so people can
still use spark.ui.acls.enable. The acls should apply to any web ui as well as
any CLI interfaces.
- send view and modify acls information on to YARN so that YARN interfaces can
use (yarn cli for killing applications for example).
Author: Thomas Graves
Closes #1196 from tgravescs/SPARK-1890 and squashes the following commits:
8292eb1 [Thomas Graves] review comments
b92ec89 [Thomas Graves] remove unneeded variable from applistener
4c765f4 [Thomas Graves] Add in admin acls
72eb0ac [Thomas Graves] Add modify acls
(cherry picked from commit 1ca23d3aa40423d658cfbf2c956ad415a6b1)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e3fe6571
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e3fe6571
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e3fe6571
Branch: refs/heads/branch-1.1
Commit: e3fe6571decfdc406ec6d505fd92f9f2b85a618c
Parents: 6c0c65f
Author: Thomas Graves
Authored: Tue Aug 5 12:52:52 2014 -0500
Committer: Thomas Graves
Committed: Tue Aug 5 12:53:05 2014 -0500
--
.../org/apache/spark/SecurityManager.scala | 107 ---
.../deploy/history/FsHistoryProvider.scala | 4 +-
.../scheduler/ApplicationEventListener.scala| 4 +-
.../apache/spark/ui/jobs/JobProgressTab.scala | 2 +-
.../org/apache/spark/SecurityManagerSuite.scala | 83 --
docs/configuration.md | 27 -
docs/security.md| 7 +-
.../apache/spark/deploy/yarn/ClientBase.scala | 9 +-
8 files changed, 206 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e3fe6571/core/src/main/scala/org/apache/spark/SecurityManager.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala
b/core/src/main/scala/org/apache/spark/SecurityManager.scala
index 74aa441..25c2c9f 100644
--- a/core/src/main/scala/org/apache/spark/SecurityManager.scala
+++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala
@@ -41,10 +41,19 @@ import org.apache.spark.deploy.SparkHadoopUtil
* secure the UI if it has data that other users should not be allowed to see.
The javax
* servlet filter specified by the user can authenticate the user and then
once the user
* is logged in, Spark can compare that user versus the view acls to make sure
they are
- * authorized to view the UI. The configs 'spark.ui.acls.enable' and
'spark.ui.view.acls'
+ * authorized to view the UI. The configs 'spark.acls.enable' and
'spark.ui.view.acls'
* control the behavior of the acls. Note that the person who started the
application
* always has view access to the UI.
*
+ * Spark has a set of modify acls (`spark.modify.acls`) that controls which
users have permission
+ * to modify a single application. This would include things like killing the
application. By
+ * default the person who started the application has modify access. For
modify access through
+ * the UI, you must have a filter that does authentication in place for the
modify acls to work
+ * properly.
+ *
+ * Spark also has a set of admin acls (`spark.admin.acls`) which is a set of
users/administrators
+ * who always have permission to view or modify the Spark application.
+ *
* Spark does not currently support encryption after authentication.
*
* At this point spark has multiple communication protocols that need to be
secured and
@@ -137,18 +146,32 @@ private[spark] class SecurityManager(sparkConf:
SparkConf) extends Logging {
private val sparkSecretLookupKey = "sparkCookie"
private val authOn = sparkConf.getBoolean("spark.authenticate", false)
- private var uiAclsOn = sparkConf.getBoolean("spark.ui.acls.enable", false)
+ // keep spark.ui.acls.enable for backwards compatibility with 1.0
+ private var aclsOn = sparkConf.getOption("spark.acls.enable").getOrElse(
+sparkConf.get("spark.ui.acls.enable", "false")).toBoolean
+
+ // admin acls should be set before view or modify acls
+ private var adminAcls: Set[String] =
+stringToSet(sparkConf.get("spark.admin.acls", ""))
private var viewAcls: Set[String] = _
+
+ // list of users who have
git commit: SPARK-1680: use configs for specifying environment variables on YARN
Repository: spark Updated Branches: refs/heads/master 74f82c71b -> 41e0a21b2 SPARK-1680: use configs for specifying environment variables on YARN Note that this also documents spark.executorEnv.* which to me means its public. If we don't want that please speak up. Author: Thomas Graves Closes #1512 from tgravescs/SPARK-1680 and squashes the following commits: 11525df [Thomas Graves] more doc changes 553bad0 [Thomas Graves] fix documentation 152bf7c [Thomas Graves] fix docs 5382326 [Thomas Graves] try fix docs 32f86a4 [Thomas Graves] use configs for specifying environment variables on YARN Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/41e0a21b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/41e0a21b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/41e0a21b Branch: refs/heads/master Commit: 41e0a21b22ccd2788dc079790788e505b0d4e37d Parents: 74f82c7 Author: Thomas Graves Authored: Tue Aug 5 15:57:32 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 15:57:32 2014 -0500 -- docs/configuration.md | 8 +++ docs/running-on-yarn.md | 22 +++- .../apache/spark/deploy/yarn/ClientBase.scala | 13 .../deploy/yarn/ExecutorRunnableUtil.scala | 6 +- 4 files changed, 43 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/41e0a21b/docs/configuration.md -- diff --git a/docs/configuration.md b/docs/configuration.md index 25adea2..5e7556c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -206,6 +206,14 @@ Apart from these, the following properties are also available, and may be useful used during aggregation goes above this amount, it will spill the data into disks. + + spark.executorEnv.[EnvironmentVariableName] + (none) + +Add the environment variable specified by EnvironmentVariableName to the Executor +process. The user can specify multiple of these and to set multiple environment variables. + + Shuffle Behavior http://git-wip-us.apache.org/repos/asf/spark/blob/41e0a21b/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 573930d..9bc20db 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -17,10 +17,6 @@ To build Spark yourself, refer to the [building with Maven guide](building-with- Most of the configs are the same for Spark on YARN as for other deployment modes. See the [configuration page](configuration.html) for more information on those. These are configs that are specific to Spark on YARN. - Environment Variables - -* `SPARK_YARN_USER_ENV`, to add environment variables to the Spark processes launched on YARN. This can be a comma separated list of environment variables, e.g. `SPARK_YARN_USER_ENV="JAVA_HOME=/jdk64,FOO=bar"`. - Spark Properties @@ -110,7 +106,23 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.access.namenodes (none) -A list of secure HDFS namenodes your Spark application is going to access. For example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The Spark application must have acess to the namenodes listed and Kerberos must be properly configured to be able to access them (either in the same realm or in a trusted realm). Spark acquires security tokens for each of the namenodes so that the Spark application can access those remote HDFS clusters. +A list of secure HDFS namenodes your Spark application is going to access. For +example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. +The Spark application must have acess to the namenodes listed and Kerberos must +be properly configured to be able to access them (either in the same realm or in +a trusted realm). Spark acquires security tokens for each of the namenodes so that +the Spark application can access those remote HDFS clusters. + + + + spark.yarn.appMasterEnv.[EnvironmentVariableName] + (none) + + Add the environment variable specified by EnvironmentVariableName to the + Application Master process launched on YARN. The user can specify multiple of + these and to set multiple environment variables. In yarn-cluster mode this controls + the environment of the SPARK driver and in yarn-client mode it only controls + the environment of the executor launcher. http://git-wip-us.apache.org/repos/asf/spark/blob/41e0a21b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala ---
git commit: SPARK-1680: use configs for specifying environment variables on YARN
Repository: spark Updated Branches: refs/heads/branch-1.1 46b698307 -> 7b798e10e SPARK-1680: use configs for specifying environment variables on YARN Note that this also documents spark.executorEnv.* which to me means its public. If we don't want that please speak up. Author: Thomas Graves Closes #1512 from tgravescs/SPARK-1680 and squashes the following commits: 11525df [Thomas Graves] more doc changes 553bad0 [Thomas Graves] fix documentation 152bf7c [Thomas Graves] fix docs 5382326 [Thomas Graves] try fix docs 32f86a4 [Thomas Graves] use configs for specifying environment variables on YARN (cherry picked from commit 41e0a21b22ccd2788dc079790788e505b0d4e37d) Signed-off-by: Thomas Graves Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7b798e10 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7b798e10 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7b798e10 Branch: refs/heads/branch-1.1 Commit: 7b798e10e214cd407d3399e2cab9e3789f9a929e Parents: 46b6983 Author: Thomas Graves Authored: Tue Aug 5 15:57:32 2014 -0500 Committer: Thomas Graves Committed: Tue Aug 5 15:57:42 2014 -0500 -- docs/configuration.md | 8 +++ docs/running-on-yarn.md | 22 +++- .../apache/spark/deploy/yarn/ClientBase.scala | 13 .../deploy/yarn/ExecutorRunnableUtil.scala | 6 +- 4 files changed, 43 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/7b798e10/docs/configuration.md -- diff --git a/docs/configuration.md b/docs/configuration.md index 1333465..6ae453d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -206,6 +206,14 @@ Apart from these, the following properties are also available, and may be useful used during aggregation goes above this amount, it will spill the data into disks. + + spark.executorEnv.[EnvironmentVariableName] + (none) + +Add the environment variable specified by EnvironmentVariableName to the Executor +process. The user can specify multiple of these and to set multiple environment variables. + + Shuffle Behavior http://git-wip-us.apache.org/repos/asf/spark/blob/7b798e10/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 573930d..9bc20db 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -17,10 +17,6 @@ To build Spark yourself, refer to the [building with Maven guide](building-with- Most of the configs are the same for Spark on YARN as for other deployment modes. See the [configuration page](configuration.html) for more information on those. These are configs that are specific to Spark on YARN. - Environment Variables - -* `SPARK_YARN_USER_ENV`, to add environment variables to the Spark processes launched on YARN. This can be a comma separated list of environment variables, e.g. `SPARK_YARN_USER_ENV="JAVA_HOME=/jdk64,FOO=bar"`. - Spark Properties @@ -110,7 +106,23 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.access.namenodes (none) -A list of secure HDFS namenodes your Spark application is going to access. For example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The Spark application must have acess to the namenodes listed and Kerberos must be properly configured to be able to access them (either in the same realm or in a trusted realm). Spark acquires security tokens for each of the namenodes so that the Spark application can access those remote HDFS clusters. +A list of secure HDFS namenodes your Spark application is going to access. For +example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. +The Spark application must have acess to the namenodes listed and Kerberos must +be properly configured to be able to access them (either in the same realm or in +a trusted realm). Spark acquires security tokens for each of the namenodes so that +the Spark application can access those remote HDFS clusters. + + + + spark.yarn.appMasterEnv.[EnvironmentVariableName] + (none) + + Add the environment variable specified by EnvironmentVariableName to the + Application Master process launched on YARN. The user can specify multiple of + these and to set multiple environment variables. In yarn-cluster mode this controls + the environment of the SPARK driver and in yarn-client mode it only controls + the environment of the executor launcher. http://git-wip-us.apache.org/repos/asf/spark/blob/7
git commit: [SPARK-3072] YARN - Exit when reach max number failed executors
Repository: spark
Updated Branches:
refs/heads/master cd0720ca7 -> 7eb9cbc27
[SPARK-3072] YARN - Exit when reach max number failed executors
In some cases on hadoop 2.x the spark application master doesn't properly exit
and hangs around for 10 minutes after its really done. We should make sure it
exits properly and stops the driver.
Author: Thomas Graves
Closes #2022 from tgravescs/SPARK-3072 and squashes the following commits:
665701d [Thomas Graves] Exit when reach max number failed executors
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7eb9cbc2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7eb9cbc2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7eb9cbc2
Branch: refs/heads/master
Commit: 7eb9cbc273d758522e787fcb2ef68ef65911475f
Parents: cd0720c
Author: Thomas Graves
Authored: Tue Aug 19 09:40:31 2014 -0500
Committer: Thomas Graves
Committed: Tue Aug 19 09:40:31 2014 -0500
--
.../spark/deploy/yarn/ApplicationMaster.scala | 33 +---
.../spark/deploy/yarn/ExecutorLauncher.scala| 5 +--
.../spark/deploy/yarn/ApplicationMaster.scala | 16 +++---
.../spark/deploy/yarn/ExecutorLauncher.scala| 5 +--
4 files changed, 40 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb9cbc2/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 62b5c3b..46a01f5 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -267,12 +267,10 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
// TODO: This is a bit ugly. Can we make it nicer?
// TODO: Handle container failure
- // Exists the loop if the user thread exits.
- while (yarnAllocator.getNumExecutorsRunning < args.numExecutors &&
userThread.isAlive) {
-if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
- finishApplicationMaster(FinalApplicationStatus.FAILED,
-"max number of executor failures reached")
-}
+ // Exits the loop if the user thread exits.
+ while (yarnAllocator.getNumExecutorsRunning < args.numExecutors &&
userThread.isAlive
+ && !isFinished) {
+checkNumExecutorsFailed()
yarnAllocator.allocateContainers(
math.max(args.numExecutors - yarnAllocator.getNumExecutorsRunning,
0))
Thread.sleep(ApplicationMaster.ALLOCATE_HEARTBEAT_INTERVAL)
@@ -303,11 +301,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
val t = new Thread {
override def run() {
-while (userThread.isAlive) {
- if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
-finishApplicationMaster(FinalApplicationStatus.FAILED,
- "max number of executor failures reached")
- }
+while (userThread.isAlive && !isFinished) {
+ checkNumExecutorsFailed()
val missingExecutorCount = args.numExecutors -
yarnAllocator.getNumExecutorsRunning
if (missingExecutorCount > 0) {
logInfo("Allocating %d containers to make up for (potentially)
lost containers".
@@ -327,6 +322,22 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
t
}
+ private def checkNumExecutorsFailed() {
+if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
+ logInfo("max number of executor failures reached")
+ finishApplicationMaster(FinalApplicationStatus.FAILED,
+"max number of executor failures reached")
+ // make sure to stop the user thread
+ val sparkContext = ApplicationMaster.sparkContextRef.get()
+ if (sparkContext != null) {
+logInfo("Invoking sc stop from checkNumExecutorsFailed")
+sparkContext.stop()
+ } else {
+logError("sparkContext is null when should shutdown")
+ }
+}
+ }
+
private def sendProgress() {
logDebug("Sending progress")
// Simulated with an allocate request with no nodes requested ...
http://git-wip-us.apache.org/repos/asf/spark/blob/7eb9cbc2/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLaunche
git commit: [SPARK-3072] YARN - Exit when reach max number failed executors
Repository: spark
Updated Branches:
refs/heads/branch-1.1 f3b0f34b4 -> 1418893da
[SPARK-3072] YARN - Exit when reach max number failed executors
In some cases on hadoop 2.x the spark application master doesn't properly exit
and hangs around for 10 minutes after its really done. We should make sure it
exits properly and stops the driver.
Author: Thomas Graves
Closes #2022 from tgravescs/SPARK-3072 and squashes the following commits:
665701d [Thomas Graves] Exit when reach max number failed executors
(cherry picked from commit 7eb9cbc273d758522e787fcb2ef68ef65911475f)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1418893d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1418893d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1418893d
Branch: refs/heads/branch-1.1
Commit: 1418893da557892b86fc47f1e41e91880d4f8eda
Parents: f3b0f34
Author: Thomas Graves
Authored: Tue Aug 19 09:40:31 2014 -0500
Committer: Thomas Graves
Committed: Tue Aug 19 09:40:55 2014 -0500
--
.../spark/deploy/yarn/ApplicationMaster.scala | 33 +---
.../spark/deploy/yarn/ExecutorLauncher.scala| 5 +--
.../spark/deploy/yarn/ApplicationMaster.scala | 16 +++---
.../spark/deploy/yarn/ExecutorLauncher.scala| 5 +--
4 files changed, 40 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1418893d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 62b5c3b..46a01f5 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -267,12 +267,10 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
// TODO: This is a bit ugly. Can we make it nicer?
// TODO: Handle container failure
- // Exists the loop if the user thread exits.
- while (yarnAllocator.getNumExecutorsRunning < args.numExecutors &&
userThread.isAlive) {
-if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
- finishApplicationMaster(FinalApplicationStatus.FAILED,
-"max number of executor failures reached")
-}
+ // Exits the loop if the user thread exits.
+ while (yarnAllocator.getNumExecutorsRunning < args.numExecutors &&
userThread.isAlive
+ && !isFinished) {
+checkNumExecutorsFailed()
yarnAllocator.allocateContainers(
math.max(args.numExecutors - yarnAllocator.getNumExecutorsRunning,
0))
Thread.sleep(ApplicationMaster.ALLOCATE_HEARTBEAT_INTERVAL)
@@ -303,11 +301,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
val t = new Thread {
override def run() {
-while (userThread.isAlive) {
- if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
-finishApplicationMaster(FinalApplicationStatus.FAILED,
- "max number of executor failures reached")
- }
+while (userThread.isAlive && !isFinished) {
+ checkNumExecutorsFailed()
val missingExecutorCount = args.numExecutors -
yarnAllocator.getNumExecutorsRunning
if (missingExecutorCount > 0) {
logInfo("Allocating %d containers to make up for (potentially)
lost containers".
@@ -327,6 +322,22 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
t
}
+ private def checkNumExecutorsFailed() {
+if (yarnAllocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
+ logInfo("max number of executor failures reached")
+ finishApplicationMaster(FinalApplicationStatus.FAILED,
+"max number of executor failures reached")
+ // make sure to stop the user thread
+ val sparkContext = ApplicationMaster.sparkContextRef.get()
+ if (sparkContext != null) {
+logInfo("Invoking sc stop from checkNumExecutorsFailed")
+sparkContext.stop()
+ } else {
+logError("sparkContext is null when should shutdown")
+ }
+}
+ }
+
private def sendProgress() {
logDebug("Sending progress")
// Simulated with an allocate request with no nodes requested ...
http://git-wip-us.apache.org/repos/asf/spark/blob/1418893d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/
git commit: [SPARK-2742][yarn] delete useless variables
Repository: spark
Updated Branches:
refs/heads/master 050f8d01e -> 220c2d769
[SPARK-2742][yarn] delete useless variables
Author: XuTingjun <1039320...@qq.com>
Closes #1614 from XuTingjun/yarn-bug and squashes the following commits:
f07096e [XuTingjun] Update ClientArguments.scala
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/220c2d76
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/220c2d76
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/220c2d76
Branch: refs/heads/master
Commit: 220c2d7699a522c50d26cd6fdd94f4635823e408
Parents: 050f8d0
Author: XuTingjun <1039320...@qq.com>
Authored: Fri Aug 22 10:45:00 2014 -0500
Committer: Thomas Graves
Committed: Fri Aug 22 10:45:00 2014 -0500
--
.../scala/org/apache/spark/deploy/yarn/ClientArguments.scala| 5 +
1 file changed, 1 insertion(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/220c2d76/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
index 62f9b3cf..afa4fd4 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
@@ -39,7 +39,6 @@ class ClientArguments(val args: Array[String], val sparkConf:
SparkConf) {
var amMemory: Int = 512 // MB
var amClass: String = "org.apache.spark.deploy.yarn.ApplicationMaster"
var appName: String = "Spark"
- var inputFormatInfo: List[InputFormatInfo] = null
var priority = 0
parseArgs(args.toList)
@@ -58,8 +57,7 @@ class ClientArguments(val args: Array[String], val sparkConf:
SparkConf) {
private def parseArgs(inputArgs: List[String]): Unit = {
val userArgsBuffer: ArrayBuffer[String] = new ArrayBuffer[String]()
-val inputFormatMap: HashMap[String, InputFormatInfo] = new HashMap[String,
InputFormatInfo]()
-
+
var args = inputArgs
while (!args.isEmpty) {
@@ -145,7 +143,6 @@ class ClientArguments(val args: Array[String], val
sparkConf: SparkConf) {
}
userArgs = userArgsBuffer.readOnly
-inputFormatInfo = inputFormatMap.values.toList
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[1/2] [SPARK-2933] [yarn] Refactor and cleanup Yarn AM code.
Repository: spark
Updated Branches:
refs/heads/master 6f671d04f -> b92d823ad
http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala
b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala
index 3474112..d162b4c 100644
---
a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientClusterScheduler.scala
@@ -19,22 +19,21 @@ package org.apache.spark.scheduler.cluster
import org.apache.spark._
import org.apache.hadoop.conf.Configuration
-import org.apache.spark.deploy.yarn.YarnAllocationHandler
+import org.apache.spark.deploy.yarn.YarnSparkHadoopUtil
import org.apache.spark.scheduler.TaskSchedulerImpl
import org.apache.spark.util.Utils
/**
- *
- * This scheduler launches executors through Yarn - by calling into Client to
launch ExecutorLauncher as AM.
+ * This scheduler launches executors through Yarn - by calling into Client to
launch the Spark AM.
*/
-private[spark] class YarnClientClusterScheduler(sc: SparkContext, conf:
Configuration) extends TaskSchedulerImpl(sc) {
+private[spark] class YarnClientClusterScheduler(sc: SparkContext, conf:
Configuration)
+ extends TaskSchedulerImpl(sc) {
def this(sc: SparkContext) = this(sc, new Configuration())
// By default, rack is unknown
override def getRackForHost(hostPort: String): Option[String] = {
val host = Utils.parseHostPort(hostPort)._1
-val retval = YarnAllocationHandler.lookupRack(conf, host)
-if (retval != null) Some(retval) else None
+Option(YarnSparkHadoopUtil.lookupRack(conf, host))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
index 833e249..a5f537d 100644
---
a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler.cluster
import org.apache.hadoop.yarn.api.records.{ApplicationId, YarnApplicationState}
import org.apache.spark.{SparkException, Logging, SparkContext}
-import org.apache.spark.deploy.yarn.{Client, ClientArguments,
ExecutorLauncher, YarnSparkHadoopUtil}
+import org.apache.spark.deploy.yarn.{Client, ClientArguments,
YarnSparkHadoopUtil}
import org.apache.spark.scheduler.TaskSchedulerImpl
import scala.collection.mutable.ArrayBuffer
@@ -60,10 +60,7 @@ private[spark] class YarnClientSchedulerBackend(
val argsArrayBuf = new ArrayBuffer[String]()
argsArrayBuf += (
- "--class", "notused",
- "--jar", null, // The primary jar will be added dynamically in
SparkContext.
- "--args", hostport,
- "--am-class", classOf[ExecutorLauncher].getName
+ "--args", hostport
)
// process any optional arguments, given either as environment variables
http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala
b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala
index 9aeca4a..69f4022 100644
---
a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala
@@ -18,16 +18,17 @@
package org.apache.spark.scheduler.cluster
import org.apache.spark._
-import org.apache.spark.deploy.yarn.{ApplicationMaster, YarnAllocationHandler}
+import org.apache.spark.deploy.yarn.{ApplicationMaster, YarnSparkHadoopUtil}
import org.apache.spark.scheduler.TaskSchedulerImpl
import org.apache.spark.util.Utils
import org.apache.hadoop.conf.Configuration
/**
- *
- * This is a simple extension to ClusterScheduler - to ensure that appropriate
initialization of ApplicationMaster, etc is done
+ * This is a simple extension to ClusterScheduler - to ensure that appropriate
initialization of
+ * ApplicationMaster, etc is done
*/
-private[spark] class YarnClusterScheduler(sc: SparkContext, conf:
Configuration) extends TaskSchedulerImpl(sc) {
+private
[2/2] git commit: [SPARK-2933] [yarn] Refactor and cleanup Yarn AM code.
[SPARK-2933] [yarn] Refactor and cleanup Yarn AM code. This change modifies the Yarn module so that all the logic related to running the ApplicationMaster is localized. Instead of, previously, 4 different classes with mostly identical code, now we have: - A single, shared ApplicationMaster class, which can operate both in client and cluster mode, and substitutes the old ApplicationMaster (for cluster mode) and ExecutorLauncher (for client mode). The benefit here is that all different execution modes for all supported yarn versions use the same shared code for monitoring executor allocation, setting up configuration, and monitoring the process's lifecycle. - A new YarnRMClient interface, which defines basic RM functionality needed by the ApplicationMaster. This interface has concrete implementations for each supported Yarn version. - A new YarnAllocator interface, which just abstracts the existing interface of the YarnAllocationHandler class. This is to avoid having to touch the allocator code too much in this change, although it might benefit from a similar effort in the future. The end result is much easier to understand code, with much less duplication, making it much easier to fix bugs, add features, and test everything knowing that all supported versions will behave the same. Author: Marcelo Vanzin Closes #2020 from vanzin/SPARK-2933 and squashes the following commits: 3bbf3e7 [Marcelo Vanzin] Merge branch 'master' into SPARK-2933 ff389ed [Marcelo Vanzin] Do not interrupt reporter thread from within itself. 3a8ed37 [Marcelo Vanzin] Remote stale comment. 0f5142c [Marcelo Vanzin] Review feedback. 41f8c8a [Marcelo Vanzin] Fix app status reporting. c0794be [Marcelo Vanzin] Correctly clean up staging directory. 92770cc [Marcelo Vanzin] Merge branch 'master' into SPARK-2933 ecaf332 [Marcelo Vanzin] Small fix to shutdown code. f02d3f8 [Marcelo Vanzin] Merge branch 'master' into SPARK-2933 f581122 [Marcelo Vanzin] Review feedback. 557fdeb [Marcelo Vanzin] Cleanup a couple more constants. be6068d [Marcelo Vanzin] Restore shutdown hook to clean up staging dir. 5150993 [Marcelo Vanzin] Some more cleanup. b6289ab [Marcelo Vanzin] Move cluster/client code to separate methods. ecb23cd [Marcelo Vanzin] More trivial cleanup. 34f1e63 [Marcelo Vanzin] Fix some questionable error handling. 5657c7d [Marcelo Vanzin] Finish app if SparkContext initialization times out. 0e4be3d [Marcelo Vanzin] Keep "ExecutorLauncher" as the main class for client-mode AM. 91beabb [Marcelo Vanzin] Fix UI filter registration. 8c72239 [Marcelo Vanzin] Trivial cleanups. 99a52d5 [Marcelo Vanzin] Changes to the yarn-alpha project to use common AM code. 848ca6d [Marcelo Vanzin] [SPARK-2933] [yarn] Refactor and cleanup Yarn AM code. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b92d823a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b92d823a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b92d823a Branch: refs/heads/master Commit: b92d823ad13f6fcc325eeb99563bea543871c6aa Parents: 6f671d0 Author: Marcelo Vanzin Authored: Wed Aug 27 11:02:04 2014 -0500 Committer: Thomas Graves Committed: Wed Aug 27 11:02:04 2014 -0500 -- .../spark/deploy/yarn/ApplicationMaster.scala | 453 --- .../spark/deploy/yarn/ExecutorLauncher.scala| 315 - .../deploy/yarn/YarnAllocationHandler.scala | 192 ++-- .../spark/deploy/yarn/YarnRMClientImpl.scala| 103 + .../spark/deploy/yarn/ApplicationMaster.scala | 430 ++ .../yarn/ApplicationMasterArguments.scala | 26 +- .../spark/deploy/yarn/ClientArguments.scala | 9 +- .../apache/spark/deploy/yarn/ClientBase.scala | 54 ++- .../spark/deploy/yarn/YarnAllocator.scala | 34 ++ .../apache/spark/deploy/yarn/YarnRMClient.scala | 67 +++ .../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 51 +++ .../cluster/YarnClientClusterScheduler.scala| 11 +- .../cluster/YarnClientSchedulerBackend.scala| 7 +- .../cluster/YarnClusterScheduler.scala | 17 +- .../spark/deploy/yarn/ApplicationMaster.scala | 413 - .../spark/deploy/yarn/ExecutorLauncher.scala| 276 --- .../deploy/yarn/YarnAllocationHandler.scala | 196 ++-- .../spark/deploy/yarn/YarnRMClientImpl.scala| 76 18 files changed, 892 insertions(+), 1838 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b92d823a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala -- diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala deleted file mode 100644 index
git commit: [SPARK-3347] [yarn] Fix yarn-alpha compilation.
Repository: spark
Updated Branches:
refs/heads/master 8f1f9aaf4 -> 066f31a6b
[SPARK-3347] [yarn] Fix yarn-alpha compilation.
Missing import. Oops.
Author: Marcelo Vanzin
Closes #2236 from vanzin/SPARK-3347 and squashes the following commits:
594fc39 [Marcelo Vanzin] [SPARK-3347] [yarn] Fix yarn-alpha compilation.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/066f31a6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/066f31a6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/066f31a6
Branch: refs/heads/master
Commit: 066f31a6b213121441fc9618abd5bae4a706a215
Parents: 8f1f9aa
Author: Marcelo Vanzin
Authored: Tue Sep 2 13:33:23 2014 -0500
Committer: Thomas Graves
Committed: Tue Sep 2 13:33:23 2014 -0500
--
yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/066f31a6/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 12f1cd3..10fc39b 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{Apps, Records}
import org.apache.spark.{Logging, SparkConf}
+import org.apache.spark.deploy.SparkHadoopUtil
/**
* Version of [[org.apache.spark.deploy.yarn.ClientBase]] tailored to YARN's
alpha API.
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-3187] [yarn] Cleanup allocator code.
Repository: spark
Updated Branches:
refs/heads/master c64cc435e -> 6a72a3694
[SPARK-3187] [yarn] Cleanup allocator code.
Move all shared logic to the base YarnAllocator class, and leave
the version-specific logic in the version-specific module.
Author: Marcelo Vanzin
Closes #2169 from vanzin/SPARK-3187 and squashes the following commits:
46c2826 [Marcelo Vanzin] Hide the privates.
4dc9c83 [Marcelo Vanzin] Actually release containers.
8b1a077 [Marcelo Vanzin] Changes to the Yarn alpha allocator.
f3f5f1d [Marcelo Vanzin] [SPARK-3187] [yarn] Cleanup allocator code.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a72a369
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6a72a369
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6a72a369
Branch: refs/heads/master
Commit: 6a72a36940311fcb3429bd34c8818bc7d513115c
Parents: c64cc43
Author: Marcelo Vanzin
Authored: Wed Sep 3 08:22:50 2014 -0500
Committer: Thomas Graves
Committed: Wed Sep 3 08:22:50 2014 -0500
--
.../deploy/yarn/YarnAllocationHandler.scala | 462 +++
.../spark/deploy/yarn/YarnAllocator.scala | 425 -
.../deploy/yarn/YarnAllocationHandler.scala | 402 +---
3 files changed, 495 insertions(+), 794 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6a72a369/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index 629cd13..9f9e16c 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -17,35 +17,21 @@
package org.apache.spark.deploy.yarn
-import java.util.concurrent.{CopyOnWriteArrayList, ConcurrentHashMap}
+import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.atomic.AtomicInteger
import scala.collection.JavaConversions._
-import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
+import scala.collection.mutable.{ArrayBuffer, HashMap}
-import org.apache.spark.{Logging, SparkConf, SparkEnv}
-import org.apache.spark.scheduler.{SplitInfo, TaskSchedulerImpl}
-import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
-import org.apache.spark.util.Utils
+import org.apache.spark.SparkConf
+import org.apache.spark.scheduler.SplitInfo
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.AMRMProtocol
-import org.apache.hadoop.yarn.api.records.{AMResponse, ApplicationAttemptId}
-import org.apache.hadoop.yarn.api.records.{Container, ContainerId}
-import org.apache.hadoop.yarn.api.records.{Priority, Resource, ResourceRequest}
-import org.apache.hadoop.yarn.api.protocolrecords.{AllocateRequest,
AllocateResponse}
+import org.apache.hadoop.yarn.api.records._
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest
import org.apache.hadoop.yarn.util.Records
-// TODO:
-// Too many params.
-// Needs to be mt-safe
-// Need to refactor this to make it 'cleaner' ... right now, all computation
is reactive - should
-// make it more proactive and decoupled.
-
-// Note that right now, we assume all node asks as uniform in terms of
capabilities and priority
-// Refer to
http://developer.yahoo.com/blogs/hadoop/posts/2011/03/mapreduce-nextgen-scheduler/
for
-// more info on how we are requesting for containers.
-
/**
* Acquires resources for executors from a ResourceManager and launches
executors in new containers.
*/
@@ -56,357 +42,20 @@ private[yarn] class YarnAllocationHandler(
appAttemptId: ApplicationAttemptId,
args: ApplicationMasterArguments,
preferredNodes: collection.Map[String, collection.Set[SplitInfo]])
- extends YarnAllocator with Logging {
-
- // These three are locked on allocatedHostToContainersMap. Complementary
data structures
- // allocatedHostToContainersMap : containers which are running : host,
Set
- // allocatedContainerToHostMap: container to host mapping.
- private val allocatedHostToContainersMap =
-new HashMap[String, collection.mutable.Set[ContainerId]]()
-
- private val allocatedContainerToHostMap = new HashMap[ContainerId, String]()
-
- // allocatedRackCount is populated ONLY if allocation happens (or
decremented if this is an
- // allocated node)
- // As with the two data structures above, tightly coupled with them, and to
be locked on
- // allocatedHostToContainersMap
- private val allocatedRackCount = new HashMap[String, Int]()
-
- // Containers which have been released.
- private val releas
git commit: [SPARK-3260] yarn - pass acls along with executor launch
Repository: spark
Updated Branches:
refs/heads/master 6a37ed838 -> 51b53a758
[SPARK-3260] yarn - pass acls along with executor launch
Pass along the acl settings when we launch a container so that they can be
applied to viewing the logs on a running NodeManager.
Author: Thomas Graves
Closes #2185 from tgravescs/SPARK-3260 and squashes the following commits:
6f94b5a [Thomas Graves] make unit test more robust
28b9dd3 [Thomas Graves] yarn - pass acls along with executor launch
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/51b53a75
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/51b53a75
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/51b53a75
Branch: refs/heads/master
Commit: 51b53a758c85f2e20ad9bd73ed815fcfa9c7180b
Parents: 6a37ed8
Author: Thomas Graves
Authored: Fri Sep 5 09:54:40 2014 -0500
Committer: Thomas Graves
Committed: Fri Sep 5 09:54:40 2014 -0500
--
.../spark/deploy/yarn/ExecutorRunnable.scala| 7 +-
.../deploy/yarn/YarnAllocationHandler.scala | 7 +-
.../spark/deploy/yarn/YarnRMClientImpl.scala| 7 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 13 ++--
.../apache/spark/deploy/yarn/ClientBase.scala | 6 +-
.../spark/deploy/yarn/YarnAllocator.scala | 10 +--
.../apache/spark/deploy/yarn/YarnRMClient.scala | 5 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 11 ++-
.../deploy/yarn/YarnSparkHadoopUtilSuite.scala | 76 +++-
.../spark/deploy/yarn/ExecutorRunnable.scala| 7 +-
.../deploy/yarn/YarnAllocationHandler.scala | 7 +-
.../spark/deploy/yarn/YarnRMClientImpl.scala| 7 +-
12 files changed, 129 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/51b53a75/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 7dae248..10cbeb8 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{Apps, ConverterUtils, Records, ProtoUtils}
-import org.apache.spark.{SparkConf, Logging}
+import org.apache.spark.{SecurityManager, SparkConf, Logging}
class ExecutorRunnable(
@@ -46,7 +46,8 @@ class ExecutorRunnable(
slaveId: String,
hostname: String,
executorMemory: Int,
-executorCores: Int)
+executorCores: Int,
+securityMgr: SecurityManager)
extends Runnable with ExecutorRunnableUtil with Logging {
var rpc: YarnRPC = YarnRPC.create(conf)
@@ -86,6 +87,8 @@ class ExecutorRunnable(
logInfo("Setting up executor with commands: " + commands)
ctx.setCommands(commands)
+
ctx.setApplicationACLs(YarnSparkHadoopUtil.getApplicationAclsForYarn(securityMgr))
+
// Send the start request to the ContainerManager
val startReq = Records.newRecord(classOf[StartContainerRequest])
.asInstanceOf[StartContainerRequest]
http://git-wip-us.apache.org/repos/asf/spark/blob/51b53a75/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index 9f9e16c..85d6274 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger
import scala.collection.JavaConversions._
import scala.collection.mutable.{ArrayBuffer, HashMap}
-import org.apache.spark.SparkConf
+import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.scheduler.SplitInfo
import org.apache.hadoop.conf.Configuration
@@ -41,8 +41,9 @@ private[yarn] class YarnAllocationHandler(
resourceManager: AMRMProtocol,
appAttemptId: ApplicationAttemptId,
args: ApplicationMasterArguments,
-preferredNodes: collection.Map[String, collection.Set[SplitInfo]])
- extends YarnAllocator(conf, sparkConf, args, preferredNodes) {
+preferredNodes: collection.Map[String, collection.Set[SplitInfo]],
+securityMgr: SecurityManager)
+ extends YarnAllocator(conf, sparkConf, args, preferredNodes, securityMgr) {
private val lastResponseId = new AtomicInteg
git commit: [SPARK-3375] spark on yarn container allocation issues
Repository: spark
Updated Branches:
refs/heads/master 51b53a758 -> 62c557609
[SPARK-3375] spark on yarn container allocation issues
If yarn doesn't get the containers immediately it stops asking for them and the
yarn application hangs with never getting any executors.
The issue here is that we are sending the number of containers as 0 after we
send the original one of X. on the yarn side this clears out the original
request.
For a ping we should just send empty asks.
Author: Thomas Graves
Closes #2275 from tgravescs/SPARK-3375 and squashes the following commits:
74b6820 [Thomas Graves] send empty resource requests when we aren't asking for
containers
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/62c55760
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/62c55760
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/62c55760
Branch: refs/heads/master
Commit: 62c557609929982eeec170fe12f810bedfcf97f2
Parents: 51b53a7
Author: Thomas Graves
Authored: Fri Sep 5 09:56:22 2014 -0500
Committer: Thomas Graves
Committed: Fri Sep 5 09:56:22 2014 -0500
--
.../spark/deploy/yarn/YarnAllocationHandler.scala | 13 +++--
.../spark/deploy/yarn/YarnAllocationHandler.scala | 8 +---
2 files changed, 12 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/62c55760/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index 85d6274..5a1b42c 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -51,12 +51,13 @@ private[yarn] class YarnAllocationHandler(
override protected def allocateContainers(count: Int): YarnAllocateResponse
= {
var resourceRequests: List[ResourceRequest] = null
-// default.
-if (count <= 0 || preferredHostToCount.isEmpty) {
- logDebug("numExecutors: " + count + ", host preferences: " +
-preferredHostToCount.isEmpty)
- resourceRequests = List(createResourceRequest(
-AllocationType.ANY, null, count,
YarnSparkHadoopUtil.RM_REQUEST_PRIORITY))
+logDebug("numExecutors: " + count)
+if (count <= 0) {
+ resourceRequests = List()
+} else if (preferredHostToCount.isEmpty) {
+logDebug("host preferences is empty")
+resourceRequests = List(createResourceRequest(
+ AllocationType.ANY, null, count,
YarnSparkHadoopUtil.RM_REQUEST_PRIORITY))
} else {
// request for all hosts in preferred nodes and for numExecutors -
// candidates.size, request by default allocation policy.
http://git-wip-us.apache.org/repos/asf/spark/blob/62c55760/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
--
diff --git
a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index c887cb5..5438f15 100644
---
a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++
b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -88,9 +88,11 @@ private[yarn] class YarnAllocationHandler(
private def addResourceRequests(numExecutors: Int) {
val containerRequests: List[ContainerRequest] =
- if (numExecutors <= 0 || preferredHostToCount.isEmpty) {
-logDebug("numExecutors: " + numExecutors + ", host preferences: " +
- preferredHostToCount.isEmpty)
+ if (numExecutors <= 0) {
+logDebug("numExecutors: " + numExecutors)
+List()
+ } else if (preferredHostToCount.isEmpty) {
+logDebug("host preferences is empty")
createResourceRequests(
AllocationType.ANY,
resource = null,
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-3286] - Cannot view ApplicationMaster UI when Yarn’s url scheme i...
Repository: spark
Updated Branches:
refs/heads/master b734ed0c2 -> 6f7a76838
[SPARK-3286] - Cannot view ApplicationMaster UI when Yarnâs url scheme i...
...s https
Author: Benoy Antony
Closes #2276 from benoyantony/SPARK-3286 and squashes the following commits:
c3d51ee [Benoy Antony] Use address with scheme, but Allpha version removes the
scheme
e82f94e [Benoy Antony] Use address with scheme, but Allpha version removes the
scheme
92127c9 [Benoy Antony] rebasing from master
450c536 [Benoy Antony] [SPARK-3286] - Cannot view ApplicationMaster UI when
Yarnâs url scheme is https
f060c02 [Benoy Antony] [SPARK-3286] - Cannot view ApplicationMaster UI when
Yarnâs url scheme is https
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6f7a7683
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6f7a7683
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6f7a7683
Branch: refs/heads/master
Commit: 6f7a76838f15687583e3b0ab43309a3c079368c4
Parents: b734ed0
Author: Benoy Antony
Authored: Wed Sep 10 11:59:39 2014 -0500
Committer: Thomas Graves
Committed: Wed Sep 10 11:59:39 2014 -0500
--
.../scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala| 4 +++-
.../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6f7a7683/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
index ad27a9a..fc30953 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
@@ -18,6 +18,7 @@
package org.apache.spark.deploy.yarn
import scala.collection.{Map, Set}
+import java.net.URI;
import org.apache.hadoop.net.NetUtils
import org.apache.hadoop.yarn.api._
@@ -97,7 +98,8 @@ private class YarnRMClientImpl(args:
ApplicationMasterArguments) extends YarnRMC
// Users can then monitor stderr/stdout on that node if required.
appMasterRequest.setHost(Utils.localHostName())
appMasterRequest.setRpcPort(0)
-appMasterRequest.setTrackingUrl(uiAddress)
+//remove the scheme from the url if it exists since Hadoop does not expect
scheme
+appMasterRequest.setTrackingUrl(new URI(uiAddress).getAuthority())
resourceManager.registerApplicationMaster(appMasterRequest)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6f7a7683/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index a879c83..5756263 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -189,7 +189,7 @@ private[spark] class ApplicationMaster(args:
ApplicationMasterArguments,
if (sc == null) {
finish(FinalApplicationStatus.FAILED, "Timed out waiting for
SparkContext.")
} else {
- registerAM(sc.ui.appUIHostPort, securityMgr)
+ registerAM(sc.ui.appUIAddress, securityMgr)
try {
userThread.join()
} finally {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
git commit: SPARK-1713. Use a thread pool for launching executors.
Repository: spark
Updated Branches:
refs/heads/master 26503fdf2 -> 1f4a648d4
SPARK-1713. Use a thread pool for launching executors.
This patch copies the approach used in the MapReduce application master for
launching containers.
Author: Sandy Ryza
Closes #663 from sryza/sandy-spark-1713 and squashes the following commits:
036550d [Sandy Ryza] SPARK-1713. [YARN] Use a threadpool for launching executor
containers
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1f4a648d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1f4a648d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1f4a648d
Branch: refs/heads/master
Commit: 1f4a648d4e30e837d6cf3ea8de1808e2254ad70b
Parents: 26503fd
Author: Sandy Ryza
Authored: Wed Sep 10 14:34:24 2014 -0500
Committer: Thomas Graves
Committed: Wed Sep 10 14:34:24 2014 -0500
--
docs/running-on-yarn.md | 7 +++
.../org/apache/spark/deploy/yarn/YarnAllocator.scala | 14 --
2 files changed, 19 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1f4a648d/docs/running-on-yarn.md
--
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 943f06b..d8b22f3 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -125,6 +125,13 @@ Most of the configs are the same for Spark on YARN as for
other deployment modes
the environment of the executor launcher.
+
+ spark.yarn.containerLauncherMaxThreads
+ 25
+
+The maximum number of threads to use in the application master for
launching executor containers.
+
+
# Launching Spark on YARN
http://git-wip-us.apache.org/repos/asf/spark/blob/1f4a648d/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index 02b9a81..0b8744f 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy.yarn
import java.util.{List => JList}
-import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent._
import java.util.concurrent.atomic.AtomicInteger
import scala.collection.JavaConversions._
@@ -32,6 +32,8 @@ import org.apache.spark.{Logging, SecurityManager, SparkConf,
SparkEnv}
import org.apache.spark.scheduler.{SplitInfo, TaskSchedulerImpl}
import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
+import com.google.common.util.concurrent.ThreadFactoryBuilder
+
object AllocationType extends Enumeration {
type AllocationType = Value
val HOST, RACK, ANY = Value
@@ -95,6 +97,14 @@ private[yarn] abstract class YarnAllocator(
protected val (preferredHostToCount, preferredRackToCount) =
generateNodeToWeight(conf, preferredNodes)
+ private val launcherPool = new ThreadPoolExecutor(
+// max pool size of Integer.MAX_VALUE is ignored because we use an
unbounded queue
+sparkConf.getInt("spark.yarn.containerLauncherMaxThreads", 25),
Integer.MAX_VALUE,
+1, TimeUnit.MINUTES,
+new LinkedBlockingQueue[Runnable](),
+new ThreadFactoryBuilder().setNameFormat("ContainerLauncher
#%d").setDaemon(true).build())
+ launcherPool.allowCoreThreadTimeOut(true)
+
def getNumExecutorsRunning: Int = numExecutorsRunning.intValue
def getNumExecutorsFailed: Int = numExecutorsFailed.intValue
@@ -283,7 +293,7 @@ private[yarn] abstract class YarnAllocator(
executorMemory,
executorCores,
securityMgr)
- new Thread(executorRunnable).start()
+ launcherPool.execute(executorRunnable)
}
}
logDebug("""
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Repository: spark
Updated Branches:
refs/heads/master c27718f37 -> ed1980ffa
[SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Updated pull request, reflecting YARN stable and alpha states. I am getting
intermittent test failures on my own test infrastructure. Is that tracked
anywhere yet?
Author: Chris Cope
Closes #2253 from copester/master and squashes the following commits:
5ad89da [Chris Cope] [SPARK-2140] Removing calculateAMMemory functions since
they are no longer needed.
52b4e45 [Chris Cope] [SPARK-2140] Updating heap memory calculation for YARN
stable and alpha.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ed1980ff
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ed1980ff
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ed1980ff
Branch: refs/heads/master
Commit: ed1980ffa9ccb87d76694ba910ef22df034bca49
Parents: c27718f
Author: Chris Cope
Authored: Thu Sep 11 08:13:07 2014 -0500
Committer: Thomas Graves
Committed: Thu Sep 11 08:13:07 2014 -0500
--
.../main/scala/org/apache/spark/deploy/yarn/Client.scala| 8
.../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 4 +---
.../org/apache/spark/deploy/yarn/ClientBaseSuite.scala | 3 ---
.../main/scala/org/apache/spark/deploy/yarn/Client.scala| 9 -
4 files changed, 1 insertion(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 10fc39b..aff9ab7 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -103,14 +103,6 @@ class Client(clientArgs: ClientArguments, hadoopConf:
Configuration, spConf: Spa
appContext
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int = {
-val minResMemory = newApp.getMinimumResourceCapability().getMemory()
-val amMemory = ((args.amMemory / minResMemory) * minResMemory) +
- ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
- memoryOverhead)
-amMemory
- }
-
def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 8075b7a..c96f731 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -300,8 +300,6 @@ trait ClientBase extends Logging {
retval.toString
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int
-
def setupSecurityToken(amContainer: ContainerLaunchContext)
def createContainerLaunchContext(
@@ -346,7 +344,7 @@ trait ClientBase extends Logging {
}
amContainer.setEnvironment(env)
-val amMemory = calculateAMMemory(newApp)
+val amMemory = args.amMemory
val javaOpts = ListBuffer[String]()
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
--
diff --git
a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
index 68cc289..5480eca 100644
---
a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
+++
b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
@@ -238,9 +238,6 @@ class ClientBaseSuite extends FunSuite with Matchers {
val sparkConf: SparkConf,
val yarnConf: YarnConfiguration) extends ClientBase {
-override def calculateAMMemory(newApp: GetNewApplicationResponse): Int =
- throw new UnsupportedOperationException()
-
override def setupSecurityToken(amContainer: ContainerLaunchContext): Unit
=
throw new UnsupportedOperationException()
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
--
diff --git
a/yarn/stable/src/main/scala/o
git commit: [SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Repository: spark
Updated Branches:
refs/heads/branch-1.1 e51ce9a55 -> 06fb2d057
[SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Updated pull request, reflecting YARN stable and alpha states. I am getting
intermittent test failures on my own test infrastructure. Is that tracked
anywhere yet?
Author: Chris Cope
Closes #2253 from copester/master and squashes the following commits:
5ad89da [Chris Cope] [SPARK-2140] Removing calculateAMMemory functions since
they are no longer needed.
52b4e45 [Chris Cope] [SPARK-2140] Updating heap memory calculation for YARN
stable and alpha.
(cherry picked from commit ed1980ffa9ccb87d76694ba910ef22df034bca49)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06fb2d05
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06fb2d05
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06fb2d05
Branch: refs/heads/branch-1.1
Commit: 06fb2d057beb50e9b690bf8b6d5bb7bdb16d8546
Parents: e51ce9a
Author: Chris Cope
Authored: Thu Sep 11 08:13:07 2014 -0500
Committer: Thomas Graves
Committed: Thu Sep 11 08:13:27 2014 -0500
--
.../main/scala/org/apache/spark/deploy/yarn/Client.scala| 8
.../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 4 +---
.../org/apache/spark/deploy/yarn/ClientBaseSuite.scala | 3 ---
.../main/scala/org/apache/spark/deploy/yarn/Client.scala| 9 -
4 files changed, 1 insertion(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 9be7854..3607eed 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -102,14 +102,6 @@ class Client(clientArgs: ClientArguments, hadoopConf:
Configuration, spConf: Spa
appContext
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int = {
-val minResMemory = newApp.getMinimumResourceCapability().getMemory()
-val amMemory = ((args.amMemory / minResMemory) * minResMemory) +
- ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
- memoryOverhead)
-amMemory
- }
-
def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 3897b3a..6da3b16 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -309,8 +309,6 @@ trait ClientBase extends Logging {
retval.toString
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int
-
def setupSecurityToken(amContainer: ContainerLaunchContext)
def createContainerLaunchContext(
@@ -353,7 +351,7 @@ trait ClientBase extends Logging {
}
amContainer.setEnvironment(env)
-val amMemory = calculateAMMemory(newApp)
+val amMemory = args.amMemory
val javaOpts = ListBuffer[String]()
http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
--
diff --git
a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
index 68cc289..5480eca 100644
---
a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
+++
b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
@@ -238,9 +238,6 @@ class ClientBaseSuite extends FunSuite with Matchers {
val sparkConf: SparkConf,
val yarnConf: YarnConfiguration) extends ClientBase {
-override def calculateAMMemory(newApp: GetNewApplicationResponse): Int =
- throw new UnsupportedOperationException()
-
override def setupSecurityToken(amContainer: ContainerLaunchContext): Unit
=
throw new UnsupportedOperationException()
http://git-wip-us.apache.org/repos/asf/spark/blob/06fb2d05/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
-
git commit: [SPARK-2558][DOCS] Add --queue example to YARN doc
Repository: spark Updated Branches: refs/heads/master b8634df1f -> f116f76bf [SPARK-2558][DOCS] Add --queue example to YARN doc Put original YARN queue spark-submit arg description in running-on-yarn html table and example command line Author: Mark G. Whitney Closes #2218 from kramimus/2258-yarndoc and squashes the following commits: 4b5d808 [Mark G. Whitney] remove yarn queue config f8cda0d [Mark G. Whitney] [SPARK-2558][DOCS] Add spark.yarn.queue description to YARN doc Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f116f76b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f116f76b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f116f76b Branch: refs/heads/master Commit: f116f76bf1f1610905ca094c8edc53151a78d2f4 Parents: b8634df Author: Mark G. Whitney Authored: Fri Sep 12 08:08:58 2014 -0500 Committer: Thomas Graves Committed: Fri Sep 12 08:08:58 2014 -0500 -- docs/running-on-yarn.md | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/f116f76b/docs/running-on-yarn.md -- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index d8b22f3..212248b 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -155,6 +155,7 @@ For example: --driver-memory 4g \ --executor-memory 2g \ --executor-cores 1 \ +--queue thequeue \ lib/spark-examples*.jar \ 10 - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
git commit: SPARK-3014. Log a more informative messages in a couple failure scenario...
Repository: spark
Updated Branches:
refs/heads/master 15a564598 -> 1d767967e
SPARK-3014. Log a more informative messages in a couple failure scenario...
...s
Author: Sandy Ryza
Closes #1934 from sryza/sandy-spark-3014 and squashes the following commits:
ae19cc1 [Sandy Ryza] SPARK-3014. Log a more informative messages in a couple
failure scenarios
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1d767967
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1d767967
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1d767967
Branch: refs/heads/master
Commit: 1d767967e925f1d727957c2d43383ef6ad2c5d5e
Parents: 15a5645
Author: Sandy Ryza
Authored: Fri Sep 12 16:48:28 2014 -0500
Committer: Thomas Graves
Committed: Fri Sep 12 16:48:28 2014 -0500
--
core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 6 --
.../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 6 ++
2 files changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1d767967/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 0fdb5ae..5ed3575 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy
import java.io.{File, PrintStream}
-import java.lang.reflect.InvocationTargetException
+import java.lang.reflect.{Modifier, InvocationTargetException}
import java.net.URL
import scala.collection.mutable.{ArrayBuffer, HashMap, Map}
@@ -323,7 +323,9 @@ object SparkSubmit {
}
val mainMethod = mainClass.getMethod("main", new Array[String](0).getClass)
-
+if (!Modifier.isStatic(mainMethod.getModifiers)) {
+ throw new IllegalStateException("The main method in the given main class
must be static")
+}
try {
mainMethod.invoke(null, childArgs.toArray)
} catch {
http://git-wip-us.apache.org/repos/asf/spark/blob/1d767967/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 878b6db..735d772 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -283,11 +283,9 @@ private[spark] class ApplicationMaster(args:
ApplicationMasterArguments,
}
val sparkContext = sparkContextRef.get()
-assert(sparkContext != null || count >= numTries)
if (sparkContext == null) {
- logError(
-"Unable to retrieve sparkContext inspite of waiting for %d,
numTries = %d".format(
- count * waitTime, numTries))
+ logError(("SparkContext did not initialize after waiting for %d ms.
Please check earlier"
++ " log output for errors. Failing the
application.").format(numTries * waitTime))
}
sparkContext
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
git commit: [SPARK-3456] YarnAllocator on alpha can lose container requests to RM
Repository: spark
Updated Branches:
refs/heads/master af2583826 -> 25311c2c5
[SPARK-3456] YarnAllocator on alpha can lose container requests to RM
Author: Thomas Graves
Closes #2373 from tgravescs/SPARK-3456 and squashes the following commits:
77e9532 [Thomas Graves] [SPARK-3456] YarnAllocator on alpha can lose container
requests to RM
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/25311c2c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/25311c2c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/25311c2c
Branch: refs/heads/master
Commit: 25311c2c545a60eb9dcf704814d4600987852155
Parents: af25838
Author: Thomas Graves
Authored: Fri Sep 12 20:31:11 2014 -0500
Committer: Thomas Graves
Committed: Fri Sep 12 20:31:11 2014 -0500
--
.../apache/spark/deploy/yarn/YarnAllocationHandler.scala | 11 ++-
.../org/apache/spark/deploy/yarn/YarnAllocator.scala | 8 ++--
.../apache/spark/deploy/yarn/YarnAllocationHandler.scala | 3 ++-
3 files changed, 14 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/25311c2c/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index 5a1b42c..6c93d85 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -48,16 +48,17 @@ private[yarn] class YarnAllocationHandler(
private val lastResponseId = new AtomicInteger()
private val releaseList: CopyOnWriteArrayList[ContainerId] = new
CopyOnWriteArrayList()
- override protected def allocateContainers(count: Int): YarnAllocateResponse
= {
+ override protected def allocateContainers(count: Int, pending: Int):
YarnAllocateResponse = {
var resourceRequests: List[ResourceRequest] = null
-logDebug("numExecutors: " + count)
+logDebug("asking for additional executors: " + count + " with already
pending: " + pending)
+val totalNumAsk = count + pending
if (count <= 0) {
resourceRequests = List()
} else if (preferredHostToCount.isEmpty) {
logDebug("host preferences is empty")
resourceRequests = List(createResourceRequest(
- AllocationType.ANY, null, count,
YarnSparkHadoopUtil.RM_REQUEST_PRIORITY))
+ AllocationType.ANY, null, totalNumAsk,
YarnSparkHadoopUtil.RM_REQUEST_PRIORITY))
} else {
// request for all hosts in preferred nodes and for numExecutors -
// candidates.size, request by default allocation policy.
@@ -80,7 +81,7 @@ private[yarn] class YarnAllocationHandler(
val anyContainerRequests: ResourceRequest = createResourceRequest(
AllocationType.ANY,
resource = null,
-count,
+totalNumAsk,
YarnSparkHadoopUtil.RM_REQUEST_PRIORITY)
val containerRequests: ArrayBuffer[ResourceRequest] = new
ArrayBuffer[ResourceRequest](
@@ -103,7 +104,7 @@ private[yarn] class YarnAllocationHandler(
req.addAllReleases(releasedContainerList)
if (count > 0) {
- logInfo("Allocating %d executor containers with %d of memory
each.".format(count,
+ logInfo("Allocating %d executor containers with %d of memory
each.".format(totalNumAsk,
executorMemory + memoryOverhead))
} else {
logDebug("Empty allocation req .. release : " + releasedContainerList)
http://git-wip-us.apache.org/repos/asf/spark/blob/25311c2c/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index 0b8744f..299e38a 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -112,6 +112,9 @@ private[yarn] abstract class YarnAllocator(
def allocateResources() = {
val missing = maxExecutors - numPendingAllocate.get() -
numExecutorsRunning.get()
+// this is needed by alpha, do it here since we add numPending right after
this
+val executorsPending = numPendingAllocate.get()
+
if (missing > 0) {
numPendingAllocate.addAndGet(missing)
logInfo("Will Allocate %d executor containers, each with %d
memory".format(
@@ -121,7 +124,7 @@ private[yarn] abstract class YarnAllocator(
logDebug("Empty allocation req
git commit: [SPARK-3410] The priority of shutdownhook for ApplicationMaster should not be integer literal
Repository: spark
Updated Branches:
refs/heads/master f493f7982 -> cc1464446
[SPARK-3410] The priority of shutdownhook for ApplicationMaster should not be
integer literal
I think, it need to keep the priority of shutdown hook for ApplicationMaster
than the priority of shutdown hook for o.a.h.FileSystem depending on changing
the priority for FileSystem.
Author: Kousuke Saruta
Closes #2283 from sarutak/SPARK-3410 and squashes the following commits:
1d44fef [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark
into SPARK-3410
bd6cc53 [Kousuke Saruta] Modified style
ee6f1aa [Kousuke Saruta] Added constant "SHUTDOWN_HOOK_PRIORITY" to
ApplicationMaster
54eb68f [Kousuke Saruta] Changed Shutdown hook priority to 20
2f0aee3 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark
into SPARK-3410
4c5cb93 [Kousuke Saruta] Modified the priority for AM's shutdown hook
217d1a4 [Kousuke Saruta] Removed unused import statements
717aba2 [Kousuke Saruta] Modified ApplicationMaster to make to keep the
priority of shutdown hook for ApplicationMaster higher than the priority of
shutdown hook for HDFS
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cc146444
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cc146444
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cc146444
Branch: refs/heads/master
Commit: cc14644460872efb344e8d895859d70213a40840
Parents: f493f79
Author: Kousuke Saruta
Authored: Mon Sep 15 08:53:58 2014 -0500
Committer: Thomas Graves
Committed: Mon Sep 15 08:53:58 2014 -0500
--
.../apache/spark/deploy/yarn/ApplicationMaster.scala | 13 +++--
1 file changed, 7 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/cc146444/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 735d772..cde5fff 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -21,12 +21,8 @@ import java.io.IOException
import java.net.Socket
import java.util.concurrent.atomic.AtomicReference
-import scala.collection.JavaConversions._
-import scala.util.Try
-
import akka.actor._
import akka.remote._
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.util.ShutdownHookManager
import org.apache.hadoop.yarn.api._
@@ -107,8 +103,11 @@ private[spark] class ApplicationMaster(args:
ApplicationMasterArguments,
}
}
}
-// Use priority 30 as it's higher than HDFS. It's the same priority
MapReduce is using.
-ShutdownHookManager.get().addShutdownHook(cleanupHook, 30)
+
+// Use higher priority than FileSystem.
+assert(ApplicationMaster.SHUTDOWN_HOOK_PRIORITY >
FileSystem.SHUTDOWN_HOOK_PRIORITY)
+ShutdownHookManager
+ .get().addShutdownHook(cleanupHook,
ApplicationMaster.SHUTDOWN_HOOK_PRIORITY)
// Call this to force generation of secret so it gets populated into the
// Hadoop UGI. This has to happen before the startUserClass which does a
@@ -407,6 +406,8 @@ private[spark] class ApplicationMaster(args:
ApplicationMasterArguments,
object ApplicationMaster extends Logging {
+ val SHUTDOWN_HOOK_PRIORITY: Int = 30
+
private var master: ApplicationMaster = _
def main(args: Array[String]) = {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH
Repository: spark
Updated Branches:
refs/heads/master c2f0821aa -> ebff7327a
[SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH
Based on https://github.com/apache/spark/pull/5478 that provide a
PYSPARK_ARCHIVES_PATH env. within this PR, we just should export
PYSPARK_ARCHIVES_PATH=/user/spark/pyspark.zip,/user/spark/python/lib/py4j-0.8.2.1-src.zip
in conf/spark-env.sh when we don't install PySpark on each node of Yarn. i run
python application successfully on yarn-client and yarn-cluster with this PR.
andrewor14 sryza Sephiroth-Lin Can you take a look at this?thanks.
Author: Lianhui Wang
Closes #5580 from lianhuiwang/SPARK-6869 and squashes the following commits:
66ffa43 [Lianhui Wang] Update Client.scala
c2ad0f9 [Lianhui Wang] Update Client.scala
1c8f664 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
008850a [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
f0b4ed8 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
150907b [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
20402cd [Lianhui Wang] use ZipEntry
9d87c3f [Lianhui Wang] update scala style
e7bd971 [Lianhui Wang] address vanzin's comments
4b8a3ed [Lianhui Wang] use pyArchivesEnvOpt
e6b573b [Lianhui Wang] address vanzin's comments
f11f84a [Lianhui Wang] zip pyspark archives
5192cca [Lianhui Wang] update import path
3b1e4c8 [Lianhui Wang] address tgravescs's comments
9396346 [Lianhui Wang] put zip to make-distribution.sh
0d2baf7 [Lianhui Wang] update import paths
e0179be [Lianhui Wang] add zip pyspark archives in build or sparksubmit
31e8e06 [Lianhui Wang] update code style
9f31dac [Lianhui Wang] update code and add comments
f72987c [Lianhui Wang] add archives path to PYTHONPATH
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ebff7327
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ebff7327
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ebff7327
Branch: refs/heads/master
Commit: ebff7327af5efa9f57c605284de4fae6b050ae0f
Parents: c2f0821
Author: Lianhui Wang
Authored: Fri May 8 08:44:46 2015 -0500
Committer: Thomas Graves
Committed: Fri May 8 08:44:46 2015 -0500
--
assembly/pom.xml| 21 ++
.../org/apache/spark/deploy/SparkSubmit.scala | 41
project/SparkBuild.scala| 37 +-
.../org/apache/spark/deploy/yarn/Client.scala | 23 ---
4 files changed, 114 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ebff7327/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 2b4d0a9..626c857 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -92,6 +92,27 @@
true
+
+
+ org.apache.maven.plugins
+maven-antrun-plugin
+
+
+package
+
+run
+
+
+
+
+
+
+
+
+
+
+
+
org.apache.maven.plugins
http://git-wip-us.apache.org/repos/asf/spark/blob/ebff7327/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 8a03279..329fa06 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -332,6 +332,47 @@ object SparkSubmit {
}
}
+// In yarn mode for a python app, add pyspark archives to files
+// that can be distributed with the job
+if (args.isPython && clusterManager == YARN) {
+ var pyArchives: String = null
+ val pyArchivesEnvOpt = sys.env.get("PYSPARK_ARCHIVES_PATH")
+ if (pyArchivesEnvOpt.isDefined) {
+pyArchives = pyArchivesEnvOpt.get
+ } else {
+if (!sys.env.contains("SPARK_HOME")) {
+ printErrorAndExit("SPARK_HOME does not exist for python application
in yarn mode.")
+}
+val pythonPath = new ArrayBuffer[String]
+for (sparkHome <- sys.env.get("SPARK_HOME")) {
+ val pyLibPath = Seq(sparkHome, "python",
"lib").mkString(File.separator)
+ val pyArchivesFile = new File(pyLibPath, "pyspark.zip")
+ if (!pyArchivesFile.exists()) {
+printErrorAndExit("pyspark.zip does not exist for pytho
spark git commit: [SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH
Repository: spark
Updated Branches:
refs/heads/branch-1.4 f5e9678e3 -> acf4bc1ca
[SPARK-6869] [PYSPARK] Add pyspark archives path to PYTHONPATH
Based on https://github.com/apache/spark/pull/5478 that provide a
PYSPARK_ARCHIVES_PATH env. within this PR, we just should export
PYSPARK_ARCHIVES_PATH=/user/spark/pyspark.zip,/user/spark/python/lib/py4j-0.8.2.1-src.zip
in conf/spark-env.sh when we don't install PySpark on each node of Yarn. i run
python application successfully on yarn-client and yarn-cluster with this PR.
andrewor14 sryza Sephiroth-Lin Can you take a look at this?thanks.
Author: Lianhui Wang
Closes #5580 from lianhuiwang/SPARK-6869 and squashes the following commits:
66ffa43 [Lianhui Wang] Update Client.scala
c2ad0f9 [Lianhui Wang] Update Client.scala
1c8f664 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
008850a [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
f0b4ed8 [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
150907b [Lianhui Wang] Merge remote-tracking branch 'remotes/apache/master'
into SPARK-6869
20402cd [Lianhui Wang] use ZipEntry
9d87c3f [Lianhui Wang] update scala style
e7bd971 [Lianhui Wang] address vanzin's comments
4b8a3ed [Lianhui Wang] use pyArchivesEnvOpt
e6b573b [Lianhui Wang] address vanzin's comments
f11f84a [Lianhui Wang] zip pyspark archives
5192cca [Lianhui Wang] update import path
3b1e4c8 [Lianhui Wang] address tgravescs's comments
9396346 [Lianhui Wang] put zip to make-distribution.sh
0d2baf7 [Lianhui Wang] update import paths
e0179be [Lianhui Wang] add zip pyspark archives in build or sparksubmit
31e8e06 [Lianhui Wang] update code style
9f31dac [Lianhui Wang] update code and add comments
f72987c [Lianhui Wang] add archives path to PYTHONPATH
(cherry picked from commit ebff7327af5efa9f57c605284de4fae6b050ae0f)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/acf4bc1c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/acf4bc1c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/acf4bc1c
Branch: refs/heads/branch-1.4
Commit: acf4bc1caa0818493f59da9991239d3caff806bd
Parents: f5e9678
Author: Lianhui Wang
Authored: Fri May 8 08:44:46 2015 -0500
Committer: Thomas Graves
Committed: Fri May 8 08:45:13 2015 -0500
--
assembly/pom.xml| 21 ++
.../org/apache/spark/deploy/SparkSubmit.scala | 41
project/SparkBuild.scala| 37 +-
.../org/apache/spark/deploy/yarn/Client.scala | 23 ---
4 files changed, 114 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/acf4bc1c/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 2b4d0a9..626c857 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -92,6 +92,27 @@
true
+
+
+ org.apache.maven.plugins
+maven-antrun-plugin
+
+
+package
+
+run
+
+
+
+
+
+
+
+
+
+
+
+
org.apache.maven.plugins
http://git-wip-us.apache.org/repos/asf/spark/blob/acf4bc1c/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 8a03279..329fa06 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -332,6 +332,47 @@ object SparkSubmit {
}
}
+// In yarn mode for a python app, add pyspark archives to files
+// that can be distributed with the job
+if (args.isPython && clusterManager == YARN) {
+ var pyArchives: String = null
+ val pyArchivesEnvOpt = sys.env.get("PYSPARK_ARCHIVES_PATH")
+ if (pyArchivesEnvOpt.isDefined) {
+pyArchives = pyArchivesEnvOpt.get
+ } else {
+if (!sys.env.contains("SPARK_HOME")) {
+ printErrorAndExit("SPARK_HOME does not exist for python application
in yarn mode.")
+}
+val pythonPath = new ArrayBuffer[String]
+for (sparkHome <- sys.env.get("SPARK_HOME")) {
+ val pyLibPath = Seq(sparkHome, "python",
"lib").mkString(File.separator)
+ val pyArchivesFile = new File(pyLibPath, "pyspark.zip")
+
spark git commit: [SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats.
Repository: spark
Updated Branches:
refs/heads/master 09265ad7c -> 3ddf051ee
[SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats.
Added faster RM-heartbeats on pending container allocations with multiplicative
back-off.
Also updated related documentations.
Author: ehnalis
Closes #6082 from ehnalis/yarn and squashes the following commits:
a1d2101 [ehnalis] MIss-spell fixed.
90f8ba4 [ehnalis] Changed default HB values.
6120295 [ehnalis] Removed the bug, when allocation heartbeat would not start
from initial value.
08bac63 [ehnalis] Refined style, grammar, removed duplicated code.
073d283 [ehnalis] [SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats.
d4408c9 [ehnalis] [SPARK-7533] [YARN] Decrease spacing between AM-RM heartbeats.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3ddf051e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3ddf051e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3ddf051e
Branch: refs/heads/master
Commit: 3ddf051ee7256f642f8a17768d161c7b5f55c7e1
Parents: 09265ad
Author: ehnalis
Authored: Wed May 20 08:27:39 2015 -0500
Committer: Thomas Graves
Committed: Wed May 20 08:27:39 2015 -0500
--
docs/running-on-yarn.md | 15 -
.../spark/deploy/yarn/ApplicationMaster.scala | 34 ++--
2 files changed, 39 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/3ddf051e/docs/running-on-yarn.md
--
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 51c1339..9d55f43 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -71,9 +71,22 @@ Most of the configs are the same for Spark on YARN as for
other deployment modes
spark.yarn.scheduler.heartbeat.interval-ms
- 5000
+ 3000
The interval in ms in which the Spark application master heartbeats into
the YARN ResourceManager.
+The value is capped at half the value of YARN's configuration for the
expiry interval
+(yarn.am.liveness-monitor.expiry-interval-ms).
+
+
+
+ spark.yarn.scheduler.initial-allocation.interval
+ 200ms
+
+The initial interval in which the Spark application master eagerly
heartbeats to the YARN ResourceManager
+when there are pending container allocation requests. It should be no
larger than
+spark.yarn.scheduler.heartbeat.interval-ms. The allocation
interval will doubled on
+successive eager heartbeats if pending containers still exist, until
+spark.yarn.scheduler.heartbeat.interval-ms is reached.
http://git-wip-us.apache.org/repos/asf/spark/blob/3ddf051e/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 2975296..63a6f2e 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -300,11 +300,14 @@ private[spark] class ApplicationMaster(
val expiryInterval =
yarnConf.getInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 12)
// we want to be reasonably responsive without causing too many requests
to RM.
-val schedulerInterval =
- sparkConf.getTimeAsMs("spark.yarn.scheduler.heartbeat.interval-ms", "5s")
+val heartbeatInterval = math.max(0, math.min(expiryInterval / 2,
+ sparkConf.getTimeAsMs("spark.yarn.scheduler.heartbeat.interval-ms",
"3s")))
-// must be <= expiryInterval / 2.
-val interval = math.max(0, math.min(expiryInterval / 2, schedulerInterval))
+// we want to check more frequently for pending containers
+val initialAllocationInterval = math.min(heartbeatInterval,
+
sparkConf.getTimeAsMs("spark.yarn.scheduler.initial-allocation.interval",
"200ms"))
+
+var nextAllocationInterval = initialAllocationInterval
// The number of failures in a row until Reporter thread give up
val reporterMaxFailures =
sparkConf.getInt("spark.yarn.scheduler.reporterThread.maxFailures", 5)
@@ -330,15 +333,27 @@ private[spark] class ApplicationMaster(
if (!NonFatal(e) || failureCount >= reporterMaxFailures) {
finish(FinalApplicationStatus.FAILED,
ApplicationMaster.EXIT_REPORTER_FAILURE, "Exception was
thrown " +
-s"${failureCount} time(s) from Reporter thread.")
-
+s"$failureCount time(s) from Reporter thread.")
} else {
-logWarning(s"Reporter thread fails ${failureCount} time(s) in
a row.", e)
+
spark git commit: [SPARK-7524] [SPARK-7846] add configs for keytab and principal, pass these two configs with different way in different modes
Repository: spark
Updated Branches:
refs/heads/master 8db40f671 -> a51b133de
[SPARK-7524] [SPARK-7846] add configs for keytab and principal, pass these two
configs with different way in different modes
* As spark now supports long running service by updating tokens for namenode,
but only accept parameters passed with "--k=v" format which is not very
convinient. This patch add spark.* configs in properties file and system
property.
* --principal and --keytabl options are passed to client but when we started
thrift server or spark-shell these two are also passed into the Main class
(org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 and
org.apache.spark.repl.Main).
In these two main class, arguments passed in will be processed with some 3rd
libraries, which will lead to some error: "Invalid option: --principal" or
"Unrecgnised option: --principal".
We should pass these command args in different forms, say system properties.
Author: WangTaoTheTonic
Closes #6051 from WangTaoTheTonic/SPARK-7524 and squashes the following commits:
e65699a [WangTaoTheTonic] change logic to loadEnvironments
ebd9ea0 [WangTaoTheTonic] merge master
ecfe43a [WangTaoTheTonic] pass keytab and principal seperately in different mode
33a7f40 [WangTaoTheTonic] expand the use of the current configs
08bb4e8 [WangTaoTheTonic] fix wrong cite
73afa64 [WangTaoTheTonic] add configs for keytab and principal, move originals
to internal
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a51b133d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a51b133d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a51b133d
Branch: refs/heads/master
Commit: a51b133de3c65a991ab105b6f020082080121b4c
Parents: 8db40f6
Author: WangTaoTheTonic
Authored: Fri May 29 11:06:11 2015 -0500
Committer: Thomas Graves
Committed: Fri May 29 11:06:11 2015 -0500
--
.../scala/org/apache/spark/deploy/SparkSubmit.scala | 8
.../apache/spark/deploy/SparkSubmitArguments.scala | 2 ++
docs/running-on-yarn.md | 16
.../deploy/yarn/AMDelegationTokenRenewer.scala | 14 --
.../apache/spark/deploy/yarn/ClientArguments.scala | 6 ++
5 files changed, 36 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a51b133d/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 92bb505..d1b32ea 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -428,6 +428,8 @@ object SparkSubmit {
OptionAssigner(args.executorCores, YARN, CLIENT, sysProp =
"spark.executor.cores"),
OptionAssigner(args.files, YARN, CLIENT, sysProp =
"spark.yarn.dist.files"),
OptionAssigner(args.archives, YARN, CLIENT, sysProp =
"spark.yarn.dist.archives"),
+ OptionAssigner(args.principal, YARN, CLIENT, sysProp =
"spark.yarn.principal"),
+ OptionAssigner(args.keytab, YARN, CLIENT, sysProp = "spark.yarn.keytab"),
// Yarn cluster only
OptionAssigner(args.name, YARN, CLUSTER, clOption = "--name"),
@@ -440,10 +442,8 @@ object SparkSubmit {
OptionAssigner(args.files, YARN, CLUSTER, clOption = "--files"),
OptionAssigner(args.archives, YARN, CLUSTER, clOption = "--archives"),
OptionAssigner(args.jars, YARN, CLUSTER, clOption = "--addJars"),
-
- // Yarn client or cluster
- OptionAssigner(args.principal, YARN, ALL_DEPLOY_MODES, clOption =
"--principal"),
- OptionAssigner(args.keytab, YARN, ALL_DEPLOY_MODES, clOption =
"--keytab"),
+ OptionAssigner(args.principal, YARN, CLUSTER, clOption = "--principal"),
+ OptionAssigner(args.keytab, YARN, CLUSTER, clOption = "--keytab"),
// Other options
OptionAssigner(args.executorCores, STANDALONE, ALL_DEPLOY_MODES,
http://git-wip-us.apache.org/repos/asf/spark/blob/a51b133d/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index c0e4c77..cc6a7bd 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -169,6 +169,8 @@ private[deploy] class SparkSubmitArguments(args:
Seq[String], env: Map[String, S
deployMode = Option(deployMode).orElse(env.get("DEPLOY_MODE")).orNull
nu
git commit: SPARK-1053. Don't require SPARK_YARN_APP_JAR
Repository: spark
Updated Branches:
refs/heads/master c852201ce -> b8a187195
SPARK-1053. Don't require SPARK_YARN_APP_JAR
It looks this just requires taking out the checks.
I verified that, with the patch, I was able to run spark-shell through yarn
without setting the environment variable.
Author: Sandy Ryza
Closes #553 from sryza/sandy-spark-1053 and squashes the following commits:
b037676 [Sandy Ryza] SPARK-1053. Don't require SPARK_YARN_APP_JAR
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b8a18719
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b8a18719
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b8a18719
Branch: refs/heads/master
Commit: b8a1871953058c67b49b7f8455cbb417d5b50ab6
Parents: c852201
Author: Sandy Ryza
Authored: Wed Feb 26 10:00:02 2014 -0600
Committer: Thomas Graves
Committed: Wed Feb 26 10:00:02 2014 -0600
--
docs/running-on-yarn.md| 6 ++
.../scala/org/apache/spark/deploy/yarn/ClientArguments.scala | 4 ++--
.../main/scala/org/apache/spark/deploy/yarn/ClientBase.scala | 3 ++-
.../spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 6 +-
4 files changed, 7 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b8a18719/docs/running-on-yarn.md
--
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index cd4509e..ee1d892 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -99,13 +99,12 @@ With this mode, your application is actually run on the
remote machine where the
## Launch spark application with yarn-client mode.
-With yarn-client mode, the application will be launched locally. Just like
running application or spark-shell on Local / Mesos / Standalone mode. The
launch method is also the similar with them, just make sure that when you need
to specify a master url, use "yarn-client" instead. And you also need to export
the env value for SPARK_JAR and SPARK_YARN_APP_JAR
+With yarn-client mode, the application will be launched locally. Just like
running application or spark-shell on Local / Mesos / Standalone mode. The
launch method is also the similar with them, just make sure that when you need
to specify a master url, use "yarn-client" instead. And you also need to export
the env value for SPARK_JAR.
Configuration in yarn-client mode:
In order to tune worker core/number/memory etc. You need to export environment
variables or add them to the spark configuration file (./conf/spark_env.sh).
The following are the list of options.
-* `SPARK_YARN_APP_JAR`, Path to your application's JAR file (required)
* `SPARK_WORKER_INSTANCES`, Number of workers to start (Default: 2)
* `SPARK_WORKER_CORES`, Number of cores for the workers (Default: 1).
* `SPARK_WORKER_MEMORY`, Memory per Worker (e.g. 1000M, 2G) (Default: 1G)
@@ -118,12 +117,11 @@ In order to tune worker core/number/memory etc. You need
to export environment v
For example:
SPARK_JAR=./assembly/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-assembly-{{site.SPARK_VERSION}}-hadoop2.0.5-alpha.jar
\
-
SPARK_YARN_APP_JAR=examples/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar
\
./bin/run-example org.apache.spark.examples.SparkPi yarn-client
+or
SPARK_JAR=./assembly/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-assembly-{{site.SPARK_VERSION}}-hadoop2.0.5-alpha.jar
\
-
SPARK_YARN_APP_JAR=examples/target/scala-{{site.SCALA_BINARY_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar
\
MASTER=yarn-client ./bin/spark-shell
http://git-wip-us.apache.org/repos/asf/spark/blob/b8a18719/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
index 1419f21..fe37168 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala
@@ -108,7 +108,7 @@ class ClientArguments(val args: Array[String], val
sparkConf: SparkConf) {
args = tail
case Nil =>
- if (userJar == null || userClass == null) {
+ if (userClass == null) {
printUsageAndExit(1)
}
@@ -129,7 +129,7 @@ class ClientArguments(val args: Array[String], val
sparkConf: SparkConf) {
System.err.println(
"Usage: org.apache.spark.deploy.yarn.Client [options] \n" +
"Options:\n" +
- " --jar
git commit: SPARK-1032. If Yarn app fails before registering, app master stays aroun...
Repository: spark
Updated Branches:
refs/heads/master edf8a56ab -> 5f419bf9f
SPARK-1032. If Yarn app fails before registering, app master stays aroun...
...d long after
This reopens https://github.com/apache/incubator-spark/pull/648 against the new
repo.
Author: Sandy Ryza
Closes #28 from sryza/sandy-spark-1032 and squashes the following commits:
5953f50 [Sandy Ryza] SPARK-1032. If Yarn app fails before registering, app
master stays around long after
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5f419bf9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5f419bf9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5f419bf9
Branch: refs/heads/master
Commit: 5f419bf9f433e8f057237f1d5bfed9f5f4e9427c
Parents: edf8a56
Author: Sandy Ryza
Authored: Fri Feb 28 09:40:47 2014 -0600
Committer: Thomas Graves
Committed: Fri Feb 28 09:40:47 2014 -0600
--
.../spark/deploy/yarn/ApplicationMaster.scala | 34 +---
.../spark/deploy/yarn/ApplicationMaster.scala | 22 +
2 files changed, 38 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5f419bf9/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 2e46d75..560e5de 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -65,6 +65,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
private val maxNumWorkerFailures =
sparkConf.getInt("spark.yarn.max.worker.failures",
math.max(args.numWorkers * 2, 3))
+ private var registered = false
+
def run() {
// Setup the directories so things go to yarn approved directories rather
// then user specified and /tmp.
@@ -110,7 +112,12 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
waitForSparkContextInitialized()
// Do this after spark master is up and SparkContext is created so that we
can register UI Url
-val appMasterResponse: RegisterApplicationMasterResponse =
registerApplicationMaster()
+synchronized {
+ if (!isFinished) {
+registerApplicationMaster()
+registered = true
+ }
+}
// Allocate all containers
allocateWorkers()
@@ -208,7 +215,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
var count = 0
val waitTime = 1L
val numTries =
sparkConf.getInt("spark.yarn.ApplicationMaster.waitTries", 10)
-while (ApplicationMaster.sparkContextRef.get() == null && count <
numTries) {
+while (ApplicationMaster.sparkContextRef.get() == null && count <
numTries
+&& !isFinished) {
logInfo("Waiting for spark context initialization ... " + count)
count = count + 1
ApplicationMaster.sparkContextRef.wait(waitTime)
@@ -341,17 +349,19 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
return
}
isFinished = true
+
+ logInfo("finishApplicationMaster with " + status)
+ if (registered) {
+val finishReq =
Records.newRecord(classOf[FinishApplicationMasterRequest])
+ .asInstanceOf[FinishApplicationMasterRequest]
+finishReq.setAppAttemptId(appAttemptId)
+finishReq.setFinishApplicationStatus(status)
+finishReq.setDiagnostics(diagnostics)
+// Set tracking url to empty since we don't have a history server.
+finishReq.setTrackingUrl("")
+resourceManager.finishApplicationMaster(finishReq)
+ }
}
-
-logInfo("finishApplicationMaster with " + status)
-val finishReq = Records.newRecord(classOf[FinishApplicationMasterRequest])
- .asInstanceOf[FinishApplicationMasterRequest]
-finishReq.setAppAttemptId(appAttemptId)
-finishReq.setFinishApplicationStatus(status)
-finishReq.setDiagnostics(diagnostics)
-// Set tracking url to empty since we don't have a history server.
-finishReq.setTrackingUrl("")
-resourceManager.finishApplicationMaster(finishReq)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/5f419bf9/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Appl
git commit: SPARK-1051. On YARN, executors don't doAs submitting user
Repository: spark
Updated Branches:
refs/heads/master 5f419bf9f -> 46dff3445
SPARK-1051. On YARN, executors don't doAs submitting user
This reopens https://github.com/apache/incubator-spark/pull/538 against the new
repo
Author: Sandy Ryza
Closes #29 from sryza/sandy-spark-1051 and squashes the following commits:
708ce49 [Sandy Ryza] SPARK-1051. doAs submitting user in YARN
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/46dff344
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/46dff344
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/46dff344
Branch: refs/heads/master
Commit: 46dff34458096e5330073ca58e0723da52aeddcd
Parents: 5f419bf
Author: Sandy Ryza
Authored: Fri Feb 28 12:43:01 2014 -0600
Committer: Thomas Graves
Committed: Fri Feb 28 12:43:01 2014 -0600
--
.../org/apache/spark/deploy/SparkHadoopUtil.scala | 18 ++
.../spark/deploy/yarn/ApplicationMaster.scala | 6 +-
.../org/apache/spark/deploy/yarn/ClientBase.scala | 1 +
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 4
.../spark/deploy/yarn/ApplicationMaster.scala | 6 +-
5 files changed, 25 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/46dff344/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index b479225..ec15647 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -25,6 +25,8 @@ import org.apache.hadoop.security.UserGroupInformation
import org.apache.spark.{SparkContext, SparkException}
+import scala.collection.JavaConversions._
+
/**
* Contains util methods to interact with Hadoop from Spark.
*/
@@ -33,15 +35,9 @@ class SparkHadoopUtil {
UserGroupInformation.setConfiguration(conf)
def runAsUser(user: String)(func: () => Unit) {
-// if we are already running as the user intended there is no reason to do
the doAs. It
-// will actually break secure HDFS access as it doesn't fill in the
credentials. Also if
-// the user is UNKNOWN then we shouldn't be creating a remote unknown user
-// (this is actually the path spark on yarn takes) since SPARK_USER is
initialized only
-// in SparkContext.
-val currentUser = Option(System.getProperty("user.name")).
- getOrElse(SparkContext.SPARK_UNKNOWN_USER)
-if (user != SparkContext.SPARK_UNKNOWN_USER && currentUser != user) {
+if (user != SparkContext.SPARK_UNKNOWN_USER) {
val ugi = UserGroupInformation.createRemoteUser(user)
+ transferCredentials(UserGroupInformation.getCurrentUser(), ugi)
ugi.doAs(new PrivilegedExceptionAction[Unit] {
def run: Unit = func()
})
@@ -50,6 +46,12 @@ class SparkHadoopUtil {
}
}
+ def transferCredentials(source: UserGroupInformation, dest:
UserGroupInformation) {
+for (token <- source.getTokens()) {
+ dest.addToken(token)
+}
+ }
+
/**
* Return an appropriate (subclass) of Configuration. Creating config can
initializes some Hadoop
* subsystems.
http://git-wip-us.apache.org/repos/asf/spark/blob/46dff344/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 560e5de..e045b9f 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
import org.apache.spark.{SparkConf, SparkContext, Logging}
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.util.Utils
class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
@@ -67,6 +68,9 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
private var registered = false
+ private val sparkUser = Option(System.getenv("SPARK_USER")).getOrElse(
+SparkContext.SPARK_UNKNOWN_USER)
+
def run() {
// Setup the directories so things go to yarn approved directories rather
// then user specified and /tmp.
@@ -180,7 +184,7 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
false /* initialize */ ,
Thread.currentThread.getContextC
[1/3] SPARK-1189: Add Security to Spark - Akka, Http, ConnectionManager, UI use servlets
Repository: spark
Updated Branches:
refs/heads/master 40566e10a -> 7edbea41b
http://git-wip-us.apache.org/repos/asf/spark/blob/7edbea41/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
b/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
index c1e8b29..96a5a12 100644
--- a/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
@@ -18,21 +18,22 @@
package org.apache.spark.metrics
import org.scalatest.{BeforeAndAfter, FunSuite}
-
-import org.apache.spark.SparkConf
+import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.master.MasterSource
class MetricsSystemSuite extends FunSuite with BeforeAndAfter {
var filePath: String = _
var conf: SparkConf = null
+ var securityMgr: SecurityManager = null
before {
filePath =
getClass.getClassLoader.getResource("test_metrics_system.properties").getFile()
conf = new SparkConf(false).set("spark.metrics.conf", filePath)
+securityMgr = new SecurityManager(conf)
}
test("MetricsSystem with default config") {
-val metricsSystem = MetricsSystem.createMetricsSystem("default", conf)
+val metricsSystem = MetricsSystem.createMetricsSystem("default", conf,
securityMgr)
val sources = metricsSystem.sources
val sinks = metricsSystem.sinks
@@ -42,7 +43,7 @@ class MetricsSystemSuite extends FunSuite with BeforeAndAfter
{
}
test("MetricsSystem with sources add") {
-val metricsSystem = MetricsSystem.createMetricsSystem("test", conf)
+val metricsSystem = MetricsSystem.createMetricsSystem("test", conf,
securityMgr)
val sources = metricsSystem.sources
val sinks = metricsSystem.sinks
http://git-wip-us.apache.org/repos/asf/spark/blob/7edbea41/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 9f011d9..121e47c 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -28,7 +28,7 @@ import org.scalatest.concurrent.Timeouts._
import org.scalatest.matchers.ShouldMatchers._
import org.scalatest.time.SpanSugar._
-import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
import org.apache.spark.util.{AkkaUtils, ByteBufferInputStream, SizeEstimator,
Utils}
@@ -39,6 +39,8 @@ class BlockManagerSuite extends FunSuite with BeforeAndAfter
with PrivateMethodT
var actorSystem: ActorSystem = null
var master: BlockManagerMaster = null
var oldArch: String = null
+ conf.set("spark.authenticate", "false")
+ val securityMgr = new SecurityManager(conf)
// Reuse a serializer across tests to avoid creating a new thread-local
buffer on each test
conf.set("spark.kryoserializer.buffer.mb", "1")
@@ -49,7 +51,8 @@ class BlockManagerSuite extends FunSuite with BeforeAndAfter
with PrivateMethodT
def rdd(rddId: Int, splitId: Int) = RDDBlockId(rddId, splitId)
before {
-val (actorSystem, boundPort) = AkkaUtils.createActorSystem("test",
"localhost", 0, conf = conf)
+val (actorSystem, boundPort) = AkkaUtils.createActorSystem("test",
"localhost", 0, conf = conf,
+ securityManager = securityMgr)
this.actorSystem = actorSystem
conf.set("spark.driver.port", boundPort.toString)
@@ -125,7 +128,7 @@ class BlockManagerSuite extends FunSuite with
BeforeAndAfter with PrivateMethodT
}
test("master + 1 manager interaction") {
-store = new BlockManager("", actorSystem, master, serializer,
2000, conf)
+store = new BlockManager("", actorSystem, master, serializer,
2000, conf, securityMgr)
val a1 = new Array[Byte](400)
val a2 = new Array[Byte](400)
val a3 = new Array[Byte](400)
@@ -155,8 +158,9 @@ class BlockManagerSuite extends FunSuite with
BeforeAndAfter with PrivateMethodT
}
test("master + 2 managers interaction") {
-store = new BlockManager("exec1", actorSystem, master, serializer, 2000,
conf)
-store2 = new BlockManager("exec2", actorSystem, master, new
KryoSerializer(conf), 2000, conf)
+store = new BlockManager("exec1", actorSystem, master, serializer, 2000,
conf, securityMgr)
+store2 = new BlockManager("exec2", actorSystem, master, new
KryoSerializer(conf), 2000, conf,
+ securityMgr)
val peers = master.getPeers(store.blockManagerId, 1)
assert(peers.size === 1, "master did not return the other manager as a
peer")
@@ -1
git commit: SPARK-1203 fix saving to hdfs from yarn
Repository: spark
Updated Branches:
refs/heads/master d55ec86de -> 6112270c9
SPARK-1203 fix saving to hdfs from yarn
Author: Thomas Graves
Closes #173 from tgravescs/SPARK-1203 and squashes the following commits:
4fd5ded [Thomas Graves] adding import
964e3f7 [Thomas Graves] SPARK-1203 fix saving to hdfs from yarn
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6112270c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6112270c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6112270c
Branch: refs/heads/master
Commit: 6112270c94f1a30a461a91f6e56485a5eaec2606
Parents: d55ec86
Author: Thomas Graves
Authored: Wed Mar 19 08:09:20 2014 -0500
Committer: Thomas Graves
Committed: Wed Mar 19 08:09:20 2014 -0500
--
core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6112270c/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
--
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index 447deaf..75fc02a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -41,6 +41,7 @@ import
org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat => NewFileOutput
import org.apache.hadoop.mapred.SparkHadoopWriter
import org.apache.spark._
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.Partitioner.defaultPartitioner
import org.apache.spark.SparkContext._
import org.apache.spark.partial.{BoundedDouble, PartialResult}
@@ -723,6 +724,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self:
RDD[(K, V)])
if (valueClass == null) {
throw new SparkException("Output value class not set")
}
+SparkHadoopUtil.get.addCredentials(conf)
logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", "
+
valueClass.getSimpleName + ")")
git commit: SPARK-1203 fix saving to hdfs from yarn
Repository: spark
Updated Branches:
refs/heads/branch-0.9 d385b5a19 -> 250ec271c
SPARK-1203 fix saving to hdfs from yarn
Author: Thomas Graves
Closes #173 from tgravescs/SPARK-1203 and squashes the following commits:
4fd5ded [Thomas Graves] adding import
964e3f7 [Thomas Graves] SPARK-1203 fix saving to hdfs from yarn
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/250ec271
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/250ec271
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/250ec271
Branch: refs/heads/branch-0.9
Commit: 250ec271c6136b881f7ffcb9ce746c30d4abbf3c
Parents: d385b5a
Author: Thomas Graves
Authored: Wed Mar 19 08:09:20 2014 -0500
Committer: Thomas Graves
Committed: Wed Mar 19 08:19:47 2014 -0500
--
core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/250ec271/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
--
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index 0b2917b..0bc09b3 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -44,6 +44,7 @@ import
com.clearspring.analytics.stream.cardinality.HyperLogLog
import org.apache.hadoop.mapred.SparkHadoopWriter
import org.apache.hadoop.mapreduce.SparkHadoopMapReduceUtil
import org.apache.spark._
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.SparkContext._
import org.apache.spark.partial.{BoundedDouble, PartialResult}
import org.apache.spark.Partitioner.defaultPartitioner
@@ -710,6 +711,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self:
RDD[(K, V)])
if (valueClass == null) {
throw new SparkException("Output value class not set")
}
+SparkHadoopUtil.get.addCredentials(conf)
logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", "
+
valueClass.getSimpleName+ ")")
git commit: Update the yarn alpha version to 0.9.1-incubating-SNAPSHOT
Repository: spark
Updated Branches:
refs/heads/branch-0.9 250ec271c -> 72875b29b
Update the yarn alpha version to 0.9.1-incubating-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/72875b29
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/72875b29
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/72875b29
Branch: refs/heads/branch-0.9
Commit: 72875b29bb69f654f1dc33e50cc6431a1cfbc0a0
Parents: 250ec27
Author: Thomas Graves
Authored: Wed Mar 19 12:41:11 2014 -0500
Committer: Thomas Graves
Committed: Wed Mar 19 12:41:11 2014 -0500
--
dev/merge_spark_pr.py | 2 ++
yarn/alpha/pom.xml| 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/72875b29/dev/merge_spark_pr.py
--
diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py
index e8f78fc..e40a3b2 100755
--- a/dev/merge_spark_pr.py
+++ b/dev/merge_spark_pr.py
@@ -38,6 +38,8 @@ PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME",
"apache-github")
# Remote name which points to Apache git
PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache")
+print "%s" % PUSH_REMOTE_NAME
+
GIT_API_BASE = "https://api.github.com/repos/apache/spark";
# Prefix added to temporary branches
BRANCH_PREFIX = "PR_TOOL"
http://git-wip-us.apache.org/repos/asf/spark/blob/72875b29/yarn/alpha/pom.xml
--
diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml
index 349c835..8d60503 100644
--- a/yarn/alpha/pom.xml
+++ b/yarn/alpha/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
yarn-parent_2.10
-0.9.0-incubating
+0.9.1-incubating-SNAPSHOT
../pom.xml
git commit: SPARK-1051. On YARN, executors don't doAs submitting user
Repository: spark
Updated Branches:
refs/heads/branch-0.9 1e366904c -> 748f002b3
SPARK-1051. On YARN, executors don't doAs submitting user
This reopens https://github.com/apache/incubator-spark/pull/538 against the new
repo
Author: Sandy Ryza
Closes #29 from sryza/sandy-spark-1051 and squashes the following commits:
708ce49 [Sandy Ryza] SPARK-1051. doAs submitting user in YARN
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/748f002b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/748f002b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/748f002b
Branch: refs/heads/branch-0.9
Commit: 748f002b3a58af118f7de0b6ea9170895b571c78
Parents: 1e36690
Author: Sandy Ryza
Authored: Fri Feb 28 12:43:01 2014 -0600
Committer: Thomas Graves
Committed: Thu Mar 20 14:48:05 2014 -0500
--
.../org/apache/spark/deploy/SparkHadoopUtil.scala | 18 ++
.../spark/deploy/yarn/ApplicationMaster.scala | 6 +-
.../org/apache/spark/deploy/yarn/Client.scala | 1 +
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 4
.../spark/deploy/yarn/ApplicationMaster.scala | 6 +-
.../org/apache/spark/deploy/yarn/Client.scala | 1 +
6 files changed, 26 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/748f002b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index b479225..ec15647 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -25,6 +25,8 @@ import org.apache.hadoop.security.UserGroupInformation
import org.apache.spark.{SparkContext, SparkException}
+import scala.collection.JavaConversions._
+
/**
* Contains util methods to interact with Hadoop from Spark.
*/
@@ -33,15 +35,9 @@ class SparkHadoopUtil {
UserGroupInformation.setConfiguration(conf)
def runAsUser(user: String)(func: () => Unit) {
-// if we are already running as the user intended there is no reason to do
the doAs. It
-// will actually break secure HDFS access as it doesn't fill in the
credentials. Also if
-// the user is UNKNOWN then we shouldn't be creating a remote unknown user
-// (this is actually the path spark on yarn takes) since SPARK_USER is
initialized only
-// in SparkContext.
-val currentUser = Option(System.getProperty("user.name")).
- getOrElse(SparkContext.SPARK_UNKNOWN_USER)
-if (user != SparkContext.SPARK_UNKNOWN_USER && currentUser != user) {
+if (user != SparkContext.SPARK_UNKNOWN_USER) {
val ugi = UserGroupInformation.createRemoteUser(user)
+ transferCredentials(UserGroupInformation.getCurrentUser(), ugi)
ugi.doAs(new PrivilegedExceptionAction[Unit] {
def run: Unit = func()
})
@@ -50,6 +46,12 @@ class SparkHadoopUtil {
}
}
+ def transferCredentials(source: UserGroupInformation, dest:
UserGroupInformation) {
+for (token <- source.getTokens()) {
+ dest.addToken(token)
+}
+ }
+
/**
* Return an appropriate (subclass) of Configuration. Creating config can
initializes some Hadoop
* subsystems.
http://git-wip-us.apache.org/repos/asf/spark/blob/748f002b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 2e46d75..7aa1894 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
import org.apache.spark.{SparkConf, SparkContext, Logging}
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.util.Utils
class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
@@ -65,6 +66,9 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
private val maxNumWorkerFailures =
sparkConf.getInt("spark.yarn.max.worker.failures",
math.max(args.numWorkers * 2, 3))
+ private val sparkUser = Option(System.getenv("SPARK_USER")).getOrElse(
+SparkContext.SPARK_UNKNOWN_USER)
+
def run() {
// Setup the directories so things go to yarn approved directories rather
// then user specified and /tmp.
@@ -173,7 +1
git commit: SPARK-1032. If Yarn app fails before registering, app master stays aroun...
Repository: spark
Updated Branches:
refs/heads/branch-0.9 748f002b3 -> c6630d363
SPARK-1032. If Yarn app fails before registering, app master stays aroun...
...d long after
This reopens https://github.com/apache/incubator-spark/pull/648 against the new
repo.
Author: Sandy Ryza
Closes #28 from sryza/sandy-spark-1032 and squashes the following commits:
5953f50 [Sandy Ryza] SPARK-1032. If Yarn app fails before registering, app
master stays around long after
Conflicts:
yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c6630d36
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c6630d36
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c6630d36
Branch: refs/heads/branch-0.9
Commit: c6630d363a73184c8dcca9f2c0c6fc3f5c8e47bf
Parents: 748f002
Author: Sandy Ryza
Authored: Fri Feb 28 09:40:47 2014 -0600
Committer: Thomas Graves
Committed: Thu Mar 20 16:50:44 2014 -0500
--
.../spark/deploy/yarn/ApplicationMaster.scala | 34 +---
.../spark/deploy/yarn/ApplicationMaster.scala | 22 +
2 files changed, 38 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c6630d36/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 7aa1894..e045b9f 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -66,6 +66,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
private val maxNumWorkerFailures =
sparkConf.getInt("spark.yarn.max.worker.failures",
math.max(args.numWorkers * 2, 3))
+ private var registered = false
+
private val sparkUser = Option(System.getenv("SPARK_USER")).getOrElse(
SparkContext.SPARK_UNKNOWN_USER)
@@ -114,7 +116,12 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
waitForSparkContextInitialized()
// Do this after spark master is up and SparkContext is created so that we
can register UI Url
-val appMasterResponse: RegisterApplicationMasterResponse =
registerApplicationMaster()
+synchronized {
+ if (!isFinished) {
+registerApplicationMaster()
+registered = true
+ }
+}
// Allocate all containers
allocateWorkers()
@@ -212,7 +219,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
var count = 0
val waitTime = 1L
val numTries =
sparkConf.getInt("spark.yarn.ApplicationMaster.waitTries", 10)
-while (ApplicationMaster.sparkContextRef.get() == null && count <
numTries) {
+while (ApplicationMaster.sparkContextRef.get() == null && count <
numTries
+&& !isFinished) {
logInfo("Waiting for spark context initialization ... " + count)
count = count + 1
ApplicationMaster.sparkContextRef.wait(waitTime)
@@ -345,17 +353,19 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
return
}
isFinished = true
+
+ logInfo("finishApplicationMaster with " + status)
+ if (registered) {
+val finishReq =
Records.newRecord(classOf[FinishApplicationMasterRequest])
+ .asInstanceOf[FinishApplicationMasterRequest]
+finishReq.setAppAttemptId(appAttemptId)
+finishReq.setFinishApplicationStatus(status)
+finishReq.setDiagnostics(diagnostics)
+// Set tracking url to empty since we don't have a history server.
+finishReq.setTrackingUrl("")
+resourceManager.finishApplicationMaster(finishReq)
+ }
}
-
-logInfo("finishApplicationMaster with " + status)
-val finishReq = Records.newRecord(classOf[FinishApplicationMasterRequest])
- .asInstanceOf[FinishApplicationMasterRequest]
-finishReq.setAppAttemptId(appAttemptId)
-finishReq.setFinishApplicationStatus(status)
-finishReq.setDiagnostics(diagnostics)
-// Set tracking url to empty since we don't have a history server.
-finishReq.setTrackingUrl("")
-resourceManager.finishApplicationMaster(finishReq)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/c6630d36/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
git commit: SPARK-1330 removed extra echo from comput_classpath.sh
Repository: spark
Updated Branches:
refs/heads/master 5b2d863e3 -> 426042ad2
SPARK-1330 removed extra echo from comput_classpath.sh
remove the extra echo which prevents spark-class from working. Note that I did
not update the comment above it, which is also wrong because I'm not sure what
it should do.
Should hive only be included if explicitly built with sbt hive/assembly or
should sbt assembly build it?
Author: Thomas Graves
Closes #241 from tgravescs/SPARK-1330 and squashes the following commits:
b10d708 [Thomas Graves] SPARK-1330 removed extra echo from comput_classpath.sh
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/426042ad
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/426042ad
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/426042ad
Branch: refs/heads/master
Commit: 426042ad24a54b4b776085cbf4e1896464efc613
Parents: 5b2d863
Author: Thomas Graves
Authored: Thu Mar 27 11:54:43 2014 -0500
Committer: Thomas Graves
Committed: Thu Mar 27 11:54:43 2014 -0500
--
bin/compute-classpath.sh | 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/426042ad/bin/compute-classpath.sh
--
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index d6f1ff9..bef42df 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -36,7 +36,6 @@ CLASSPATH="$SPARK_CLASSPATH:$FWDIR/conf"
# Hopefully we will find a way to avoid uber-jars entirely and deploy only the
needed packages in
# the future.
if [ -f
"$FWDIR"/sql/hive/target/scala-$SCALA_VERSION/spark-hive-assembly-*.jar ]; then
- echo "Hive assembly found, including hive support. If this isn't desired
run sbt hive/clean."
# Datanucleus jars do not work if only included in the uberjar as plugin.xml
metadata is lost.
DATANUCLEUSJARS=$(JARS=("$FWDIR/lib_managed/jars"/datanucleus-*.jar); IFS=:;
echo "${JARS[*]}")
git commit: SPARK-1350. Always use JAVA_HOME to run executor container JVMs.
Repository: spark
Updated Branches:
refs/heads/master ee6e9e7d8 -> 7f32fd42a
SPARK-1350. Always use JAVA_HOME to run executor container JVMs.
Author: Sandy Ryza
Closes #313 from sryza/sandy-spark-1350 and squashes the following commits:
bb6d187 [Sandy Ryza] SPARK-1350. Always use JAVA_HOME to run executor container
JVMs.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7f32fd42
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7f32fd42
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7f32fd42
Branch: refs/heads/master
Commit: 7f32fd42aaadcf6626b4d8bcf6295203b43b2037
Parents: ee6e9e7
Author: Sandy Ryza
Authored: Fri Apr 4 08:54:04 2014 -0500
Committer: Thomas Graves
Committed: Fri Apr 4 08:54:04 2014 -0500
--
.../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 8 +---
.../org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala | 9 ++---
2 files changed, 3 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/7f32fd42/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 6568003..bc26790 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -345,14 +345,8 @@ trait ClientBase extends Logging {
}
// Command for the ApplicationMaster
-var javaCommand = "java"
-val javaHome = System.getenv("JAVA_HOME")
-if ((javaHome != null && !javaHome.isEmpty()) ||
env.isDefinedAt("JAVA_HOME")) {
- javaCommand = Environment.JAVA_HOME.$() + "/bin/java"
-}
-
val commands = List[String](
- javaCommand +
+ Environment.JAVA_HOME.$() + "/bin/java" +
" -server " +
JAVA_OPTS +
" " + args.amClass +
http://git-wip-us.apache.org/repos/asf/spark/blob/7f32fd42/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
index da0a6f7..2079697 100644
---
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
+++
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
@@ -88,13 +88,8 @@ trait ExecutorRunnableUtil extends Logging {
}
*/
-var javaCommand = "java"
-val javaHome = System.getenv("JAVA_HOME")
-if ((javaHome != null && !javaHome.isEmpty()) ||
env.isDefinedAt("JAVA_HOME")) {
- javaCommand = Environment.JAVA_HOME.$() + "/bin/java"
-}
-
-val commands = List[String](javaCommand +
+val commands = List[String](
+ Environment.JAVA_HOME.$() + "/bin/java" +
" -server " +
// Kill if OOM is raised - leverage yarn's failure handling to cause
rescheduling.
// Not killing the task leaves various aspects of the executor and (to
some extent) the jvm in
git commit: SPARK-1252. On YARN, use container-log4j.properties for executors
Repository: spark
Updated Branches:
refs/heads/master 83f2a2f14 -> 9dd8b9166
SPARK-1252. On YARN, use container-log4j.properties for executors
container-log4j.properties is a file that YARN provides so that containers can
have log4j.properties distinct from that of the NodeManagers.
Logs now go to syslog, and stderr and stdout just have the process's standard
err and standard out.
I tested this on pseudo-distributed clusters for both yarn (Hadoop 2.2) and
yarn-alpha (Hadoop 0.23.7)/
Author: Sandy Ryza
Closes #148 from sryza/sandy-spark-1252 and squashes the following commits:
c0043b8 [Sandy Ryza] Put log4j.properties file under common
55823da [Sandy Ryza] Add license headers to new files
10934b8 [Sandy Ryza] Add log4j-spark-container.properties and support
SPARK_LOG4J_CONF
e74450b [Sandy Ryza] SPARK-1252. On YARN, use container-log4j.properties for
executors
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9dd8b916
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9dd8b916
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9dd8b916
Branch: refs/heads/master
Commit: 9dd8b9166225cbcfcab1f39268244c2feb42a658
Parents: 83f2a2f
Author: Sandy Ryza
Authored: Mon Apr 7 13:28:14 2014 -0500
Committer: Thomas Graves
Committed: Mon Apr 7 13:28:14 2014 -0500
--
.../spark/deploy/yarn/ExecutorRunnable.scala| 3 ++-
.../resources/log4j-spark-container.properties | 24
.../apache/spark/deploy/yarn/ClientBase.scala | 8 +--
.../deploy/yarn/ExecutorRunnableUtil.scala | 7 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 7 ++
yarn/pom.xml| 6 +
.../spark/deploy/yarn/ExecutorRunnable.scala| 3 ++-
7 files changed, 53 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9dd8b916/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 981e8b0..3469b7d 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -81,7 +81,8 @@ class ExecutorRunnable(
credentials.writeTokenStorageToStream(dob)
ctx.setContainerTokens(ByteBuffer.wrap(dob.getData()))
-val commands = prepareCommand(masterAddress, slaveId, hostname,
executorMemory, executorCores)
+val commands = prepareCommand(masterAddress, slaveId, hostname,
executorMemory, executorCores,
+ localResources.contains(ClientBase.LOG4J_PROP))
logInfo("Setting up executor with commands: " + commands)
ctx.setCommands(commands)
http://git-wip-us.apache.org/repos/asf/spark/blob/9dd8b916/yarn/common/src/main/resources/log4j-spark-container.properties
--
diff --git a/yarn/common/src/main/resources/log4j-spark-container.properties
b/yarn/common/src/main/resources/log4j-spark-container.properties
new file mode 100644
index 000..a1e37a0
--- /dev/null
+++ b/yarn/common/src/main/resources/log4j-spark-container.properties
@@ -0,0 +1,24 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p
%c{1}: %m%n
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.eclipse.jetty=WARN
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
http://git-wip-us.apache.org/repos/asf/spark/blob/9dd8b916/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/Client
git commit: SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken
Repository: spark
Updated Branches:
refs/heads/branch-1.0 a1bb4c695 -> 9e90c464d
SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken
Author: Thomas Graves
Closes #344 from tgravescs/SPARK-1417 and squashes the following commits:
c450b5f [Thomas Graves] fix test
e1c1d7e [Thomas Graves] add missing $ to appUIAddress
e982ddb [Thomas Graves] use appUIHostPort in appUIAddress
0803ec2 [Thomas Graves] Review comment updates - remove extra newline, simplify
assert in test
658a8ec [Thomas Graves] Add a appUIHostPort routine
0614208 [Thomas Graves] Fix test
2a6b1b7 [Thomas Graves] SPARK-1417: Spark on Yarn - spark UI link from
resourcemanager is broken
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9e90c464
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9e90c464
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9e90c464
Branch: refs/heads/branch-1.0
Commit: 9e90c464dec3258b57ecad828d7a8609eb5528aa
Parents: a1bb4c6
Author: Thomas Graves
Authored: Fri Apr 11 13:17:48 2014 +0530
Committer: Thomas Graves
Committed: Fri Apr 11 10:27:37 2014 -0500
--
.../scala/org/apache/spark/ui/SparkUI.scala | 7 +++-
.../scala/org/apache/spark/SparkUISuite.scala | 35
.../spark/deploy/yarn/ApplicationMaster.scala | 2 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 2 +-
4 files changed, 43 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9e90c464/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
--
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index 4c891d7..7fa4fd3 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -113,7 +113,12 @@ private[spark] class SparkUI(
logInfo("Stopped Spark Web UI at %s".format(appUIAddress))
}
- private[spark] def appUIAddress = "http://"; + publicHost + ":" + boundPort
+ /**
+ * Return the application UI host:port. This does not include the scheme
(http://).
+ */
+ private[spark] def appUIHostPort = publicHost + ":" + boundPort
+
+ private[spark] def appUIAddress = s"http://$appUIHostPort";
}
http://git-wip-us.apache.org/repos/asf/spark/blob/9e90c464/core/src/test/scala/org/apache/spark/SparkUISuite.scala
--
diff --git a/core/src/test/scala/org/apache/spark/SparkUISuite.scala
b/core/src/test/scala/org/apache/spark/SparkUISuite.scala
new file mode 100644
index 000..d0d119c
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/SparkUISuite.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.net.URI
+
+import org.scalatest.FunSuite
+
+class SparkUISuite extends FunSuite with SharedSparkContext {
+
+ test("verify appUIAddress contains the scheme") {
+val uiAddress = sc.ui.appUIAddress
+assert(uiAddress.equals("http://"; + sc.ui.appUIHostPort))
+ }
+
+ test("verify appUIAddress contains the port") {
+val splitUIAddress = sc.ui.appUIAddress.split(':')
+assert(splitUIAddress(2).toInt == sc.ui.boundPort)
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/9e90c464/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 910484e..67ec95c 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -234,7 +234,7 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
assert(sparkContext != null || c
git commit: SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release
Repository: spark
Updated Branches:
refs/heads/master e269c24db -> 725925cf2
SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release
YARN-1824 changes the APIs (addToEnvironment, setEnvFromInputString) in Apps,
which causes the spark build to break if built against a version 2.4.0. To fix
this, create the spark own function to do that functionality which will not
break compiling against 2.3 and other 2.x versions.
Author: xuan
Author: xuan
Closes #396 from xgong/master and squashes the following commits:
42b5984 [xuan] Remove two extra imports
bc0926f [xuan] Remove usage of org.apache.hadoop.util.Shell
be89fa7 [xuan] fix Spark compilation is broken with the latest hadoop-2.4.0
release
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/725925cf
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/725925cf
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/725925cf
Branch: refs/heads/master
Commit: 725925cf2120e998651f7d1406fdb34fc2405b9f
Parents: e269c24
Author: xuan
Authored: Wed Apr 16 14:41:22 2014 -0500
Committer: Thomas Graves
Committed: Wed Apr 16 14:41:22 2014 -0500
--
.../apache/spark/deploy/yarn/ClientBase.scala | 33 +-
.../deploy/yarn/ExecutorRunnableUtil.scala | 4 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 63
3 files changed, 85 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/725925cf/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index eb42922..628dd98 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -17,6 +17,7 @@
package org.apache.spark.deploy.yarn
+import java.io.File
import java.net.{InetAddress, UnknownHostException, URI}
import java.nio.ByteBuffer
@@ -280,7 +281,8 @@ trait ClientBase extends Logging {
distCacheMgr.setDistArchivesEnv(env)
// Allow users to specify some environment variables.
-Apps.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV"))
+YarnSparkHadoopUtil.setEnvFromInputString(env,
System.getenv("SPARK_YARN_USER_ENV"),
+ File.pathSeparator)
// Add each SPARK_* key to the environment.
System.getenv().filterKeys(_.startsWith("SPARK")).foreach { case (k,v) =>
env(k) = v }
@@ -382,7 +384,8 @@ object ClientBase {
YarnConfiguration.YARN_APPLICATION_CLASSPATH)).getOrElse(
getDefaultYarnApplicationClasspath())
for (c <- classpathEntries) {
- Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim)
+ YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
c.trim,
+File.pathSeparator)
}
val mrClasspathEntries = Option(conf.getStrings(
@@ -390,7 +393,8 @@ object ClientBase {
getDefaultMRApplicationClasspath())
if (mrClasspathEntries != null) {
for (c <- mrClasspathEntries) {
-Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim)
+YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
c.trim,
+ File.pathSeparator)
}
}
}
@@ -425,28 +429,29 @@ object ClientBase {
}
def populateClasspath(conf: Configuration, sparkConf: SparkConf, addLog4j:
Boolean, env: HashMap[String, String]) {
-Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$())
+YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$(),
+ File.pathSeparator)
// If log4j present, ensure ours overrides all others
if (addLog4j) {
- Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
-Path.SEPARATOR + LOG4J_PROP)
+ YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
+Path.SEPARATOR + LOG4J_PROP, File.pathSeparator)
}
// Normally the users app.jar is last in case conflicts with spark jars
val userClasspathFirst = sparkConf.get("spark.yarn.user.classpath.first",
"false")
.toBoolean
if (userClasspathFirst) {
- Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
-Path.SEPARATOR + APP_JAR)
+ YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
+Path.SEPARATOR + APP_JAR, File.pathSeparator)
}
-Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$()
+
- Path.SEPARATOR + SPARK_JAR)
+YarnSparkH
git commit: SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release
Repository: spark
Updated Branches:
refs/heads/branch-1.0 b75301f1f -> d8fc4a4b6
SPARK-1465: Spark compilation is broken with the latest hadoop-2.4.0 release
YARN-1824 changes the APIs (addToEnvironment, setEnvFromInputString) in Apps,
which causes the spark build to break if built against a version 2.4.0. To fix
this, create the spark own function to do that functionality which will not
break compiling against 2.3 and other 2.x versions.
Author: xuan
Author: xuan
Closes #396 from xgong/master and squashes the following commits:
42b5984 [xuan] Remove two extra imports
bc0926f [xuan] Remove usage of org.apache.hadoop.util.Shell
be89fa7 [xuan] fix Spark compilation is broken with the latest hadoop-2.4.0
release
(cherry picked from commit 725925cf2120e998651f7d1406fdb34fc2405b9f)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d8fc4a4b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d8fc4a4b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d8fc4a4b
Branch: refs/heads/branch-1.0
Commit: d8fc4a4b6c3871efb2cc024eeb01f5a988f6c91b
Parents: b75301f
Author: xuan
Authored: Wed Apr 16 14:41:22 2014 -0500
Committer: Thomas Graves
Committed: Wed Apr 16 14:43:07 2014 -0500
--
.../apache/spark/deploy/yarn/ClientBase.scala | 33 +-
.../deploy/yarn/ExecutorRunnableUtil.scala | 4 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 63
3 files changed, 85 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d8fc4a4b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index eb42922..628dd98 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -17,6 +17,7 @@
package org.apache.spark.deploy.yarn
+import java.io.File
import java.net.{InetAddress, UnknownHostException, URI}
import java.nio.ByteBuffer
@@ -280,7 +281,8 @@ trait ClientBase extends Logging {
distCacheMgr.setDistArchivesEnv(env)
// Allow users to specify some environment variables.
-Apps.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV"))
+YarnSparkHadoopUtil.setEnvFromInputString(env,
System.getenv("SPARK_YARN_USER_ENV"),
+ File.pathSeparator)
// Add each SPARK_* key to the environment.
System.getenv().filterKeys(_.startsWith("SPARK")).foreach { case (k,v) =>
env(k) = v }
@@ -382,7 +384,8 @@ object ClientBase {
YarnConfiguration.YARN_APPLICATION_CLASSPATH)).getOrElse(
getDefaultYarnApplicationClasspath())
for (c <- classpathEntries) {
- Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim)
+ YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
c.trim,
+File.pathSeparator)
}
val mrClasspathEntries = Option(conf.getStrings(
@@ -390,7 +393,8 @@ object ClientBase {
getDefaultMRApplicationClasspath())
if (mrClasspathEntries != null) {
for (c <- mrClasspathEntries) {
-Apps.addToEnvironment(env, Environment.CLASSPATH.name, c.trim)
+YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
c.trim,
+ File.pathSeparator)
}
}
}
@@ -425,28 +429,29 @@ object ClientBase {
}
def populateClasspath(conf: Configuration, sparkConf: SparkConf, addLog4j:
Boolean, env: HashMap[String, String]) {
-Apps.addToEnvironment(env, Environment.CLASSPATH.name, Environment.PWD.$())
+YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$(),
+ File.pathSeparator)
// If log4j present, ensure ours overrides all others
if (addLog4j) {
- Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
-Path.SEPARATOR + LOG4J_PROP)
+ YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
+Path.SEPARATOR + LOG4J_PROP, File.pathSeparator)
}
// Normally the users app.jar is last in case conflicts with spark jars
val userClasspathFirst = sparkConf.get("spark.yarn.user.classpath.first",
"false")
.toBoolean
if (userClasspathFirst) {
- Apps.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
-Path.SEPARATOR + APP_JAR)
+ YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
Environment.PWD.$() +
+Path.SEPARATOR + APP_JAR, File.pathSeparator)
}
-Apps.addToEnvironmen
git commit: [SPARK-1395] Allow "local:" URIs to work on Yarn.
Repository: spark
Updated Branches:
refs/heads/master bb76eae1b -> 69047506b
[SPARK-1395] Allow "local:" URIs to work on Yarn.
This only works for the three paths defined in the environment
(SPARK_JAR, SPARK_YARN_APP_JAR and SPARK_LOG4J_CONF).
Tested by running SparkPi with local: and file: URIs against Yarn cluster (no
"upload" shows up in logs in the local case).
Author: Marcelo Vanzin
Closes #303 from vanzin/yarn-local and squashes the following commits:
82219c1 [Marcelo Vanzin] [SPARK-1395] Allow "local:" URIs to work on Yarn.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/69047506
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/69047506
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/69047506
Branch: refs/heads/master
Commit: 69047506bf97e6e37e4079c87cb0327d3760ac41
Parents: bb76eae
Author: Marcelo Vanzin
Authored: Thu Apr 17 10:29:38 2014 -0500
Committer: Thomas Graves
Committed: Thu Apr 17 10:29:38 2014 -0500
--
.../org/apache/spark/deploy/SparkSubmit.scala | 4 +-
.../spark/deploy/yarn/ExecutorRunnable.scala| 2 +-
.../apache/spark/deploy/yarn/ClientBase.scala | 190 +--
.../deploy/yarn/ExecutorRunnableUtil.scala | 17 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 6 +-
.../spark/deploy/yarn/ExecutorRunnable.scala| 2 +-
6 files changed, 142 insertions(+), 79 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/69047506/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index e05fbfe..e5d593c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy
import java.io.{PrintStream, File}
-import java.net.URL
+import java.net.{URI, URL}
import org.apache.spark.executor.ExecutorURLClassLoader
@@ -216,7 +216,7 @@ object SparkSubmit {
}
private def addJarToClasspath(localJar: String, loader:
ExecutorURLClassLoader) {
-val localJarFile = new File(localJar)
+val localJarFile = new File(new URI(localJar).getPath())
if (!localJarFile.exists()) {
printWarning(s"Jar $localJar does not exist, skipping.")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/69047506/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 3469b7d..7dae248 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -82,7 +82,7 @@ class ExecutorRunnable(
ctx.setContainerTokens(ByteBuffer.wrap(dob.getData()))
val commands = prepareCommand(masterAddress, slaveId, hostname,
executorMemory, executorCores,
- localResources.contains(ClientBase.LOG4J_PROP))
+ localResources)
logInfo("Setting up executor with commands: " + commands)
ctx.setCommands(commands)
http://git-wip-us.apache.org/repos/asf/spark/blob/69047506/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 628dd98..566de71 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy.yarn
import java.io.File
-import java.net.{InetAddress, UnknownHostException, URI}
+import java.net.{InetAddress, UnknownHostException, URI, URISyntaxException}
import java.nio.ByteBuffer
import scala.collection.JavaConversions._
@@ -209,53 +209,35 @@ trait ClientBase extends Logging {
Map(
ClientBase.SPARK_JAR -> System.getenv("SPARK_JAR"), ClientBase.APP_JAR
-> args.userJar,
- ClientBase.LOG4J_PROP -> System.getenv("SPARK_LOG4J_CONF")
+ ClientBase.LOG4J_PROP -> System.getenv(ClientBase.LOG4J_CONF_ENV_KEY)
).foreach { case(destName, _localPath) =>
val localPath: String = if (_localPath != null) _localPath.trim() else ""
if (! localPath.isEmpty()) {
val localURI = new URI(localPath)
-val setPermissions = if (destNam
git commit: [SPARK-1395] Allow "local:" URIs to work on Yarn.
Repository: spark
Updated Branches:
refs/heads/branch-1.0 b3ad707c4 -> a83a794f1
[SPARK-1395] Allow "local:" URIs to work on Yarn.
This only works for the three paths defined in the environment
(SPARK_JAR, SPARK_YARN_APP_JAR and SPARK_LOG4J_CONF).
Tested by running SparkPi with local: and file: URIs against Yarn cluster (no
"upload" shows up in logs in the local case).
Author: Marcelo Vanzin
Closes #303 from vanzin/yarn-local and squashes the following commits:
82219c1 [Marcelo Vanzin] [SPARK-1395] Allow "local:" URIs to work on Yarn.
(cherry picked from commit 69047506bf97e6e37e4079c87cb0327d3760ac41)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a83a794f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a83a794f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a83a794f
Branch: refs/heads/branch-1.0
Commit: a83a794f1accc616cfccde78af44c5cbf066c647
Parents: b3ad707
Author: Marcelo Vanzin
Authored: Thu Apr 17 10:29:38 2014 -0500
Committer: Thomas Graves
Committed: Thu Apr 17 10:33:15 2014 -0500
--
.../org/apache/spark/deploy/SparkSubmit.scala | 4 +-
.../spark/deploy/yarn/ExecutorRunnable.scala| 2 +-
.../apache/spark/deploy/yarn/ClientBase.scala | 190 +--
.../deploy/yarn/ExecutorRunnableUtil.scala | 17 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 6 +-
.../spark/deploy/yarn/ExecutorRunnable.scala| 2 +-
6 files changed, 142 insertions(+), 79 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a83a794f/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index e05fbfe..e5d593c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy
import java.io.{PrintStream, File}
-import java.net.URL
+import java.net.{URI, URL}
import org.apache.spark.executor.ExecutorURLClassLoader
@@ -216,7 +216,7 @@ object SparkSubmit {
}
private def addJarToClasspath(localJar: String, loader:
ExecutorURLClassLoader) {
-val localJarFile = new File(localJar)
+val localJarFile = new File(new URI(localJar).getPath())
if (!localJarFile.exists()) {
printWarning(s"Jar $localJar does not exist, skipping.")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a83a794f/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 3469b7d..7dae248 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -82,7 +82,7 @@ class ExecutorRunnable(
ctx.setContainerTokens(ByteBuffer.wrap(dob.getData()))
val commands = prepareCommand(masterAddress, slaveId, hostname,
executorMemory, executorCores,
- localResources.contains(ClientBase.LOG4J_PROP))
+ localResources)
logInfo("Setting up executor with commands: " + commands)
ctx.setCommands(commands)
http://git-wip-us.apache.org/repos/asf/spark/blob/a83a794f/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 628dd98..566de71 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy.yarn
import java.io.File
-import java.net.{InetAddress, UnknownHostException, URI}
+import java.net.{InetAddress, UnknownHostException, URI, URISyntaxException}
import java.nio.ByteBuffer
import scala.collection.JavaConversions._
@@ -209,53 +209,35 @@ trait ClientBase extends Logging {
Map(
ClientBase.SPARK_JAR -> System.getenv("SPARK_JAR"), ClientBase.APP_JAR
-> args.userJar,
- ClientBase.LOG4J_PROP -> System.getenv("SPARK_LOG4J_CONF")
+ ClientBase.LOG4J_PROP -> System.getenv(ClientBase.LOG4J_CONF_ENV_KEY)
).foreach { case(destName, _localPath) =>
val localPath: String = if (_localPath != null) _localPath.trim() else ""
if (!
git commit: SPARK-1408 Modify Spark on Yarn to point to the history server when app ...
Repository: spark
Updated Branches:
refs/heads/master 69047506b -> 0058b5d2c
SPARK-1408 Modify Spark on Yarn to point to the history server when app ...
...finishes
Note this is dependent on https://github.com/apache/spark/pull/204 to have a
working history server, but there are no code dependencies.
This also fixes SPARK-1288 yarn stable finishApplicationMaster incomplete.
Since I was in there I made the diagnostic message be passed properly.
Author: Thomas Graves
Closes #362 from tgravescs/SPARK-1408 and squashes the following commits:
ec89705 [Thomas Graves] Fix typo.
446122d [Thomas Graves] Make config yarn specific
f5d5373 [Thomas Graves] SPARK-1408 Modify Spark on Yarn to point to the history
server when app finishes
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0058b5d2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0058b5d2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0058b5d2
Branch: refs/heads/master
Commit: 0058b5d2c74147d24b127a5432f89ebc7050dc18
Parents: 6904750
Author: Thomas Graves
Authored: Thu Apr 17 16:36:37 2014 -0500
Committer: Thomas Graves
Committed: Thu Apr 17 16:36:37 2014 -0500
--
docs/running-on-yarn.md | 1 +
.../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 3 +--
.../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0058b5d2/docs/running-on-yarn.md
--
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 9825143..9765062 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -42,6 +42,7 @@ System Properties:
* `spark.yarn.preserve.staging.files`, set to true to preserve the staged
files(spark jar, app jar, distributed cache files) at the end of the job rather
then delete them.
* `spark.yarn.scheduler.heartbeat.interval-ms`, the interval in ms in which
the Spark application master heartbeats into the YARN ResourceManager. Default
is 5 seconds.
* `spark.yarn.max.executor.failures`, the maximum number of executor failures
before failing the application. Default is the number of executors requested
times 2 with minimum of 3.
+* `spark.yarn.historyServer.address`, the address of the Spark history server
(i.e. host.com:18080). The address should not contain a scheme (http://).
Defaults to not being set since the history server is an optional service. This
address is given to the Yarn ResourceManager when the Spark application
finishes to link the application from the ResourceManager UI to the Spark
history server UI.
# Launching Spark on YARN
http://git-wip-us.apache.org/repos/asf/spark/blob/0058b5d2/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 67ec95c..f078d06 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -366,8 +366,7 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
finishReq.setAppAttemptId(appAttemptId)
finishReq.setFinishApplicationStatus(status)
finishReq.setDiagnostics(diagnostics)
-// Set tracking url to empty since we don't have a history server.
-finishReq.setTrackingUrl("")
+
finishReq.setTrackingUrl(sparkConf.get("spark.yarn.historyServer.address", ""))
resourceManager.finishApplicationMaster(finishReq)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0058b5d2/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 581cfe4..b225be6 100644
---
a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -347,8 +347,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
logInfo("finishApplicationMaster with " + status)
if (registered) {
-// Set tracking URL to empty since we don't have a history server.
-amClient.unregisterApplicationMaster(status, "" /* appMessa
git commit: SPARK-1408 Modify Spark on Yarn to point to the history server when app ...
Repository: spark
Updated Branches:
refs/heads/branch-1.0 a83a794f1 -> 6195fb8db
SPARK-1408 Modify Spark on Yarn to point to the history server when app ...
...finishes
Note this is dependent on https://github.com/apache/spark/pull/204 to have a
working history server, but there are no code dependencies.
This also fixes SPARK-1288 yarn stable finishApplicationMaster incomplete.
Since I was in there I made the diagnostic message be passed properly.
Author: Thomas Graves
Closes #362 from tgravescs/SPARK-1408 and squashes the following commits:
ec89705 [Thomas Graves] Fix typo.
446122d [Thomas Graves] Make config yarn specific
f5d5373 [Thomas Graves] SPARK-1408 Modify Spark on Yarn to point to the history
server when app finishes
(cherry picked from commit 0058b5d2c74147d24b127a5432f89ebc7050dc18)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6195fb8d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6195fb8d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6195fb8d
Branch: refs/heads/branch-1.0
Commit: 6195fb8db9f194856df7f5c02f5023596d0c3253
Parents: a83a794
Author: Thomas Graves
Authored: Thu Apr 17 16:36:37 2014 -0500
Committer: Thomas Graves
Committed: Thu Apr 17 16:36:50 2014 -0500
--
docs/running-on-yarn.md | 1 +
.../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 3 +--
.../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6195fb8d/docs/running-on-yarn.md
--
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 9825143..9765062 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -42,6 +42,7 @@ System Properties:
* `spark.yarn.preserve.staging.files`, set to true to preserve the staged
files(spark jar, app jar, distributed cache files) at the end of the job rather
then delete them.
* `spark.yarn.scheduler.heartbeat.interval-ms`, the interval in ms in which
the Spark application master heartbeats into the YARN ResourceManager. Default
is 5 seconds.
* `spark.yarn.max.executor.failures`, the maximum number of executor failures
before failing the application. Default is the number of executors requested
times 2 with minimum of 3.
+* `spark.yarn.historyServer.address`, the address of the Spark history server
(i.e. host.com:18080). The address should not contain a scheme (http://).
Defaults to not being set since the history server is an optional service. This
address is given to the Yarn ResourceManager when the Spark application
finishes to link the application from the ResourceManager UI to the Spark
history server UI.
# Launching Spark on YARN
http://git-wip-us.apache.org/repos/asf/spark/blob/6195fb8d/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 67ec95c..f078d06 100644
---
a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -366,8 +366,7 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
finishReq.setAppAttemptId(appAttemptId)
finishReq.setFinishApplicationStatus(status)
finishReq.setDiagnostics(diagnostics)
-// Set tracking url to empty since we don't have a history server.
-finishReq.setTrackingUrl("")
+
finishReq.setTrackingUrl(sparkConf.get("spark.yarn.historyServer.address", ""))
resourceManager.finishApplicationMaster(finishReq)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6195fb8d/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
--
diff --git
a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 581cfe4..b225be6 100644
---
a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -347,8 +347,8 @@ class ApplicationMaster(args: ApplicationMasterArguments,
conf: Configuration,
logInfo("finishApplicationMaster with " + status)
if (registered) {
-// Set tracking URL to empt
git commit: SPARK-1557 Set permissions on event log files/directories
Repository: spark
Updated Branches:
refs/heads/master 9a1184a8a -> 8db0f7e28
SPARK-1557 Set permissions on event log files/directories
This adds minimal setting of event log directory/files permissions. To have a
secure environment the user must manually create the top level event log
directory and set permissions up. We can add logic to do that automatically
later if we want.
Author: Thomas Graves
Closes #538 from tgravescs/SPARK-1557 and squashes the following commits:
e471d8e [Thomas Graves] rework
d8b6620 [Thomas Graves] update use of octal
3ca9b79 [Thomas Graves] Updated based on comments
5a09709 [Thomas Graves] add in missing import
3150ed6 [Thomas Graves] SPARK-1557 Set permissions on event log
files/directories
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8db0f7e2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8db0f7e2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8db0f7e2
Branch: refs/heads/master
Commit: 8db0f7e28f5f0330a3344705ff48d8e7b97c383f
Parents: 9a1184a
Author: Thomas Graves
Authored: Tue Apr 29 09:19:48 2014 -0500
Committer: Thomas Graves
Committed: Tue Apr 29 09:19:48 2014 -0500
--
.../spark/scheduler/EventLoggingListener.scala | 6 +-
.../org/apache/spark/util/FileLogger.scala | 22 +++-
docs/security.md| 2 ++
3 files changed, 24 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/8db0f7e2/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 2fe65cd..d822a8e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -21,6 +21,7 @@ import scala.collection.mutable
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.fs.permission.FsPermission
import org.json4s.jackson.JsonMethods._
import org.apache.spark.{Logging, SparkConf, SparkContext}
@@ -54,7 +55,7 @@ private[spark] class EventLoggingListener(
private val logger =
new FileLogger(logDir, conf, hadoopConfiguration, outputBufferSize,
shouldCompress,
- shouldOverwrite)
+ shouldOverwrite, Some(LOG_FILE_PERMISSIONS))
/**
* Begin logging events.
@@ -124,6 +125,9 @@ private[spark] object EventLoggingListener extends Logging {
val SPARK_VERSION_PREFIX = "SPARK_VERSION_"
val COMPRESSION_CODEC_PREFIX = "COMPRESSION_CODEC_"
val APPLICATION_COMPLETE = "APPLICATION_COMPLETE"
+ val LOG_FILE_PERMISSIONS: FsPermission =
+FsPermission.createImmutable(Integer.parseInt("770", 8).toShort)
+
// A cache for compression codecs to avoid creating the same codec many times
private val codecMap = new mutable.HashMap[String, CompressionCodec]
http://git-wip-us.apache.org/repos/asf/spark/blob/8db0f7e2/core/src/main/scala/org/apache/spark/util/FileLogger.scala
--
diff --git a/core/src/main/scala/org/apache/spark/util/FileLogger.scala
b/core/src/main/scala/org/apache/spark/util/FileLogger.scala
index 1ed3b70..0965e0f 100644
--- a/core/src/main/scala/org/apache/spark/util/FileLogger.scala
+++ b/core/src/main/scala/org/apache/spark/util/FileLogger.scala
@@ -24,6 +24,7 @@ import java.util.Date
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path}
+import org.apache.hadoop.fs.permission.FsPermission
import org.apache.spark.{Logging, SparkConf}
import org.apache.spark.io.CompressionCodec
@@ -42,7 +43,8 @@ private[spark] class FileLogger(
hadoopConfiguration: Configuration,
outputBufferSize: Int = 8 * 1024, // 8 KB
compress: Boolean = false,
-overwrite: Boolean = true)
+overwrite: Boolean = true,
+dirPermissions: Option[FsPermission] = None)
extends Logging {
private val dateFormat = new ThreadLocal[SimpleDateFormat]() {
@@ -79,16 +81,25 @@ private[spark] class FileLogger(
if (!fileSystem.mkdirs(path)) {
throw new IOException("Error in creating log directory:
%s".format(logDir))
}
+if (dirPermissions.isDefined) {
+ val fsStatus = fileSystem.getFileStatus(path)
+ if (fsStatus.getPermission().toShort() != dirPermissions.get.toShort) {
+fileSystem.setPermission(path, dirPermissions.get)
+ }
+}
}
/**
* Create a new writer for the file identified by the given path.
+ * If the permissions are not passed in, it will def
git commit: SPARK-1557 Set permissions on event log files/directories
Repository: spark
Updated Branches:
refs/heads/branch-1.0 c27ce2b4f -> bccd13ec1
SPARK-1557 Set permissions on event log files/directories
This adds minimal setting of event log directory/files permissions. To have a
secure environment the user must manually create the top level event log
directory and set permissions up. We can add logic to do that automatically
later if we want.
Author: Thomas Graves
Closes #538 from tgravescs/SPARK-1557 and squashes the following commits:
e471d8e [Thomas Graves] rework
d8b6620 [Thomas Graves] update use of octal
3ca9b79 [Thomas Graves] Updated based on comments
5a09709 [Thomas Graves] add in missing import
3150ed6 [Thomas Graves] SPARK-1557 Set permissions on event log
files/directories
(cherry picked from commit 8db0f7e28f5f0330a3344705ff48d8e7b97c383f)
Signed-off-by: Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bccd13ec
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bccd13ec
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bccd13ec
Branch: refs/heads/branch-1.0
Commit: bccd13ec1825016b7f072370344a4c98ff078dea
Parents: c27ce2b
Author: Thomas Graves
Authored: Tue Apr 29 09:19:48 2014 -0500
Committer: Thomas Graves
Committed: Tue Apr 29 09:20:31 2014 -0500
--
.../spark/scheduler/EventLoggingListener.scala | 6 +-
.../org/apache/spark/util/FileLogger.scala | 22 +++-
docs/security.md| 2 ++
3 files changed, 24 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/bccd13ec/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 2fe65cd..d822a8e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -21,6 +21,7 @@ import scala.collection.mutable
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.fs.permission.FsPermission
import org.json4s.jackson.JsonMethods._
import org.apache.spark.{Logging, SparkConf, SparkContext}
@@ -54,7 +55,7 @@ private[spark] class EventLoggingListener(
private val logger =
new FileLogger(logDir, conf, hadoopConfiguration, outputBufferSize,
shouldCompress,
- shouldOverwrite)
+ shouldOverwrite, Some(LOG_FILE_PERMISSIONS))
/**
* Begin logging events.
@@ -124,6 +125,9 @@ private[spark] object EventLoggingListener extends Logging {
val SPARK_VERSION_PREFIX = "SPARK_VERSION_"
val COMPRESSION_CODEC_PREFIX = "COMPRESSION_CODEC_"
val APPLICATION_COMPLETE = "APPLICATION_COMPLETE"
+ val LOG_FILE_PERMISSIONS: FsPermission =
+FsPermission.createImmutable(Integer.parseInt("770", 8).toShort)
+
// A cache for compression codecs to avoid creating the same codec many times
private val codecMap = new mutable.HashMap[String, CompressionCodec]
http://git-wip-us.apache.org/repos/asf/spark/blob/bccd13ec/core/src/main/scala/org/apache/spark/util/FileLogger.scala
--
diff --git a/core/src/main/scala/org/apache/spark/util/FileLogger.scala
b/core/src/main/scala/org/apache/spark/util/FileLogger.scala
index 1ed3b70..0965e0f 100644
--- a/core/src/main/scala/org/apache/spark/util/FileLogger.scala
+++ b/core/src/main/scala/org/apache/spark/util/FileLogger.scala
@@ -24,6 +24,7 @@ import java.util.Date
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path}
+import org.apache.hadoop.fs.permission.FsPermission
import org.apache.spark.{Logging, SparkConf}
import org.apache.spark.io.CompressionCodec
@@ -42,7 +43,8 @@ private[spark] class FileLogger(
hadoopConfiguration: Configuration,
outputBufferSize: Int = 8 * 1024, // 8 KB
compress: Boolean = false,
-overwrite: Boolean = true)
+overwrite: Boolean = true,
+dirPermissions: Option[FsPermission] = None)
extends Logging {
private val dateFormat = new ThreadLocal[SimpleDateFormat]() {
@@ -79,16 +81,25 @@ private[spark] class FileLogger(
if (!fileSystem.mkdirs(path)) {
throw new IOException("Error in creating log directory:
%s".format(logDir))
}
+if (dirPermissions.isDefined) {
+ val fsStatus = fileSystem.getFileStatus(path)
+ if (fsStatus.getPermission().toShort() != dirPermissions.get.toShort) {
+fileSystem.setPermission(path, dirPermissions.get)
+ }
+}
}
/**
* Create a n
git commit: [SPARK-1522] : YARN ClientBase throws a NPE if there is no YARN Application CP
Repository: spark
Updated Branches:
refs/heads/master 6cf335d79 -> e27344768
[SPARK-1522] : YARN ClientBase throws a NPE if there is no YARN Application CP
The current implementation of ClientBase.getDefaultYarnApplicationClasspath
inspects
the MRJobConfig class for the field DEFAULT_YARN_APPLICATION_CLASSPATH when it
should
be really looking into YarnConfiguration. If the Application Configuration has
no
yarn.application.classpath defined a NPE exception will be thrown.
Additional Changes include:
* Test Suite for ClientBase added
[ticket: SPARK-1522] : https://issues.apache.org/jira/browse/SPARK-1522
Author : bernardo.gomezpala...@gmail.com
Testing : SPARK_HADOOP_VERSION=2.3.0 SPARK_YARN=true ./sbt/sbt test
Author: Bernardo Gomez Palacio
Closes #433 from berngp/feature/SPARK-1522 and squashes the following commits:
2c2e118 [Bernardo Gomez Palacio] [SPARK-1522]: YARN ClientBase throws a NPE if
there is no YARN Application specific CP
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e2734476
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e2734476
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e2734476
Branch: refs/heads/master
Commit: e273447684779a18bd61d733bfe7958b78657ffd
Parents: 6cf335d
Author: Bernardo Gomez Palacio
Authored: Mon Jun 9 16:14:54 2014 -0500
Committer: Thomas Graves
Committed: Mon Jun 9 16:14:54 2014 -0500
--
.../apache/spark/deploy/yarn/ClientBase.scala | 89 +--
.../spark/deploy/yarn/ClientBaseSuite.scala | 112 +++
2 files changed, 167 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e2734476/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
--
diff --git
a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index aeb3f00..4b5e0ef 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer
import scala.collection.JavaConversions._
import scala.collection.mutable.{HashMap, ListBuffer, Map}
+import scala.util.{Try, Success, Failure}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs._
@@ -378,7 +379,7 @@ trait ClientBase extends Logging {
}
}
-object ClientBase {
+object ClientBase extends Logging {
val SPARK_JAR: String = "__spark__.jar"
val APP_JAR: String = "__app__.jar"
val LOG4J_PROP: String = "log4j.properties"
@@ -388,37 +389,47 @@ object ClientBase {
def getSparkJar =
sys.env.get("SPARK_JAR").getOrElse(SparkContext.jarOfClass(this.getClass).head)
- // Based on code from org.apache.hadoop.mapreduce.v2.util.MRApps
- def populateHadoopClasspath(conf: Configuration, env: HashMap[String,
String]) {
-val classpathEntries = Option(conf.getStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH)).getOrElse(
-getDefaultYarnApplicationClasspath())
-if (classpathEntries != null) {
- for (c <- classpathEntries) {
-YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
c.trim,
- File.pathSeparator)
- }
+ def populateHadoopClasspath(conf: Configuration, env: HashMap[String,
String]) = {
+val classPathElementsToAdd = getYarnAppClasspath(conf) ++
getMRAppClasspath(conf)
+for (c <- classPathElementsToAdd.flatten) {
+ YarnSparkHadoopUtil.addToEnvironment(
+env,
+Environment.CLASSPATH.name,
+c.trim,
+File.pathSeparator)
}
+classPathElementsToAdd
+ }
-val mrClasspathEntries = Option(conf.getStrings(
- "mapreduce.application.classpath")).getOrElse(
-getDefaultMRApplicationClasspath())
-if (mrClasspathEntries != null) {
- for (c <- mrClasspathEntries) {
-YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name,
c.trim,
- File.pathSeparator)
- }
-}
+ private def getYarnAppClasspath(conf: Configuration): Option[Seq[String]] =
+Option(conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH))
match {
+ case Some(s) => Some(s.toSeq)
+ case None => getDefaultYarnApplicationClasspath
}
- def getDefaultYarnApplicationClasspath(): Array[String] = {
-try {
- val field =
classOf[MRJobConfig].getField("DEFAULT_YARN_APPLICATION_CLASSPATH")
- field.get(null).asInstanceOf[Array[String]]
-} catch {
- case err: NoSuchFieldError => null
- case err: NoSuchFieldException => null
+ private def getMRAppClasspath(conf: Configuration): Option[Seq[String]] =
+Option(conf
