spark git commit: [SPARK-24021][CORE] fix bug in BlacklistTracker's updateBlacklistForFetchFailure
Repository: spark
Updated Branches:
refs/heads/branch-2.3 32bec6ca3 -> 7fb11176f
[SPARK-24021][CORE] fix bug in BlacklistTracker's updateBlacklistForFetchFailure
## What changes were proposed in this pull request?
Thereâs a miswrite in BlacklistTracker's updateBlacklistForFetchFailure:
```
val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(exec, HashSet[String]())
blacklistedExecsOnNode += exec
```
where first **exec** should be **host**.
## How was this patch tested?
adjust existed test.
Author: wuyi
Closes #21104 from Ngone51/SPARK-24021.
(cherry picked from commit 0deaa5251326a32a3d2d2b8851193ca926303972)
Signed-off-by: Imran Rashid
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7fb11176
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7fb11176
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7fb11176
Branch: refs/heads/branch-2.3
Commit: 7fb11176f285b4de47e61511c09acbbb79e5c44c
Parents: 32bec6c
Author: wuyi
Authored: Thu Apr 19 09:00:33 2018 -0500
Committer: Imran Rashid
Committed: Thu Apr 19 09:00:46 2018 -0500
--
.../scala/org/apache/spark/scheduler/BlacklistTracker.scala | 2 +-
.../org/apache/spark/scheduler/BlacklistTrackerSuite.scala | 5 +
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/7fb11176/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
index 952598f..30cf75d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
@@ -210,7 +210,7 @@ private[scheduler] class BlacklistTracker (
updateNextExpiryTime()
killBlacklistedExecutor(exec)
-val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(exec, HashSet[String]())
+val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(host, HashSet[String]())
blacklistedExecsOnNode += exec
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7fb11176/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
index 88a57c5..00867ef 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
@@ -574,6 +574,9 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
verify(allocationClientMock, never).killExecutors(any(), any(), any(),
any())
verify(allocationClientMock, never).killExecutorsOnHost(any())
+assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
+assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
+
// Enable auto-kill. Blacklist an executor and make sure killExecutors is
called.
conf.set(config.BLACKLIST_KILL_ENABLED, true)
blacklist = new BlacklistTracker(listenerBusMock, conf,
Some(allocationClientMock), clock)
@@ -589,6 +592,8 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
1000 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
assert(blacklist.nextExpiryTime === 1000 +
blacklist.BLACKLIST_TIMEOUT_MILLIS)
assert(blacklist.nodeIdToBlacklistExpiryTime.isEmpty)
+assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
+assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
// Enable external shuffle service to see if all the executors on this
node will be killed.
conf.set(config.SHUFFLE_SERVICE_ENABLED, true)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24021][CORE] fix bug in BlacklistTracker's updateBlacklistForFetchFailure
Repository: spark
Updated Branches:
refs/heads/master d96c3e33c -> 0deaa5251
[SPARK-24021][CORE] fix bug in BlacklistTracker's updateBlacklistForFetchFailure
## What changes were proposed in this pull request?
Thereâs a miswrite in BlacklistTracker's updateBlacklistForFetchFailure:
```
val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(exec, HashSet[String]())
blacklistedExecsOnNode += exec
```
where first **exec** should be **host**.
## How was this patch tested?
adjust existed test.
Author: wuyi
Closes #21104 from Ngone51/SPARK-24021.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0deaa525
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0deaa525
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0deaa525
Branch: refs/heads/master
Commit: 0deaa5251326a32a3d2d2b8851193ca926303972
Parents: d96c3e3
Author: wuyi
Authored: Thu Apr 19 09:00:33 2018 -0500
Committer: Imran Rashid
Committed: Thu Apr 19 09:00:33 2018 -0500
--
.../scala/org/apache/spark/scheduler/BlacklistTracker.scala | 2 +-
.../org/apache/spark/scheduler/BlacklistTrackerSuite.scala | 5 +
2 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0deaa525/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
index 952598f..30cf75d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
@@ -210,7 +210,7 @@ private[scheduler] class BlacklistTracker (
updateNextExpiryTime()
killBlacklistedExecutor(exec)
-val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(exec, HashSet[String]())
+val blacklistedExecsOnNode =
nodeToBlacklistedExecs.getOrElseUpdate(host, HashSet[String]())
blacklistedExecsOnNode += exec
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0deaa525/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
index 06d7afa..96c8404 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BlacklistTrackerSuite.scala
@@ -574,6 +574,9 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
verify(allocationClientMock, never).killExecutors(any(), any(), any(),
any())
verify(allocationClientMock, never).killExecutorsOnHost(any())
+assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
+assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
+
// Enable auto-kill. Blacklist an executor and make sure killExecutors is
called.
conf.set(config.BLACKLIST_KILL_ENABLED, true)
blacklist = new BlacklistTracker(listenerBusMock, conf,
Some(allocationClientMock), clock)
@@ -589,6 +592,8 @@ class BlacklistTrackerSuite extends SparkFunSuite with
BeforeAndAfterEach with M
1000 + blacklist.BLACKLIST_TIMEOUT_MILLIS)
assert(blacklist.nextExpiryTime === 1000 +
blacklist.BLACKLIST_TIMEOUT_MILLIS)
assert(blacklist.nodeIdToBlacklistExpiryTime.isEmpty)
+assert(blacklist.nodeToBlacklistedExecs.contains("hostA"))
+assert(blacklist.nodeToBlacklistedExecs("hostA").contains("1"))
// Enable external shuffle service to see if all the executors on this
node will be killed.
conf.set(config.SHUFFLE_SERVICE_ENABLED, true)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
