spark git commit: SPARK-5841 [CORE] [HOTFIX] Memory leak in DiskBlockManager
Repository: spark
Updated Branches:
refs/heads/branch-1.3 420bc9b3a - e64afcd84
SPARK-5841 [CORE] [HOTFIX] Memory leak in DiskBlockManager
Avoid call to remove shutdown hook being called from shutdown hook
CC pwendell JoshRosen MattWhelan
Author: Sean Owen so...@cloudera.com
Closes #4648 from srowen/SPARK-5841.2 and squashes the following commits:
51548db [Sean Owen] Avoid call to remove shutdown hook being called from
shutdown hook
(cherry picked from commit 49c19fdbad57f0609bbcc9278f9eaa8115a73604)
Signed-off-by: Sean Owen so...@cloudera.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e64afcd8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e64afcd8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e64afcd8
Branch: refs/heads/branch-1.3
Commit: e64afcd8449c161ea134125ca469f5b9c09dbc60
Parents: 420bc9b
Author: Sean Owen so...@cloudera.com
Authored: Tue Feb 17 19:40:06 2015 +
Committer: Sean Owen so...@cloudera.com
Committed: Tue Feb 17 19:40:15 2015 +
--
.../main/scala/org/apache/spark/storage/DiskBlockManager.scala | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e64afcd8/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index ae9df8c..b297f3f 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -138,7 +138,7 @@ private[spark] class DiskBlockManager(blockManager:
BlockManager, conf: SparkCon
val shutdownHook = new Thread(delete Spark local dirs) {
override def run(): Unit = Utils.logUncaughtExceptions {
logDebug(Shutdown hook called)
-DiskBlockManager.this.stop()
+DiskBlockManager.this.doStop()
}
}
Runtime.getRuntime.addShutdownHook(shutdownHook)
@@ -149,7 +149,10 @@ private[spark] class DiskBlockManager(blockManager:
BlockManager, conf: SparkCon
private[spark] def stop() {
// Remove the shutdown hook. It causes memory leaks if we leave it around.
Runtime.getRuntime.removeShutdownHook(shutdownHook)
+doStop()
+ }
+ private def doStop(): Unit = {
// Only perform cleanup if an external service is not serving our shuffle
files.
if (!blockManager.externalShuffleServiceEnabled ||
blockManager.blockManagerId.isDriver) {
localDirs.foreach { localDir =
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: SPARK-5841 [CORE] [HOTFIX] Memory leak in DiskBlockManager
Repository: spark
Updated Branches:
refs/heads/master 24f358b9d - 49c19fdba
SPARK-5841 [CORE] [HOTFIX] Memory leak in DiskBlockManager
Avoid call to remove shutdown hook being called from shutdown hook
CC pwendell JoshRosen MattWhelan
Author: Sean Owen so...@cloudera.com
Closes #4648 from srowen/SPARK-5841.2 and squashes the following commits:
51548db [Sean Owen] Avoid call to remove shutdown hook being called from
shutdown hook
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/49c19fdb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/49c19fdb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/49c19fdb
Branch: refs/heads/master
Commit: 49c19fdbad57f0609bbcc9278f9eaa8115a73604
Parents: 24f358b
Author: Sean Owen so...@cloudera.com
Authored: Tue Feb 17 19:40:06 2015 +
Committer: Sean Owen so...@cloudera.com
Committed: Tue Feb 17 19:40:06 2015 +
--
.../main/scala/org/apache/spark/storage/DiskBlockManager.scala | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/49c19fdb/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index ae9df8c..b297f3f 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -138,7 +138,7 @@ private[spark] class DiskBlockManager(blockManager:
BlockManager, conf: SparkCon
val shutdownHook = new Thread(delete Spark local dirs) {
override def run(): Unit = Utils.logUncaughtExceptions {
logDebug(Shutdown hook called)
-DiskBlockManager.this.stop()
+DiskBlockManager.this.doStop()
}
}
Runtime.getRuntime.addShutdownHook(shutdownHook)
@@ -149,7 +149,10 @@ private[spark] class DiskBlockManager(blockManager:
BlockManager, conf: SparkCon
private[spark] def stop() {
// Remove the shutdown hook. It causes memory leaks if we leave it around.
Runtime.getRuntime.removeShutdownHook(shutdownHook)
+doStop()
+ }
+ private def doStop(): Unit = {
// Only perform cleanup if an external service is not serving our shuffle
files.
if (!blockManager.externalShuffleServiceEnabled ||
blockManager.blockManagerId.isDriver) {
localDirs.foreach { localDir =
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
