Repository: spark Updated Branches: refs/heads/branch-1.6 1dc71ec77 -> 3b903e44b
Revert "[SPARK-12365][CORE] Use ShutdownHookManager where Runtime.getRuntime.addShutdownHook() is called" This reverts commit 4af64385b085002d94c54d11bbd144f9f026bbd8. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3b903e44 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3b903e44 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3b903e44 Branch: refs/heads/branch-1.6 Commit: 3b903e44b912cd36ec26e9e95444656eee7b0c46 Parents: 1dc71ec Author: Andrew Or <and...@databricks.com> Authored: Fri Dec 18 12:56:03 2015 -0800 Committer: Andrew Or <and...@databricks.com> Committed: Fri Dec 18 12:56:03 2015 -0800 ---------------------------------------------------------------------- .../spark/deploy/ExternalShuffleService.scala | 18 ++++++++++++------ .../deploy/mesos/MesosClusterDispatcher.scala | 13 ++++++++----- .../apache/spark/util/ShutdownHookManager.scala | 4 ---- scalastyle-config.xml | 12 ------------ 4 files changed, 20 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala index 7fc96e4..e8a1e35 100644 --- a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala +++ b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala @@ -28,7 +28,7 @@ import org.apache.spark.network.sasl.SaslServerBootstrap import org.apache.spark.network.server.{TransportServerBootstrap, TransportServer} import org.apache.spark.network.shuffle.ExternalShuffleBlockHandler import org.apache.spark.network.util.TransportConf -import org.apache.spark.util.{ShutdownHookManager, Utils} +import org.apache.spark.util.Utils /** * Provides a server from which Executors can read shuffle files (rather than reading directly from @@ -118,13 +118,19 @@ object ExternalShuffleService extends Logging { server = newShuffleService(sparkConf, securityManager) server.start() - ShutdownHookManager.addShutdownHook { () => - logInfo("Shutting down shuffle service.") - server.stop() - barrier.countDown() - } + installShutdownHook() // keep running until the process is terminated barrier.await() } + + private def installShutdownHook(): Unit = { + Runtime.getRuntime.addShutdownHook(new Thread("External Shuffle Service shutdown thread") { + override def run() { + logInfo("Shutting down shuffle service.") + server.stop() + barrier.countDown() + } + }) + } } http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala index 389eff5..5d4e5b8 100644 --- a/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala +++ b/core/src/main/scala/org/apache/spark/deploy/mesos/MesosClusterDispatcher.scala @@ -22,7 +22,7 @@ import java.util.concurrent.CountDownLatch import org.apache.spark.deploy.mesos.ui.MesosClusterUI import org.apache.spark.deploy.rest.mesos.MesosRestServer import org.apache.spark.scheduler.cluster.mesos._ -import org.apache.spark.util.{ShutdownHookManager, SignalLogger} +import org.apache.spark.util.SignalLogger import org.apache.spark.{Logging, SecurityManager, SparkConf} /* @@ -103,11 +103,14 @@ private[mesos] object MesosClusterDispatcher extends Logging { } val dispatcher = new MesosClusterDispatcher(dispatcherArgs, conf) dispatcher.start() - ShutdownHookManager.addShutdownHook { () => - logInfo("Shutdown hook is shutting down dispatcher") - dispatcher.stop() - dispatcher.awaitShutdown() + val shutdownHook = new Thread() { + override def run() { + logInfo("Shutdown hook is shutting down dispatcher") + dispatcher.stop() + dispatcher.awaitShutdown() + } } + Runtime.getRuntime.addShutdownHook(shutdownHook) dispatcher.awaitShutdown() } } http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala index 1a0f3b4..620f226 100644 --- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala +++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala @@ -162,9 +162,7 @@ private[spark] object ShutdownHookManager extends Logging { val hook = new Thread { override def run() {} } - // scalastyle:off runtimeaddshutdownhook Runtime.getRuntime.addShutdownHook(hook) - // scalastyle:on runtimeaddshutdownhook Runtime.getRuntime.removeShutdownHook(hook) } catch { case ise: IllegalStateException => return true @@ -230,9 +228,7 @@ private [util] class SparkShutdownHookManager { .invoke(shm, hookTask, Integer.valueOf(fsPriority + 30)) case Failure(_) => - // scalastyle:off runtimeaddshutdownhook Runtime.getRuntime.addShutdownHook(new Thread(hookTask, "Spark Shutdown Hook")); - // scalastyle:on runtimeaddshutdownhook } } http://git-wip-us.apache.org/repos/asf/spark/blob/3b903e44/scalastyle-config.xml ---------------------------------------------------------------------- diff --git a/scalastyle-config.xml b/scalastyle-config.xml index 4af6dea..050c3f3 100644 --- a/scalastyle-config.xml +++ b/scalastyle-config.xml @@ -157,18 +157,6 @@ This file is divided into 3 sections: ]]></customMessage> </check> - <check customId="runtimeaddshutdownhook" level="error" class="org.scalastyle.file.RegexChecker" enabled="true"> - <parameters><parameter name="regex">Runtime\.getRuntime\.addShutdownHook</parameter></parameters> - <customMessage><![CDATA[ - Are you sure that you want to use Runtime.getRuntime.addShutdownHook? In most cases, you should use - ShutdownHookManager.addShutdownHook instead. - If you must use Runtime.getRuntime.addShutdownHook, wrap the code block with - // scalastyle:off runtimeaddshutdownhook - Runtime.getRuntime.addShutdownHook(...) - // scalastyle:on runtimeaddshutdownhook - ]]></customMessage> - </check> - <check customId="classforname" level="error" class="org.scalastyle.file.RegexChecker" enabled="true"> <parameters><parameter name="regex">Class\.forName</parameter></parameters> <customMessage><![CDATA[ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org