Updated Branches: refs/heads/trunk 57821392a -> 7b724131f
GIRAPH-666: Netty execs threads and metrics threads don't get cleaned up properly (aching) Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/7b724131 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/7b724131 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/7b724131 Branch: refs/heads/trunk Commit: 7b724131fd7fd7f67119258a532a651789beec68 Parents: 5782139 Author: Avery Ching <[email protected]> Authored: Mon May 13 12:41:20 2013 -0700 Committer: Avery Ching <[email protected]> Committed: Mon May 13 13:41:41 2013 -0700 ---------------------------------------------------------------------- CHANGELOG | 3 +++ .../org/apache/giraph/comm/netty/NettyClient.java | 7 ++++++- .../org/apache/giraph/comm/netty/NettyServer.java | 7 ++++++- .../org/apache/giraph/graph/GraphTaskManager.java | 4 ++++ .../org/apache/giraph/metrics/GiraphMetrics.java | 9 +++++++++ .../giraph/metrics/GiraphMetricsRegistry.java | 7 +++++++ 6 files changed, 35 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/7b724131/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 947f12d..50547c0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ Giraph Change Log Release 1.1.0 - unreleased + GIRAPH-666: Netty execs threads and metrics threads don't get + cleaned up properly (aching) + GIRAPH-665: Reduce ZooKeeper output in tests by changing log level from INFO to ERROR (aching) http://git-wip-us.apache.org/repos/asf/giraph/blob/7b724131/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java index 30c32fd..83f1654 100644 --- a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java +++ b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java @@ -164,6 +164,8 @@ public class NettyClient { private final int maxPoolSize; /** Maximum number of attempts to resolve an address*/ private final int maxResolveAddressAttempts; + /** Use execution handler? */ + private final boolean useExecutionHandler; /** Execution handler (if used) */ private final ExecutionHandler executionHandler; /** Name of the handler before the execution handler (if used) */ @@ -218,7 +220,7 @@ public class NettyClient { handlerBeforeExecutionHandler = NETTY_CLIENT_EXECUTION_AFTER_HANDLER.get(conf); - boolean useExecutionHandler = NETTY_CLIENT_USE_EXECUTION_HANDLER.get(conf); + useExecutionHandler = NETTY_CLIENT_USE_EXECUTION_HANDLER.get(conf); if (useExecutionHandler) { int executionThreads = NETTY_CLIENT_EXECUTION_THREADS.get(conf); executionHandler = new ExecutionHandler( @@ -551,6 +553,9 @@ public class NettyClient { bossExecutorService.shutdownNow(); workerExecutorService.shutdownNow(); bootstrap.releaseExternalResources(); + if (useExecutionHandler) { + executionHandler.releaseExternalResources(); + } } } }); http://git-wip-us.apache.org/repos/asf/giraph/blob/7b724131/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyServer.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyServer.java b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyServer.java index 0bfc2d6..9d4afa9 100644 --- a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyServer.java +++ b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyServer.java @@ -118,6 +118,8 @@ public class NettyServer { private final ExecutorService workerExecutorService; /** Request completed map per worker */ private final WorkerRequestReservedMap workerRequestReservedMap; + /** Use execution handler? */ + private final boolean useExecutionHandler; /** Execution handler (if used) */ private final ExecutionHandler executionHandler; /** Name of the handler before the execution handler (if used) */ @@ -172,7 +174,7 @@ public class NettyServer { handlerBeforeExecutionHandler = GiraphConstants.NETTY_SERVER_EXECUTION_AFTER_HANDLER.get(conf); - boolean useExecutionHandler = + useExecutionHandler = GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.get(conf); if (useExecutionHandler) { int executionThreads = conf.getNettyServerExecutionThreads(); @@ -367,6 +369,9 @@ public class NettyServer { } bootstrap.releaseExternalResources(); channelFactory.releaseExternalResources(); + if (useExecutionHandler) { + executionHandler.releaseExternalResources(); + } if (LOG.isInfoEnabled()) { LOG.info("stop: Netty server halted"); } http://git-wip-us.apache.org/repos/asf/giraph/blob/7b724131/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java b/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java index 9caf1b9..82e1b1e 100644 --- a/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java +++ b/giraph-core/src/main/java/org/apache/giraph/graph/GraphTaskManager.java @@ -890,6 +890,8 @@ public class GraphTaskManager<I extends WritableComparable, V extends Writable, if (zkManager != null) { zkManager.offlineZooKeeperServers(ZooKeeperManager.State.FINISHED); } + // Stop tracking metrics + GiraphMetrics.get().shutdown(); } /** @@ -914,6 +916,8 @@ public class GraphTaskManager<I extends WritableComparable, V extends Writable, if (graphFunctions.isWorker()) { serviceWorker.failureCleanup(); } + // Stop tracking metrics + GiraphMetrics.get().shutdown(); // Checkstyle exception due to needing to get the original // exception on failure // CHECKSTYLE: stop IllegalCatch http://git-wip-us.apache.org/repos/asf/giraph/blob/7b724131/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java index 0454a5e..7d980ea 100644 --- a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java +++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetrics.java @@ -146,4 +146,13 @@ public class GiraphMetrics { perJobRequired.printToStream(out); perSuperstep.printToStream(out); } + + /** + * Stop using metrics (for cleanup) + */ + public void shutdown() { + perJobOptional.shutdown(); + perJobRequired.shutdown(); + perSuperstep.shutdown(); + } } http://git-wip-us.apache.org/repos/asf/giraph/blob/7b724131/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java index 3c61872..149f352 100644 --- a/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java +++ b/giraph-core/src/main/java/org/apache/giraph/metrics/GiraphMetricsRegistry.java @@ -303,4 +303,11 @@ public class GiraphMetricsRegistry { protected MetricName makeMetricName(String name) { return new MetricName(groupName, type, name); } + + /** + * Nothing will be captured after this is called. + */ + public void shutdown() { + registry.shutdown(); + } }
