Repository: hadoop Updated Branches: refs/heads/branch-2.7 b6f3e1aca -> 11e6c9678
HADOOP-14440. Add metrics for connections dropped. Contributed by Eric Badger. (cherry picked from commit abdd609e51a80388493417126c3bc9b1badc0ac1) (cherry picked from commit 9854682ea46006d98b6d865739b228db804c72ca) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11e6c967 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11e6c967 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11e6c967 Branch: refs/heads/branch-2.7 Commit: 11e6c96788ee3236919df0b19048752c31fdaf6e Parents: b6f3e1a Author: Kihwal Lee <kih...@apache.org> Authored: Mon Jun 5 11:34:40 2017 -0500 Committer: Zhe Zhang <z...@apache.org> Committed: Mon Jun 12 09:45:52 2017 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../main/java/org/apache/hadoop/ipc/Server.java | 20 +++++++++++++++++++- .../apache/hadoop/ipc/metrics/RpcMetrics.java | 4 ++++ .../hadoop-common/src/site/markdown/Metrics.md | 1 + .../java/org/apache/hadoop/ipc/TestIPC.java | 8 +++++--- 5 files changed, 31 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e6c967/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 724299a..600a56d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -39,6 +39,8 @@ Release 2.7.4 - UNRELEASED HADOOP-11703. git should ignore .DS_Store files on Mac OS X (Abin Shahab via aw) + HADOOP-14440. Add metrics for connections dropped. (Eric Badger via kihwal) + OPTIMIZATIONS HADOOP-14138. Remove S3A ref from META-INF service discovery, rely on http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e6c967/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 487fd21..e1cb246 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -63,6 +63,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import javax.security.sasl.Sasl; import javax.security.sasl.SaslException; @@ -876,6 +877,7 @@ public abstract class Server { if (channel.isOpen()) { IOUtils.cleanup(null, channel); } + connectionManager.droppedConnections.getAndIncrement(); continue; } key.attach(c); // so closeCurrentConnection can get the object @@ -2709,6 +2711,16 @@ public abstract class Server { } /** + * The number of RPC connections dropped due to + * too many connections. + * @return the number of dropped rpc connections + */ + public long getNumDroppedConnections() { + return connectionManager.getDroppedConnections(); + + } + + /** * The number of rpc calls in the queue. * @return The number of rpc calls in the queue. */ @@ -2817,7 +2829,8 @@ public abstract class Server { } private class ConnectionManager { - final private AtomicInteger count = new AtomicInteger(); + final private AtomicInteger count = new AtomicInteger(); + final private AtomicLong droppedConnections = new AtomicLong(); final private Set<Connection> connections; /* Map to maintain the statistics per User */ final private Map<String, Integer> userToConnectionsMap; @@ -2904,6 +2917,11 @@ public abstract class Server { return userToConnectionsMap; } + + long getDroppedConnections() { + return droppedConnections.get(); + } + int size() { return count.get(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e6c967/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index 2a0e127..a359160 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -113,6 +113,10 @@ public class RpcMetrics { return server.getCallQueueLen(); } + @Metric("Number of dropped connections") public long numDroppedConnections() { + return server.getNumDroppedConnections(); + } + // Public instrumentation methods that could be extracted to an // abstract class if we decide to do custom instrumentation classes a la // JobTrackerInstrumenation. The methods with //@Override comment are http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e6c967/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index d8edee5..19cc79a 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -101,6 +101,7 @@ Each metrics record contains tags such as Hostname and port (number to which ser | `RpcAuthorizationSuccesses` | Total number of authorization successes | | `NumOpenConnections` | Current number of open connections | | `CallQueueLength` | Current length of the call queue | +| `numDroppedConnections` | Total number of dropped connections | | `rpcQueueTime`*num*`sNumOps` | Shows total number of RPC calls (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | | `rpcQueueTime`*num*`s50thPercentileLatency` | Shows the 50th percentile of RPC queue time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | | `rpcQueueTime`*num*`s75thPercentileLatency` | Shows the 75th percentile of RPC queue time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. | http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e6c967/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 7e31dc4..47e13cb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -1221,7 +1221,7 @@ public class TestIPC { @Test public void testMaxConnections() throws Exception { - conf.setInt("ipc.server.max.connections", 5); + conf.setInt("ipc.server.max.connections", 6); Server server = null; Thread connectors[] = new Thread[10]; @@ -1256,8 +1256,10 @@ public class TestIPC { } Thread.sleep(1000); - // server should only accept up to 5 connections - assertEquals(5, server.getNumOpenConnections()); + // server should only accept up to 6 connections + assertEquals(6, server.getNumOpenConnections()); + // server should drop the other 4 connections + assertEquals(4, server.getNumDroppedConnections()); for (int i = 0; i < 10; i++) { connectors[i].join(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
