Repository: hadoop Updated Branches: refs/heads/trunk 0bd421719 -> a0ebb6b39
HDFS-13634. RBF: Configurable value in xml for async connection request queue size. Contributed by CR Hota. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0ebb6b3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0ebb6b3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0ebb6b3 Branch: refs/heads/trunk Commit: a0ebb6b39f2932d3ea2fb5e287f52b841e108428 Parents: 0bd4217 Author: Yiqun Lin <yq...@apache.org> Authored: Wed Aug 29 16:15:22 2018 +0800 Committer: Yiqun Lin <yq...@apache.org> Committed: Wed Aug 29 16:15:22 2018 +0800 ---------------------------------------------------------------------- .../federation/router/ConnectionManager.java | 18 +++++++++++------- .../server/federation/router/RBFConfigKeys.java | 5 +++++ .../src/main/resources/hdfs-rbf-default.xml | 8 ++++++++ 3 files changed, 24 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ebb6b3/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java index 0b50845..9fb83e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java @@ -49,9 +49,6 @@ public class ConnectionManager { private static final Logger LOG = LoggerFactory.getLogger(ConnectionManager.class); - /** Number of parallel new connections to create. */ - protected static final int MAX_NEW_CONNECTIONS = 100; - /** Minimum amount of active connections: 50%. */ protected static final float MIN_ACTIVE_RATIO = 0.5f; @@ -77,8 +74,10 @@ public class ConnectionManager { private final Lock writeLock = readWriteLock.writeLock(); /** Queue for creating new connections. */ - private final BlockingQueue<ConnectionPool> creatorQueue = - new ArrayBlockingQueue<>(MAX_NEW_CONNECTIONS); + private final BlockingQueue<ConnectionPool> creatorQueue; + /** Max size of queue for creating new connections. */ + private final int creatorQueueMaxSize; + /** Create new connections asynchronously. */ private final ConnectionCreator creator; /** Periodic executor to remove stale connection pools. */ @@ -106,7 +105,12 @@ public class ConnectionManager { this.pools = new HashMap<>(); // Create connections in a thread asynchronously - this.creator = new ConnectionCreator(creatorQueue); + this.creatorQueueMaxSize = this.conf.getInt( + RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CREATOR_QUEUE_SIZE, + RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_CREATOR_QUEUE_SIZE_DEFAULT + ); + this.creatorQueue = new ArrayBlockingQueue<>(this.creatorQueueMaxSize); + this.creator = new ConnectionCreator(this.creatorQueue); this.creator.setDaemon(true); // Cleanup periods @@ -213,7 +217,7 @@ public class ConnectionManager { if (conn == null || !conn.isUsable()) { if (!this.creatorQueue.offer(pool)) { LOG.error("Cannot add more than {} connections at the same time", - MAX_NEW_CONNECTIONS); + this.creatorQueueMaxSize); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ebb6b3/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java index 87df5d2..997e1dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java @@ -93,6 +93,11 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { TimeUnit.SECONDS.toMillis(5); // HDFS Router NN client + public static final String + DFS_ROUTER_NAMENODE_CONNECTION_CREATOR_QUEUE_SIZE = + FEDERATION_ROUTER_PREFIX + "connection.creator.queue-size"; + public static final int + DFS_ROUTER_NAMENODE_CONNECTION_CREATOR_QUEUE_SIZE_DEFAULT = 100; public static final String DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE = FEDERATION_ROUTER_PREFIX + "connection.pool-size"; public static final int DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE_DEFAULT = http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ebb6b3/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml index 8be5b8a..ed39d4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml @@ -94,6 +94,14 @@ </property> <property> + <name>dfs.federation.router.connection.creator.queue-size</name> + <value>100</value> + <description> + Size of async connection creator queue. + </description> + </property> + + <property> <name>dfs.federation.router.connection.pool-size</name> <value>1</value> <description> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
