[hadoop] 01/02: Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)"
This is an automated email from the ASF dual-hosted git repository. weichiu pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 289f8acc644ff8f30452bec35e2833af57994112 Author: Wei-Chiu Chuang AuthorDate: Mon May 18 09:41:03 2020 -0700 Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)" This reverts commit ad9a6a0ee3d6cb3bde5e23c73151c0857d47ffd4. --- .../java/org/apache/hadoop/hdfs/ClientContext.java | 20 +-- .../hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 - .../hdfs/client/impl/BlockReaderFactory.java | 9 +- .../hadoop/hdfs/client/impl/DfsClientConf.java | 21 +-- .../src/main/resources/hdfs-default.xml| 10 -- .../hadoop/fs/TestEnhancedByteBufferAccess.java| 6 +- .../hdfs/client/impl/TestBlockReaderFactory.java | 14 +- .../hdfs/client/impl/TestBlockReaderLocal.java | 190 ++--- .../hdfs/shortcircuit/TestShortCircuitCache.java | 19 +-- 9 files changed, 76 insertions(+), 215 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java index 7a03240..cbd941b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java @@ -77,7 +77,7 @@ public class ClientContext { /** * Caches short-circuit file descriptors, mmap regions. */ - private final ShortCircuitCache[] shortCircuitCache; + private final ShortCircuitCache shortCircuitCache; /** * Caches TCP and UNIX domain sockets for reuse. @@ -132,23 +132,13 @@ public class ClientContext { */ private DeadNodeDetector deadNodeDetector = null; - /** - * ShortCircuitCache array size. - */ - private final int clientShortCircuitNum; - private ClientContext(String name, DfsClientConf conf, Configuration config) { final ShortCircuitConf scConf = conf.getShortCircuitConf(); this.name = name; this.confString = scConf.confAsString(); -this.clientShortCircuitNum = conf.getClientShortCircuitNum(); -this.shortCircuitCache = new ShortCircuitCache[this.clientShortCircuitNum]; -for (int i = 0; i < this.clientShortCircuitNum; i++) { - this.shortCircuitCache[i] = ShortCircuitCache.fromConf(scConf); -} - +this.shortCircuitCache = ShortCircuitCache.fromConf(scConf); this.peerCache = new PeerCache(scConf.getSocketCacheCapacity(), scConf.getSocketCacheExpiry()); this.keyProviderCache = new KeyProviderCache( @@ -238,11 +228,7 @@ public class ClientContext { } public ShortCircuitCache getShortCircuitCache() { -return shortCircuitCache[0]; - } - - public ShortCircuitCache getShortCircuitCache(long idx) { -return shortCircuitCache[(int) (idx % clientShortCircuitNum)]; +return shortCircuitCache; } public PeerCache getPeerCache() { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 0c35c8d..efc2766 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -144,8 +144,6 @@ public interface HdfsClientConfigKeys { "dfs.short.circuit.shared.memory.watcher.interrupt.check.ms"; int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT = 6; - String DFS_CLIENT_SHORT_CIRCUIT_NUM = "dfs.client.short.circuit.num"; - int DFS_CLIENT_SHORT_CIRCUIT_NUM_DEFAULT = 1; String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY = "dfs.client.slow.io.warning.threshold.ms"; longDFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 3; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java index 028d629..a3b611c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java @@ -476,8 +476,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { "giving up on BlockReaderLocal.", this, pathInfo); return null; } -ShortCircuitCache cache = -clientContext.getShortCircuitCache(block.getBlockId()); +ShortCircuitCache cache = clientContext.getShortCircuitCache(); ExtendedBlockId k
[hadoop] 01/02: Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)"
This is an automated email from the ASF dual-hosted git repository. weichiu pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 4525292d41482330a86f1cc3935e072f9f67c308 Author: Wei-Chiu Chuang AuthorDate: Mon May 18 09:22:05 2020 -0700 Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)" This reverts commit 86e6aa8eec538e142044e2b6415ec1caff5e9cbd. --- .../java/org/apache/hadoop/hdfs/ClientContext.java | 20 +-- .../hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 - .../hdfs/client/impl/BlockReaderFactory.java | 9 +- .../hadoop/hdfs/client/impl/DfsClientConf.java | 21 +-- .../src/main/resources/hdfs-default.xml| 10 -- .../hadoop/fs/TestEnhancedByteBufferAccess.java| 6 +- .../hdfs/client/impl/TestBlockReaderFactory.java | 14 +- .../hdfs/client/impl/TestBlockReaderLocal.java | 190 ++--- .../hdfs/shortcircuit/TestShortCircuitCache.java | 19 +-- 9 files changed, 76 insertions(+), 215 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java index 7a03240..cbd941b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java @@ -77,7 +77,7 @@ public class ClientContext { /** * Caches short-circuit file descriptors, mmap regions. */ - private final ShortCircuitCache[] shortCircuitCache; + private final ShortCircuitCache shortCircuitCache; /** * Caches TCP and UNIX domain sockets for reuse. @@ -132,23 +132,13 @@ public class ClientContext { */ private DeadNodeDetector deadNodeDetector = null; - /** - * ShortCircuitCache array size. - */ - private final int clientShortCircuitNum; - private ClientContext(String name, DfsClientConf conf, Configuration config) { final ShortCircuitConf scConf = conf.getShortCircuitConf(); this.name = name; this.confString = scConf.confAsString(); -this.clientShortCircuitNum = conf.getClientShortCircuitNum(); -this.shortCircuitCache = new ShortCircuitCache[this.clientShortCircuitNum]; -for (int i = 0; i < this.clientShortCircuitNum; i++) { - this.shortCircuitCache[i] = ShortCircuitCache.fromConf(scConf); -} - +this.shortCircuitCache = ShortCircuitCache.fromConf(scConf); this.peerCache = new PeerCache(scConf.getSocketCacheCapacity(), scConf.getSocketCacheExpiry()); this.keyProviderCache = new KeyProviderCache( @@ -238,11 +228,7 @@ public class ClientContext { } public ShortCircuitCache getShortCircuitCache() { -return shortCircuitCache[0]; - } - - public ShortCircuitCache getShortCircuitCache(long idx) { -return shortCircuitCache[(int) (idx % clientShortCircuitNum)]; +return shortCircuitCache; } public PeerCache getPeerCache() { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index e8b5402..ab3f6f2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -144,8 +144,6 @@ public interface HdfsClientConfigKeys { "dfs.short.circuit.shared.memory.watcher.interrupt.check.ms"; int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT = 6; - String DFS_CLIENT_SHORT_CIRCUIT_NUM = "dfs.client.short.circuit.num"; - int DFS_CLIENT_SHORT_CIRCUIT_NUM_DEFAULT = 1; String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY = "dfs.client.slow.io.warning.threshold.ms"; longDFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 3; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java index 028d629..a3b611c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java @@ -476,8 +476,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { "giving up on BlockReaderLocal.", this, pathInfo); return null; } -ShortCircuitCache cache = -clientContext.getShortCircuitCache(block.getBlockId()); +ShortCircuitCache cache = clientContext.getShortCircuitCache(); ExtendedBlockId key =