HDFS-13821. RBF: Add dfs.federation.router.mount-table.cache.enable so that users can disable cache. Contributed by Fei Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81847392 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81847392 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81847392 Branch: refs/heads/YARN-3409 Commit: 81847392badcd58d934333e7c3b5bf14b4fa1f3f Parents: e557c6b Author: Yiqun Lin <yq...@apache.org> Authored: Wed Aug 22 11:43:40 2018 +0800 Committer: Yiqun Lin <yq...@apache.org> Committed: Wed Aug 22 11:43:40 2018 +0800 ---------------------------------------------------------------------- .../federation/resolver/MountTableResolver.java | 37 ++++++++++++++------ .../server/federation/router/RBFConfigKeys.java | 4 +++ .../src/main/resources/hdfs-rbf-default.xml | 9 +++++ .../resolver/TestMountTableResolver.java | 31 ++++++++++++++++ 4 files changed, 71 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java index c264de3..d45441f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DeprecatedKeys. import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT; import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry; import java.io.IOException; @@ -124,12 +126,19 @@ public class MountTableResolver this.stateStore = null; } - int maxCacheSize = conf.getInt( - FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, - FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT); - this.locationCache = CacheBuilder.newBuilder() - .maximumSize(maxCacheSize) - .build(); + boolean mountTableCacheEnable = conf.getBoolean( + FEDERATION_MOUNT_TABLE_CACHE_ENABLE, + FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT); + if (mountTableCacheEnable) { + int maxCacheSize = conf.getInt( + FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, + FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT); + this.locationCache = CacheBuilder.newBuilder() + .maximumSize(maxCacheSize) + .build(); + } else { + this.locationCache = null; + } registerCacheExternal(); initDefaultNameService(conf); @@ -239,7 +248,7 @@ public class MountTableResolver */ private void invalidateLocationCache(final String path) { LOG.debug("Invalidating {} from {}", path, locationCache); - if (locationCache.size() == 0) { + if (locationCache == null || locationCache.size() == 0) { return; } @@ -359,7 +368,9 @@ public class MountTableResolver LOG.info("Clearing all mount location caches"); writeLock.lock(); try { - this.locationCache.invalidateAll(); + if (this.locationCache != null) { + this.locationCache.invalidateAll(); + } this.tree.clear(); } finally { writeLock.unlock(); @@ -372,6 +383,9 @@ public class MountTableResolver verifyMountTable(); readLock.lock(); try { + if (this.locationCache == null) { + return lookupLocation(path); + } Callable<? extends PathLocation> meh = new Callable<PathLocation>() { @Override public PathLocation call() throws Exception { @@ -603,7 +617,10 @@ public class MountTableResolver * Get the size of the cache. * @return Size of the cache. */ - protected long getCacheSize() { - return this.locationCache.size(); + protected long getCacheSize() throws IOException{ + if (this.locationCache != null) { + return this.locationCache.size(); + } + throw new IOException("localCache is null"); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java index 363db20..87df5d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java @@ -186,6 +186,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { FEDERATION_ROUTER_PREFIX + "mount-table.max-cache-size"; /** Remove cache entries if we have more than 10k. */ public static final int FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT = 10000; + public static final String FEDERATION_MOUNT_TABLE_CACHE_ENABLE = + FEDERATION_ROUTER_PREFIX + "mount-table.cache.enable"; + public static final boolean FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT = + true; // HDFS Router-based federation admin public static final String DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY = http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml index 8806cb2..8be5b8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml @@ -394,6 +394,15 @@ </property> <property> + <name>dfs.federation.router.mount-table.cache.enable</name> + <value>true</value> + <description> + Set to true to enable mount table cache (Path to Remote Location cache). + Disabling the cache is recommended when a large amount of unique paths are queried. + </description> + </property> + + <property> <name>dfs.federation.router.quota.enable</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/81847392/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java index cb3b472..b19a973 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.federation.resolver; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE; import static org.junit.Assert.assertEquals; @@ -37,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.router.Router; import org.apache.hadoop.hdfs.server.federation.store.MountTableStore; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -473,6 +475,35 @@ public class TestMountTableResolver { } @Test + public void testDisableLocalCache() throws IOException { + Configuration conf = new Configuration(); + // Disable mount table cache + conf.setBoolean(FEDERATION_MOUNT_TABLE_CACHE_ENABLE, false); + conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0"); + MountTableResolver tmpMountTable = new MountTableResolver(conf); + + // Root mount point + Map<String, String> map = getMountTableEntry("1", "/"); + tmpMountTable.addEntry(MountTable.newInstance("/", map)); + + // /tmp + map = getMountTableEntry("2", "/tmp"); + tmpMountTable.addEntry(MountTable.newInstance("/tmp", map)); + + // Check localCache is null + try { + tmpMountTable.getCacheSize(); + fail("getCacheSize call should fail."); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("localCache is null", e); + } + + // Check resolve path without cache + assertEquals("2->/tmp/tesfile1.txt", + tmpMountTable.getDestinationForPath("/tmp/tesfile1.txt").toString()); + } + + @Test public void testCacheCleaning() throws Exception { for (int i = 0; i < 1000; i++) { String filename = String.format("/user/a/file-%04d.txt", i); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org