HDFS-13508. RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries. Contributed by Ekanth S.
(cherry picked from commit 484440602c5b69fbd8106010603c61ae051056dd) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8441811 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8441811 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8441811 Branch: refs/heads/YARN-8200 Commit: c8441811fdee2379f50abe218e2afe8133786497 Parents: ae32e21 Author: Inigo Goiri <inigo...@apache.org> Authored: Fri Apr 27 16:28:17 2018 -0700 Committer: Inigo Goiri <inigo...@apache.org> Committed: Fri Apr 27 16:30:07 2018 -0700 ---------------------------------------------------------------------- .../hdfs/tools/federation/RouterAdmin.java | 16 +++ .../federation/router/TestRouterAdminCLI.java | 117 ++++++++++++++++++- 2 files changed, 130 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8441811/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index 17707dc..b0a2062 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -26,6 +26,7 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -322,6 +323,7 @@ public class RouterAdmin extends Configured implements Tool { public boolean addMount(String mount, String[] nss, String dest, boolean readonly, DestinationOrder order, ACLEntity aclInfo) throws IOException { + mount = normalizeFileSystemPath(mount); // Get the existing entry MountTableManager mountTable = client.getMountTableManager(); GetMountTableEntriesRequest getRequest = @@ -473,6 +475,7 @@ public class RouterAdmin extends Configured implements Tool { public boolean updateMount(String mount, String[] nss, String dest, boolean readonly, DestinationOrder order, ACLEntity aclInfo) throws IOException { + mount = normalizeFileSystemPath(mount); MountTableManager mountTable = client.getMountTableManager(); // Create a new entry @@ -519,6 +522,7 @@ public class RouterAdmin extends Configured implements Tool { * @throws IOException If it cannot be removed. */ public boolean removeMount(String path) throws IOException { + path = normalizeFileSystemPath(path); MountTableManager mountTable = client.getMountTableManager(); RemoveMountTableEntryRequest request = RemoveMountTableEntryRequest.newInstance(path); @@ -538,6 +542,7 @@ public class RouterAdmin extends Configured implements Tool { * @throws IOException If it cannot be listed. */ public void listMounts(String path) throws IOException { + path = normalizeFileSystemPath(path); MountTableManager mountTable = client.getMountTableManager(); GetMountTableEntriesRequest request = GetMountTableEntriesRequest.newInstance(path); @@ -798,6 +803,17 @@ public class RouterAdmin extends Configured implements Tool { } /** + * Normalize a path for that filesystem. + * + * @param path Path to normalize. + * @return Normalized path. + */ + private static String normalizeFileSystemPath(final String path) { + Path normalizedPath = new Path(path); + return normalizedPath.toString(); + } + + /** * Inner class that stores ACL info of mount table. */ static class ACLEntity { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8441811/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index 4e84c33..2537c19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -159,6 +159,45 @@ public class TestRouterAdminCLI { } @Test + public void testAddMountTableNotNormalized() throws Exception { + String nsId = "ns0"; + String src = "/test-addmounttable-notnormalized"; + String srcWithSlash = src + "/"; + String dest = "/addmounttable-notnormalized"; + String[] argv = new String[] {"-add", srcWithSlash, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest + .newInstance(src); + GetMountTableEntriesResponse getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + MountTable mountTable = getResponse.getEntries().get(0); + + List<RemoteLocation> destinations = mountTable.getDestinations(); + assertEquals(1, destinations.size()); + + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, destinations.get(0).getNameserviceId()); + assertEquals(dest, destinations.get(0).getDest()); + assertFalse(mountTable.isReadOnly()); + + // test mount table update behavior + dest = dest + "-new"; + argv = new String[] {"-add", srcWithSlash, nsId, dest, "-readonly"}; + assertEquals(0, ToolRunner.run(admin, argv)); + stateStore.loadCache(MountTableStoreImpl.class, true); + + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + mountTable = getResponse.getEntries().get(0); + assertEquals(2, mountTable.getDestinations().size()); + assertEquals(nsId, mountTable.getDestinations().get(1).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(1).getDest()); + assertTrue(mountTable.isReadOnly()); + } + + @Test public void testAddOrderMountTable() throws Exception { testAddOrderMountTable(DestinationOrder.HASH); testAddOrderMountTable(DestinationOrder.LOCAL); @@ -192,6 +231,7 @@ public class TestRouterAdminCLI { public void testListMountTable() throws Exception { String nsId = "ns0"; String src = "/test-lsmounttable"; + String srcWithSlash = src + "/"; String dest = "/lsmounttable"; String[] argv = new String[] {"-add", src, nsId, dest}; assertEquals(0, ToolRunner.run(admin, argv)); @@ -203,6 +243,11 @@ public class TestRouterAdminCLI { assertEquals(0, ToolRunner.run(admin, argv)); assertTrue(out.toString().contains(src)); + // Test with not-normalized src input + argv = new String[] {"-ls", srcWithSlash}; + assertEquals(0, ToolRunner.run(admin, argv)); + assertTrue(out.toString().contains(src)); + out.reset(); GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest .newInstance("/"); @@ -256,6 +301,33 @@ public class TestRouterAdminCLI { } @Test + public void testRemoveMountTableNotNormalized() throws Exception { + String nsId = "ns0"; + String src = "/test-rmmounttable-notnormalized"; + String srcWithSlash = src + "/"; + String dest = "/rmmounttable-notnormalized"; + String[] argv = new String[] {"-add", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest + .newInstance(src); + GetMountTableEntriesResponse getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + // ensure mount table added successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + + argv = new String[] {"-rm", srcWithSlash}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + assertEquals(0, getResponse.getEntries().size()); + } + + @Test public void testMountTableDefaultACL() throws Exception { String[] argv = new String[] {"-add", "/testpath0", "ns0", "/testdir0"}; assertEquals(0, ToolRunner.run(admin, argv)); @@ -552,12 +624,12 @@ public class TestRouterAdminCLI { } @Test - public void testUpdateNameserviceDestinationForExistingMountTable() throws + public void testUpdateDestinationForExistingMountTable() throws Exception { // Add a mount table firstly String nsId = "ns0"; - String src = "/test-updateNameserviceDestinationForExistingMountTable"; - String dest = "/UpdateNameserviceDestinationForExistingMountTable"; + String src = "/test-updateDestinationForExistingMountTable"; + String dest = "/UpdateDestinationForExistingMountTable"; String[] argv = new String[] {"-add", src, nsId, dest}; assertEquals(0, ToolRunner.run(admin, argv)); @@ -590,6 +662,45 @@ public class TestRouterAdminCLI { } @Test + public void testUpdateDestinationForExistingMountTableNotNormalized() throws + Exception { + // Add a mount table firstly + String nsId = "ns0"; + String src = "/test-updateDestinationForExistingMountTableNotNormalized"; + String srcWithSlash = src + "/"; + String dest = "/UpdateDestinationForExistingMountTableNotNormalized"; + String[] argv = new String[] {"-add", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = + GetMountTableEntriesRequest.newInstance(src); + GetMountTableEntriesResponse getResponse = + client.getMountTableManager().getMountTableEntries(getRequest); + // Ensure mount table added successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + + // Update the destination + String newNsId = "ns1"; + String newDest = "/newDestination"; + argv = new String[] {"-update", srcWithSlash, newNsId, newDest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + // Ensure the destination updated successfully + mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(newNsId, + mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(newDest, mountTable.getDestinations().get(0).getDest()); + } + + @Test public void testUpdateReadonlyUserGroupPermissionMountable() throws Exception { // Add a mount table --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org