HDFS-13326. RBF: Improve the interfaces to modify and view mount tables. Contributed by Gang Li.
(cherry picked from commit c394051a3d4d9d531f418503cb519606ae2b2e69) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c3d746f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c3d746f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c3d746f Branch: refs/heads/YARN-8200 Commit: 1c3d746fc0a4c24b885690a24ce676d58c2f394b Parents: a5fc638 Author: Inigo Goiri <inigo...@apache.org> Authored: Thu Apr 26 12:59:22 2018 -0700 Committer: Inigo Goiri <inigo...@apache.org> Committed: Thu Apr 26 13:14:08 2018 -0700 ---------------------------------------------------------------------- .../hdfs/tools/federation/RouterAdmin.java | 130 +++++++++++++++- .../federation/router/TestRouterAdminCLI.java | 150 +++++++++++++++++++ .../src/site/markdown/HDFSCommands.md | 6 +- 3 files changed, 276 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3d746f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index b686737..17707dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -94,7 +94,10 @@ public class RouterAdmin extends Configured implements Tool { */ public void printUsage() { String usage = "Federation Admin Tools:\n" - + "\t[-add <source> <nameservice> <destination> " + + "\t[-add <source> <nameservice1, nameservice2, ...> <destination> " + + "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] " + + "-owner <owner> -group <group> -mode <mode>]\n" + + "\t[-update <source> <nameservice1, nameservice2, ...> <destination> " + "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] " + "-owner <owner> -group <group> -mode <mode>]\n" + "\t[-rm <source>]\n" @@ -112,7 +115,7 @@ public class RouterAdmin extends Configured implements Tool { @Override public int run(String[] argv) throws Exception { if (argv.length < 1) { - System.err.println("Not enough parameters specificed"); + System.err.println("Not enough parameters specified"); printUsage(); return -1; } @@ -124,31 +127,37 @@ public class RouterAdmin extends Configured implements Tool { // Verify that we have enough command line parameters if ("-add".equals(cmd)) { if (argv.length < 4) { - System.err.println("Not enough parameters specificed for cmd " + cmd); + System.err.println("Not enough parameters specified for cmd " + cmd); + printUsage(); + return exitCode; + } + } else if ("-update".equals(cmd)) { + if (argv.length < 4) { + System.err.println("Not enough parameters specified for cmd " + cmd); printUsage(); return exitCode; } } else if ("-rm".equalsIgnoreCase(cmd)) { if (argv.length < 2) { - System.err.println("Not enough parameters specificed for cmd " + cmd); + System.err.println("Not enough parameters specified for cmd " + cmd); printUsage(); return exitCode; } } else if ("-setQuota".equalsIgnoreCase(cmd)) { if (argv.length < 4) { - System.err.println("Not enough parameters specificed for cmd " + cmd); + System.err.println("Not enough parameters specified for cmd " + cmd); printUsage(); return exitCode; } } else if ("-clrQuota".equalsIgnoreCase(cmd)) { if (argv.length < 2) { - System.err.println("Not enough parameters specificed for cmd " + cmd); + System.err.println("Not enough parameters specified for cmd " + cmd); printUsage(); return exitCode; } } else if ("-safemode".equalsIgnoreCase(cmd)) { if (argv.length < 2) { - System.err.println("Not enough parameters specificed for cmd " + cmd); + System.err.println("Not enough parameters specified for cmd " + cmd); printUsage(); return exitCode; } @@ -181,7 +190,11 @@ public class RouterAdmin extends Configured implements Tool { try { if ("-add".equals(cmd)) { if (addMount(argv, i)) { - System.out.println("Successfuly added mount point " + argv[i]); + System.out.println("Successfully added mount point " + argv[i]); + } + } else if ("-update".equals(cmd)) { + if (updateMount(argv, i)) { + System.out.println("Successfully updated mount point " + argv[i]); } } else if ("-rm".equals(cmd)) { if (removeMount(argv[i])) { @@ -399,6 +412,107 @@ public class RouterAdmin extends Configured implements Tool { } /** + * Update a mount table entry. + * + * @param parameters Parameters for the mount point. + * @param i Index in the parameters. + */ + public boolean updateMount(String[] parameters, int i) throws IOException { + // Mandatory parameters + String mount = parameters[i++]; + String[] nss = parameters[i++].split(","); + String dest = parameters[i++]; + + // Optional parameters + boolean readOnly = false; + String owner = null; + String group = null; + FsPermission mode = null; + DestinationOrder order = null; + while (i < parameters.length) { + if (parameters[i].equals("-readonly")) { + readOnly = true; + } else if (parameters[i].equals("-order")) { + i++; + try { + order = DestinationOrder.valueOf(parameters[i]); + } catch(Exception e) { + System.err.println("Cannot parse order: " + parameters[i]); + } + } else if (parameters[i].equals("-owner")) { + i++; + owner = parameters[i]; + } else if (parameters[i].equals("-group")) { + i++; + group = parameters[i]; + } else if (parameters[i].equals("-mode")) { + i++; + short modeValue = Short.parseShort(parameters[i], 8); + mode = new FsPermission(modeValue); + } + + i++; + } + + return updateMount(mount, nss, dest, readOnly, order, + new ACLEntity(owner, group, mode)); + } + + /** + * Update a mount table entry. + * + * @param mount Mount point. + * @param nss Nameservices where this is mounted to. + * @param dest Destination path. + * @param readonly If the mount point is read only. + * @param order Order of the destination locations. + * @param aclInfo the ACL info for mount point. + * @return If the mount point was updated. + * @throws IOException Error updating the mount point. + */ + public boolean updateMount(String mount, String[] nss, String dest, + boolean readonly, DestinationOrder order, ACLEntity aclInfo) + throws IOException { + MountTableManager mountTable = client.getMountTableManager(); + + // Create a new entry + Map<String, String> destMap = new LinkedHashMap<>(); + for (String ns : nss) { + destMap.put(ns, dest); + } + MountTable newEntry = MountTable.newInstance(mount, destMap); + + newEntry.setReadOnly(readonly); + + if (order != null) { + newEntry.setDestOrder(order); + } + + // Update ACL info of mount table entry + if (aclInfo.getOwner() != null) { + newEntry.setOwnerName(aclInfo.getOwner()); + } + + if (aclInfo.getGroup() != null) { + newEntry.setGroupName(aclInfo.getGroup()); + } + + if (aclInfo.getMode() != null) { + newEntry.setMode(aclInfo.getMode()); + } + + UpdateMountTableEntryRequest updateRequest = + UpdateMountTableEntryRequest.newInstance(newEntry); + UpdateMountTableEntryResponse updateResponse = + mountTable.updateMountTableEntry(updateRequest); + boolean updated = updateResponse.getStatus(); + if (!updated) { + System.err.println("Cannot update mount point " + mount); + } + return updated; + } + + /** * Remove mount point. * * @param path Path to remove. http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3d746f/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index 1ff07ac..4e84c33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -529,4 +529,154 @@ public class TestRouterAdminCLI { } }, 1000, 30000); } + + @Test + public void testUpdateNonExistingMountTable() throws Exception { + System.setOut(new PrintStream(out)); + String nsId = "ns0"; + String src = "/test-updateNonExistingMounttable"; + String dest = "/updateNonExistingMounttable"; + String[] argv = new String[] {"-update", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = + GetMountTableEntriesRequest.newInstance(src); + GetMountTableEntriesResponse getResponse = + client.getMountTableManager().getMountTableEntries(getRequest); + // Ensure the destination updated successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + } + + @Test + public void testUpdateNameserviceDestinationForExistingMountTable() throws + Exception { + // Add a mount table firstly + String nsId = "ns0"; + String src = "/test-updateNameserviceDestinationForExistingMountTable"; + String dest = "/UpdateNameserviceDestinationForExistingMountTable"; + String[] argv = new String[] {"-add", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = + GetMountTableEntriesRequest.newInstance(src); + GetMountTableEntriesResponse getResponse = + client.getMountTableManager().getMountTableEntries(getRequest); + // Ensure mount table added successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + + // Update the destination + String newNsId = "ns1"; + String newDest = "/newDestination"; + argv = new String[] {"-update", src, newNsId, newDest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + // Ensure the destination updated successfully + mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(newNsId, + mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(newDest, mountTable.getDestinations().get(0).getDest()); + } + + @Test + public void testUpdateReadonlyUserGroupPermissionMountable() + throws Exception { + // Add a mount table + String nsId = "ns0"; + String src = "/test-updateReadonlyUserGroupPermissionMountTable"; + String dest = "/UpdateReadonlyUserGroupPermissionMountTable"; + String[] argv = new String[] {"-add", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = + GetMountTableEntriesRequest.newInstance(src); + GetMountTableEntriesResponse getResponse = + client.getMountTableManager().getMountTableEntries(getRequest); + // Ensure mount table added successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + assertFalse(mountTable.isReadOnly()); + + // Update the readonly, owner, group and permission + String testOwner = "test_owner"; + String testGroup = "test_group"; + argv = new String[] {"-update", src, nsId, dest, "-readonly", + "-owner", testOwner, "-group", testGroup, "-mode", "0455"}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + + // Ensure the destination updated successfully + mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + assertTrue(mountTable.isReadOnly()); + assertEquals(testOwner, mountTable.getOwnerName()); + assertEquals(testGroup, mountTable.getGroupName()); + assertEquals((short)0455, mountTable.getMode().toShort()); + } + + @Test + public void testUpdateOrderMountTable() throws Exception { + testUpdateOrderMountTable(DestinationOrder.HASH); + testUpdateOrderMountTable(DestinationOrder.LOCAL); + testUpdateOrderMountTable(DestinationOrder.RANDOM); + testUpdateOrderMountTable(DestinationOrder.HASH_ALL); + } + + private void testUpdateOrderMountTable(DestinationOrder order) + throws Exception { + // Add a mount table + String nsId = "ns0"; + String src = "/test-updateOrderMountTable-"+order.toString(); + String dest = "/UpdateOrderMountTable"; + String[] argv = new String[] {"-add", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = + GetMountTableEntriesRequest.newInstance(src); + GetMountTableEntriesResponse getResponse = + client.getMountTableManager().getMountTableEntries(getRequest); + + // Ensure mount table added successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + assertEquals(DestinationOrder.HASH, mountTable.getDestOrder()); + + // Update the order + argv = new String[] {"-update", src, nsId, dest, "-order", + order.toString()}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + + // Ensure the destination updated successfully + mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + assertEquals(order, mountTable.getDestOrder()); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3d746f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index cb5d8a7..a58c6c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -409,7 +409,8 @@ Runs the DFS router. See [Router](./HDFSRouterFederation.html#Router) for more i Usage: hdfs dfsrouteradmin - [-add <source> <nameservice> <destination> [-readonly] -owner <owner> -group <group> -mode <mode>] + [-add <source> <nameservice1, nameservice2, ...> <destination> [-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] -owner <owner> -group <group> -mode <mode>] + [-update <source> <nameservice1, nameservice2, ...> <destination> [-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] -owner <owner> -group <group> -mode <mode>] [-rm <source>] [-ls <path>] [-setQuota <path> -nsQuota <nsQuota> -ssQuota <quota in bytes or quota size string>] @@ -420,7 +421,8 @@ Usage: | COMMAND\_OPTION | Description | |:---- |:---- | -| `-add` *source* *nameservice* *destination* | Add a mount table entry or update if it exists. | +| `-add` *source* *nameservices* *destination* | Add a mount table entry or update if it exists. | +| `-update` *source* *nameservices* *destination* | Update a mount table entry or create one if it does not exist. | | `-rm` *source* | Remove mount point of specified path. | | `-ls` *path* | List mount points under specified path. | | `-setQuota` *path* `-nsQuota` *nsQuota* `-ssQuota` *ssQuota* | Set quota for specified path. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) for the quota detail. | --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
