Repository: hadoop Updated Branches: refs/heads/HDFS-7240 a715f60ce -> b3a7f3b2d
HDFS-12105. Ozone: listVolumes doesn't work from ozone commandline. Contributed by Yiqun Lin. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef9ba833 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef9ba833 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef9ba833 Branch: refs/heads/HDFS-7240 Commit: ef9ba8332c90497f4d5383d7661ba3f03c874d6d Parents: 6798e6d Author: Weiwei Yang <w...@apache.org> Authored: Mon Jul 10 10:24:22 2017 +0800 Committer: Xiaoyu Yao <x...@apache.org> Committed: Wed Jul 12 17:11:46 2017 -0700 ---------------------------------------------------------------------- .../ozone/web/client/OzoneRestClient.java | 60 ++++++++++++++----- .../web/ozShell/volume/ListVolumeHandler.java | 34 +++++++++-- .../hadoop/ozone/web/client/TestVolume.java | 62 +++++++++++++++++++- 3 files changed, 132 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef9ba833/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java index 7c144ad..ebb824a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java @@ -204,24 +204,26 @@ public class OzoneRestClient implements Closeable { * List all the volumes owned by the user or Owned by the user specified in * the behalf of string. * - * @param onBehalfOf - User Name of the user if it is not the caller. for - * example, an admin wants to list some other users - * volumes. - * @param prefix - Return only volumes that match this prefix. - * @param maxKeys - Maximum number of results to return, if the result set - * is smaller than requested size, it means that list is - * complete. - * @param prevKey - The last key that client got, server will continue - * returning results from that point. + * @param onBehalfOf + * User Name of the user if it is not the caller. for example, + * an admin wants to list some other users volumes. + * @param prefix + * Return only volumes that match this prefix. + * @param maxKeys + * Maximum number of results to return, if the result set + * is smaller than requested size, it means that list is + * complete. + * @param startVolume + * The previous volume name. * @return List of Volumes * @throws OzoneException */ - public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int - maxKeys, OzoneVolume prevKey) throws OzoneException { + public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, + int maxKeys, String startVolume) throws OzoneException { HttpGet httpGet = null; try (CloseableHttpClient httpClient = newHttpClient()) { URIBuilder builder = new URIBuilder(endPointURI); - if (prefix != null) { + if (!Strings.isNullOrEmpty(prefix)) { builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix); } @@ -230,9 +232,9 @@ public class OzoneRestClient implements Closeable { .toString(maxKeys)); } - if (prevKey != null) { + if (!Strings.isNullOrEmpty(startVolume)) { builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, - prevKey.getOwnerName() + "/" + prevKey.getVolumeName()); + startVolume); } builder.setPath("/").build(); @@ -250,6 +252,33 @@ public class OzoneRestClient implements Closeable { } /** + * List all the volumes owned by the user or Owned by the user specified in + * the behalf of string. + * + * @param onBehalfOf - User Name of the user if it is not the caller. for + * example, an admin wants to list some other users + * volumes. + * @param prefix - Return only volumes that match this prefix. + * @param maxKeys - Maximum number of results to return, if the result set + * is smaller than requested size, it means that list is + * complete. + * @param prevKey - The last key that client got, server will continue + * returning results from that point. + * @return List of Volumes + * @throws OzoneException + */ + public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, + int maxKeys, OzoneVolume prevKey) throws OzoneException { + String volumeName = null; + + if (prevKey != null) { + volumeName = prevKey.getVolumeName(); + } + + return listVolumes(onBehalfOf, prefix, maxKeys, volumeName); + } + + /** * List volumes of the current user or if onBehalfof is not null lists volume * owned by that user. You need admin privilege to read other users volume * lists. @@ -260,7 +289,8 @@ public class OzoneRestClient implements Closeable { */ public List<OzoneVolume> listVolumes(String onBehalfOf) throws OzoneException { - return listVolumes(onBehalfOf, null, 1000, null); + return listVolumes(onBehalfOf, null, + Integer.parseInt(Header.OZONE_DEFAULT_LIST_SIZE), StringUtils.EMPTY); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef9ba833/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java index 3c8c7cb..84a4451 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java @@ -55,6 +55,32 @@ public class ListVolumeHandler extends Handler { "Incorrect call : listVolume is missing"); } + int maxKeys = 0; + if (cmd.hasOption(Shell.LIST_LENGTH)) { + String length = cmd.getOptionValue(Shell.LIST_LENGTH); + try { + maxKeys = Integer.parseInt(length); + } catch (NumberFormatException nfe) { + throw new OzoneRestClientException( + "Invalid max key length, the vaule should be digital."); + } + + if (maxKeys <= 0) { + throw new OzoneRestClientException( + "Invalid max key length, the vaule should be a positive number."); + } + } + + String startVolume = null; + if (cmd.hasOption(Shell.START)) { + startVolume = cmd.getOptionValue(Shell.START); + } + + String prefix = null; + if (cmd.hasOption(Shell.PREFIX)) { + prefix = cmd.getOptionValue(Shell.PREFIX); + } + String ozoneURIString = cmd.getOptionValue(Shell.LIST_VOLUME); URI ozoneURI = verifyURI(ozoneURIString); @@ -62,11 +88,6 @@ public class ListVolumeHandler extends Handler { rootName = "hdfs"; } - if (!cmd.hasOption(Shell.USER)) { - throw new OzoneRestClientException( - "User name is needed in listVolume call."); - } - if (cmd.hasOption(Shell.USER)) { userName = cmd.getOptionValue(Shell.USER); } else { @@ -80,7 +101,8 @@ public class ListVolumeHandler extends Handler { client.setUserAuth(userName); } - List<OzoneVolume> volumes = client.listVolumes(userName); + List<OzoneVolume> volumes = client.listVolumes(userName, prefix, maxKeys, + startVolume); if (volumes != null) { if (cmd.hasOption(Shell.VERBOSE)) { System.out.printf("Found : %d volumes for user : %s %n", volumes.size(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef9ba833/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java index 7dc8381..6c80514 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java @@ -20,11 +20,13 @@ package org.apache.hadoop.ozone.web.client; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status; import org.apache.hadoop.ozone.OzoneClientUtils; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.request.OzoneQuota; @@ -45,6 +47,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -80,7 +83,7 @@ public class TestVolume { Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG); cluster = new MiniOzoneCluster.Builder(conf) - .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build(); + .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); DataNode dataNode = cluster.getDataNodes().get(0); final int port = dataNode.getInfoPort(); @@ -123,8 +126,9 @@ public class TestVolume { client.createVolume("testvol", "bilbo", "100TB"); assertFalse(true); } catch (OzoneException ex) { - // OZone will throw saying volume already exists - assertEquals(ex.getShortMessage(),"volumeAlreadyExists"); + // Ozone will throw saying volume already exists + GenericTestUtils.assertExceptionContains( + Status.VOLUME_ALREADY_EXISTS.toString(), ex); } } @@ -224,6 +228,58 @@ public class TestVolume { Assert.assertEquals(volCount / step , pagecount); } + @Test + public void testListVolumes() throws OzoneException, IOException { + final int volCount = 20; + final String user1 = "test-user-a"; + final String user2 = "test-user-b"; + + client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER); + // Create 20 volumes, 10 for user1 and another 10 for user2. + for (int x = 0; x < volCount; x++) { + String volumeName; + String userName; + + if (x % 2 == 0) { + // create volume [test-vol0, test-vol2, ..., test-vol18] for user1 + userName = user1; + volumeName = "test-vol" + x; + } else { + // create volume [test-vol1, test-vol3, ..., test-vol19] for user2 + userName = user2; + volumeName = "test-vol" + x; + } + OzoneVolume vol = client.createVolume(volumeName, userName, "100TB"); + assertNotNull(vol); + } + + // list all the volumes belong to user1 + List<OzoneVolume> volumeList = client.listVolumes(user1, + null, 100, StringUtils.EMPTY); + assertEquals(10, volumeList.size()); + volumeList.stream() + .filter(item -> item.getOwnerName().equals(user1)) + .collect(Collectors.toList()); + + // test max key parameter of listing volumes + volumeList = client.listVolumes(user1, null, 2, StringUtils.EMPTY); + assertEquals(2, volumeList.size()); + + // test prefix parameter of listing volumes + volumeList = client.listVolumes(user1, "test-vol10", 100, + StringUtils.EMPTY); + assertTrue(volumeList.size() == 1 + && volumeList.get(0).getVolumeName().equals("test-vol10")); + + volumeList = client.listVolumes(user1, "test-vol1", + 100, StringUtils.EMPTY); + assertEquals(5, volumeList.size()); + + // test start key parameter of listing volumes + volumeList = client.listVolumes(user2, null, 100, "test-vol17"); + assertEquals(2, volumeList.size()); + } + /** * Returns a list of mocked {@link CloseableHttpClient} used for testing. * The mocked client replaces the actual calls in --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
