HDFS-8512. WebHDFS : GETFILESTATUS should return LocatedBlock with storage type info. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3d67390 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3d67390 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3d67390 Branch: refs/heads/HDFS-1312 Commit: e3d673901b396cf5bbede5ed6f607ce68301ec0a Parents: b21dffb Author: Xiaoyu Yao <x...@apache.org> Authored: Wed Nov 25 13:40:43 2015 -0800 Committer: Xiaoyu Yao <x...@apache.org> Committed: Wed Nov 25 13:41:06 2015 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 21 ++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/web/JsonUtil.java | 16 ++++++++++ .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 31 ++++++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3d67390/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 756f2aa..baebff2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -268,6 +268,23 @@ class JsonUtilClient { } } + /** Convert an Object[] to a StorageType[]. */ + static StorageType[] toStorageTypeArray(final List<?> objects) + throws IOException { + if (objects == null) { + return null; + } else if (objects.isEmpty()) { + return StorageType.EMPTY_ARRAY; + } else { + final StorageType[] array = new StorageType[objects.size()]; + int i = 0; + for (Object object : objects) { + array[i++] = StorageType.parseStorageType(object.toString()); + } + return array; + } + } + /** Convert a Json map to LocatedBlock. */ static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException { if (m == null) { @@ -282,8 +299,10 @@ class JsonUtilClient { final DatanodeInfo[] cachedLocations = toDatanodeInfoArray( getList(m, "cachedLocations")); + final StorageType[] storageTypes = toStorageTypeArray( + getList(m, "storageTypes")); final LocatedBlock locatedblock = new LocatedBlock(b, locations, - null, null, startOffset, isCorrupt, cachedLocations); + null, storageTypes, startOffset, isCorrupt, cachedLocations); locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken"))); return locatedblock; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3d67390/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 957087e..46a286b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1686,6 +1686,9 @@ Release 2.8.0 - UNRELEASED HDFS-9438. TestPipelinesFailover assumes Linux ifconfig. (John Zhuge via Yongjun Zhang) + HDFS-8512. WebHDFS : GETFILESTATUS should return LocatedBlock with storage + type info. (xyao) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3d67390/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index f107e66..1f5eaf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -190,6 +190,21 @@ public class JsonUtil { } } + /** Convert a StorageType[] to a Json array. */ + private static Object[] toJsonArray(final StorageType[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_OBJECT_ARRAY; + } else { + final Object[] a = new Object[array.length]; + for(int i = 0; i < array.length; i++) { + a[i] = array[i]; + } + return a; + } + } + /** Convert a LocatedBlock to a Json map. */ private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock ) throws IOException { @@ -202,6 +217,7 @@ public class JsonUtil { m.put("isCorrupt", locatedblock.isCorrupt()); m.put("startOffset", locatedblock.getStartOffset()); m.put("block", toJsonMap(locatedblock.getBlock())); + m.put("storageTypes", toJsonArray(locatedblock.getStorageTypes())); m.put("locations", toJsonArray(locatedblock.getLocations())); m.put("cachedLocations", toJsonArray(locatedblock.getCachedLocations())); return m; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3d67390/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 89a7822..0f60191 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -38,6 +38,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; @@ -745,6 +746,36 @@ public class TestWebHDFS { } } + @Test + public void testWebHdfsGetBlockLocationsWithStorageType() throws Exception{ + MiniDFSCluster cluster = null; + final Configuration conf = WebHdfsTestUtil.createConf(); + final int OFFSET = 42; + final int LENGTH = 512; + final Path PATH = new Path("/foo"); + byte[] CONTENTS = new byte[1024]; + RANDOM.nextBytes(CONTENTS); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsConstants.WEBHDFS_SCHEME); + try (OutputStream os = fs.create(PATH)) { + os.write(CONTENTS); + } + BlockLocation[] locations = fs.getFileBlockLocations(PATH, OFFSET, + LENGTH); + for (BlockLocation location: locations) { + StorageType[] storageTypes = location.getStorageTypes(); + Assert.assertTrue(storageTypes != null && storageTypes.length > 0 && + storageTypes[0] == StorageType.DISK); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + private WebHdfsFileSystem createWebHDFSAsTestUser(final Configuration conf, final URI uri, final String userName) throws Exception {
