HDFS-12139. HTTPFS liststatus returns incorrect pathSuffix for path of file. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3556e36b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3556e36b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3556e36b Branch: refs/heads/HDFS-7240 Commit: 3556e36be30211f46ac38899ce11a4d4cac6d635 Parents: 413b23e Author: Yongjun Zhang <yzh...@cloudera.com> Authored: Wed Jul 19 10:54:13 2017 -0700 Committer: Yongjun Zhang <yzh...@cloudera.com> Committed: Wed Jul 19 10:56:50 2017 -0700 ---------------------------------------------------------------------- .../hadoop/fs/http/server/FSOperations.java | 15 ++++++----- .../fs/http/client/BaseTestHttpFSWith.java | 26 +++++++++++++++++++- 2 files changed, 34 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3556e36b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 0fb665a..f1615c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -75,15 +75,17 @@ public class FSOperations { /** * @param fileStatuses list of FileStatus objects + * @param isFile is the fileStatuses from a file path * @return JSON map suitable for wire transport */ @SuppressWarnings({"unchecked"}) - private static Map<String, Object> toJson(FileStatus[] fileStatuses) { + private static Map<String, Object> toJson(FileStatus[] fileStatuses, + boolean isFile) { Map<String, Object> json = new LinkedHashMap<>(); Map<String, Object> inner = new LinkedHashMap<>(); JSONArray statuses = new JSONArray(); for (FileStatus f : fileStatuses) { - statuses.add(toJsonInner(f, false)); + statuses.add(toJsonInner(f, isFile)); } inner.put(HttpFSFileSystem.FILE_STATUS_JSON, statuses); json.put(HttpFSFileSystem.FILE_STATUSES_JSON, inner); @@ -129,13 +131,14 @@ public class FSOperations { * These two classes are slightly different, due to the impedance * mismatches between the WebHDFS and FileSystem APIs. * @param entries + * @param isFile is the entries from a file path * @return json */ private static Map<String, Object> toJson(FileSystem.DirectoryEntries - entries) { + entries, boolean isFile) { Map<String, Object> json = new LinkedHashMap<>(); Map<String, Object> inner = new LinkedHashMap<>(); - Map<String, Object> fileStatuses = toJson(entries.getEntries()); + Map<String, Object> fileStatuses = toJson(entries.getEntries(), isFile); inner.put(HttpFSFileSystem.PARTIAL_LISTING_JSON, fileStatuses); inner.put(HttpFSFileSystem.REMAINING_ENTRIES_JSON, entries.hasMore() ? 1 : 0); @@ -690,7 +693,7 @@ public class FSOperations { @Override public Map execute(FileSystem fs) throws IOException { FileStatus[] fileStatuses = fs.listStatus(path, filter); - return toJson(fileStatuses); + return toJson(fileStatuses, fs.getFileStatus(path).isFile()); } @Override @@ -735,7 +738,7 @@ public class FSOperations { WrappedFileSystem wrappedFS = new WrappedFileSystem(fs); FileSystem.DirectoryEntries entries = wrappedFS.listStatusBatch(path, token); - return toJson(entries); + return toJson(entries, wrappedFS.getFileStatus(path).isFile()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3556e36b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index 0fd3f91..e23093e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -364,8 +364,15 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { assertEquals(status2.getLen(), status1.getLen()); FileStatus[] stati = fs.listStatus(path.getParent()); - assertEquals(stati.length, 1); + assertEquals(1, stati.length); assertEquals(stati[0].getPath().getName(), path.getName()); + + // The full path should be the path to the file. See HDFS-12139 + FileStatus[] statl = fs.listStatus(path); + Assert.assertEquals(1, statl.length); + Assert.assertEquals(status2.getPath(), statl[0].getPath()); + Assert.assertEquals(statl[0].getPath().getName(), path.getName()); + Assert.assertEquals(stati[0].getPath(), statl[0].getPath()); } private static void assertSameListing(FileSystem expected, FileSystem @@ -411,6 +418,23 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { proxyFs.create(new Path(dir, "file" + i)).close(); assertSameListing(proxyFs, httpFs, dir); } + + // Test for HDFS-12139 + Path dir1 = new Path(getProxiedFSTestDir(), "dir1"); + proxyFs.mkdirs(dir1); + Path file1 = new Path(dir1, "file1"); + proxyFs.create(file1).close(); + + RemoteIterator<FileStatus> si = proxyFs.listStatusIterator(dir1); + FileStatus statusl = si.next(); + FileStatus status = proxyFs.getFileStatus(file1); + Assert.assertEquals(file1.getName(), statusl.getPath().getName()); + Assert.assertEquals(status.getPath(), statusl.getPath()); + + si = proxyFs.listStatusIterator(file1); + statusl = si.next(); + Assert.assertEquals(file1.getName(), statusl.getPath().getName()); + Assert.assertEquals(status.getPath(), statusl.getPath()); } private void testWorkingdirectory() throws Exception { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
