HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ecdd4cf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ecdd4cf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ecdd4cf Branch: refs/heads/HDFS-1312 Commit: 0ecdd4cffa51e34997321c384496efc249e3d8ff Parents: 54b2e78 Author: Steve Loughran <ste...@apache.org> Authored: Sun Apr 3 16:39:14 2016 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Sun Apr 3 16:40:19 2016 +0100 ---------------------------------------------------------------------- .../AbstractContractGetFileStatusTest.java | 23 ++++++++++++++++++++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 7 ++++-- .../src/test/resources/contract/s3a.xml | 5 +++++ 3 files changed, 33 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecdd4cf/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java index 7ed375e..3e5bb12 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java @@ -19,10 +19,14 @@ package org.apache.hadoop.fs.contract; import java.io.FileNotFoundException; +import java.io.IOException; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,4 +62,23 @@ public abstract class AbstractContractGetFileStatusTest extends handleExpectedException(e); } } + + @Test + public void testListStatusEmptyDirectory() throws IOException { + // remove the test directory + FileSystem fs = getFileSystem(); + assertTrue(fs.delete(getContract().getTestPath(), true)); + + // create a - non-qualified - Path for a subdir + Path subfolder = getContract().getTestPath().suffix("/"+testPath.getName()); + assertTrue(fs.mkdirs(subfolder)); + + // assert empty ls on the empty dir + assertEquals("ls on an empty directory not of length 0", 0, + fs.listStatus(subfolder).length); + + // assert non-empty ls on parent dir + assertTrue("ls on a non-empty directory of length 0", + fs.listStatus(getContract().getTestPath()).length > 0); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecdd4cf/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 6afb05d..fe705ce 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -788,11 +788,14 @@ public class S3AFileSystem extends FileSystem { ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); + Path fQualified = f.makeQualified(uri, workingDir); + while (true) { for (S3ObjectSummary summary : objects.getObjectSummaries()) { Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir); // Skip over keys that are ourselves and old S3N _$folder$ files - if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) { + if (keyPath.equals(fQualified) || + summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) { if (LOG.isDebugEnabled()) { LOG.debug("Ignoring: " + keyPath); } @@ -807,7 +810,7 @@ public class S3AFileSystem extends FileSystem { } else { result.add(new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()), keyPath, - getDefaultBlockSize(f.makeQualified(uri, workingDir)))); + getDefaultBlockSize(fQualified))); if (LOG.isDebugEnabled()) { LOG.debug("Adding: fi: " + keyPath); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecdd4cf/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml index 4f9c081..be1e7ca 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml @@ -78,6 +78,11 @@ </property> <property> + <name>fs.contract.supports-getfilestatus</name> + <value>true</value> + </property> + + <property> <name>fs.contract.supports-seek</name> <value>true</value> </property>
