Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 322a2bc74 -> b54e1b229
HADOOP-14036. S3Guard: intermittent duplicate item keys failure. Contributed by Mingliang Liu Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b54e1b22 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b54e1b22 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b54e1b22 Branch: refs/heads/HADOOP-13345 Commit: b54e1b2298bf5aeef38f5db6a232199dd961aa0e Parents: 322a2bc Author: Mingliang Liu <lium...@apache.org> Authored: Wed Mar 15 10:56:02 2017 -0700 Committer: Mingliang Liu <lium...@apache.org> Committed: Fri Mar 17 11:00:18 2017 -0700 ---------------------------------------------------------------------- .../hadoop/fs/FileSystemContractBaseTest.java | 16 +++++++++++++++- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 10 ++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54e1b22/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index 233e524..0274765 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@ -669,12 +669,26 @@ public abstract class FileSystemContractBaseTest extends TestCase { /** * This a sanity check to make sure that any filesystem's handling of - * renames doesn't cause any regressions + * renames empty dirs doesn't cause any regressions. + */ + public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable { + if (!renameSupported()) return; + Path parentdir = path("test/parentdir"); + fs.mkdirs(parentdir); + Path dest = path("test/parentdirdest"); + rename(parentdir, dest, true, false, true); + } + + /** + * This a sanity check to make sure that any filesystem's handling of + * renames non-empty dirs doesn't cause any regressions. */ public void testRenameToDirWithSamePrefixAllowed() throws Throwable { if (!renameSupported()) return; Path parentdir = path("test/parentdir"); fs.mkdirs(parentdir); + // Before renaming, we create one file under the source parent directory + createFile(new Path(parentdir, "mychild")); Path dest = path("test/parentdirdest"); rename(parentdir, dest, true, false, true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54e1b22/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index baa26a7..dcdd3b6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -918,8 +918,14 @@ public class S3AFileSystem extends FileSystem { } } - // We moved all the children, now move the top-level dir. - S3Guard.addMoveDir(metadataStore, srcPaths, dstMetas, src, dst, username); + // We moved all the children, now move the top-level dir if it's non-empty + // Empty directory has been added as the object summary of listObjects + if (srcStatus.isEmptyDirectory() == Tristate.FALSE) { + LOG.debug("To move the non-empty top-level dir src={} and dst={}", + src, dst); + S3Guard.addMoveDir(metadataStore, srcPaths, dstMetas, src, dst, + username); + } } metadataStore.move(srcPaths, dstMetas); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
