Repository: hadoop Updated Branches: refs/heads/HADOOP-13345 0c61010de -> 2220b787c
HADOOP-13336 S3A to support per-bucket configuration (S3Guard specific changes). Contributed by Steve Loughran Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2220b787 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2220b787 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2220b787 Branch: refs/heads/HADOOP-13345 Commit: 2220b787c363574ea9ffb2d989ccc60c29127f7f Parents: 0c61010 Author: Steve Loughran <ste...@apache.org> Authored: Wed Jan 11 18:30:46 2017 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Wed Jan 11 18:30:46 2017 +0000 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 15 +++++++++------ .../org/apache/hadoop/fs/s3a/s3guard/S3Guard.java | 11 ----------- .../hadoop/fs/s3a/ITestS3AFileOperationCost.java | 9 ++++----- .../hadoop/fs/s3a/ITestS3GuardListConsistency.java | 2 +- .../java/org/apache/hadoop/fs/s3a/S3ATestUtils.java | 1 - .../fs/s3a/fileContext/ITestS3AFileContextURI.java | 13 ++++++++++--- .../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 2 +- 7 files changed, 25 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2220b787/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index b4059d2..345deca 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -765,7 +765,7 @@ public class S3AFileSystem extends FileSystem { // If we have a MetadataStore, track deletions/creations. List<Path> srcPaths = null; List<PathMetadata> dstMetas = null; - if (!S3Guard.isNullMetadataStore(metadataStore)) { + if (hasMetadataStore()) { srcPaths = new ArrayList<>(); dstMetas = new ArrayList<>(); } @@ -834,7 +834,7 @@ public class S3AFileSystem extends FileSystem { dstKey + summary.getKey().substring(srcKey.length()); copyFile(summary.getKey(), newDstKey, length); - if (!S3Guard.isNullMetadataStore(metadataStore)) { + if (hasMetadataStore()) { Path srcPath = keyToQualifiedPath(summary.getKey()); Path dstPath = keyToQualifiedPath(newDstKey); if (objectRepresentsDirectory(summary.getKey(), length)) { @@ -885,8 +885,11 @@ public class S3AFileSystem extends FileSystem { return getObjectMetadata(pathToKey(path)); } - @VisibleForTesting - public boolean isMetadataStoreConfigured() { + /** + * Does this Filesystem have a metadata store? + * @return true if the FS has been instantiated with a metadata store + */ + public boolean hasMetadataStore() { return !S3Guard.isNullMetadataStore(metadataStore); } @@ -1549,7 +1552,7 @@ public class S3AFileSystem extends FileSystem { incrementStatistic(INVOCATION_MKDIRS); FileStatus fileStatus; List<Path> metadataStoreDirs = null; - if (!S3Guard.isNullMetadataStore(metadataStore)) { + if (hasMetadataStore()) { metadataStoreDirs = new ArrayList<>(); } @@ -1896,7 +1899,7 @@ public class S3AFileSystem extends FileSystem { // See note about failure semantics in s3guard.md doc. try { - if (!S3Guard.isNullMetadataStore(metadataStore)) { + if (hasMetadataStore()) { S3AFileStatus status = createUploadFileStatus(p, S3AUtils.objectRepresentsDirectory(key, length), length, getDefaultBlockSize(p), username); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2220b787/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java index 862fb36..afc76b5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java @@ -105,17 +105,6 @@ public final class S3Guard { } } - /** - * Predicate to check whether or not the metadata store is the null one. - * @param conf Configuration - * @return true if NullMetadataStore is configured for s3a, or if the - * configuration is missing. - */ - public static boolean isNullMetadataStoreConfigured(Configuration conf) { - Class<? extends MetadataStore> msClass = getMetadataStoreClass(conf); - return msClass.equals(NullMetadataStore.class); - } - private static Class<? extends MetadataStore> getMetadataStoreClass( Configuration conf) { if (conf == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2220b787/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index 98184e4..2ea38c5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -21,7 +21,6 @@ package org.apache.hadoop.fs.s3a; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.s3a.s3guard.S3Guard; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +64,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase { resetMetricDiffs(); S3AFileStatus status = fs.getFileStatus(simpleFile); assertTrue("not a file: " + status, status.isFile()); - if (S3Guard.isNullMetadataStoreConfigured(fs.getConf())) { + if (!fs.hasMetadataStore()) { metadataRequests.assertDiffEquals(1); } listRequests.assertDiffEquals(0); @@ -85,7 +84,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase { S3AFileStatus status = fs.getFileStatus(dir); assertTrue("not empty: " + status, status.isEmptyDirectory()); - if (S3Guard.isNullMetadataStoreConfigured(fs.getConf())) { + if (!fs.hasMetadataStore()) { metadataRequests.assertDiffEquals(2); } listRequests.assertDiffEquals(0); @@ -140,7 +139,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase { + "\n" + ContractTestUtils.ls(fs, dir) + "\n" + fsState); } - if (S3Guard.isNullMetadataStoreConfigured(fs.getConf())) { + if (!fs.hasMetadataStore()) { metadataRequests.assertDiffEquals(2); listRequests.assertDiffEquals(1); } @@ -200,7 +199,7 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase { // operations, it depends on side effects happening internally. With // metadata store enabled, it is brittle to change. We disable this test // before the internal behavior w/ or w/o metadata store. - assumeFalse(fs.isMetadataStoreConfigured()); + assumeFalse(fs.hasMetadataStore()); Path srcBaseDir = path("src"); mkdirs(srcBaseDir); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2220b787/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java index 0a9ee4f..22c17fb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java @@ -51,7 +51,7 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase { // This test will fail if NullMetadataStore (the default) is configured: // skip it. - Assume.assumeTrue(fs.isMetadataStoreConfigured()); + Assume.assumeTrue(fs.hasMetadataStore()); // Any S3 keys that contain DELAY_KEY_SUBSTRING will be delayed // in listObjects() results via InconsistentS3Client http://git-wip-us.apache.org/repos/asf/hadoop/blob/2220b787/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index fa55586..845f2e9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase; import org.junit.Assert; import org.junit.Assume; import org.junit.internal.AssumptionViolatedException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2220b787/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java index 62725d3..725646c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java @@ -16,23 +16,29 @@ package org.apache.hadoop.fs.s3a.fileContext; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContextURIBase; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; -import org.apache.hadoop.fs.s3a.s3guard.S3Guard; -import org.junit.Assume; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestFileSystem; + /** * S3a implementation of FileContextURIBase. */ public class ITestS3AFileContextURI extends FileContextURIBase { private Configuration conf; + private boolean hasMetadataStore; @Before public void setUp() throws IOException, Exception { conf = new Configuration(); + try(S3AFileSystem s3aFS = createTestFileSystem(conf)) { + hasMetadataStore = s3aFS.hasMetadataStore(); + } fc1 = S3ATestUtils.createTestFileContext(conf); fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS super.setUp(); @@ -48,7 +54,8 @@ public class ITestS3AFileContextURI extends FileContextURIBase { @Test @Override public void testModificationTime() throws IOException { - Assume.assumeTrue(S3Guard.isNullMetadataStoreConfigured(conf)); + // skip modtime tests as there may be some inconsistency during creation + assume("modification time tests are skipped", !hasMetadataStore); super.testModificationTime(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2220b787/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java index 6d77e32..03f1e22 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java @@ -113,7 +113,7 @@ public class ITestS3ADirectoryPerformance extends S3AScaleTestBase { listContinueRequests, listStatusCalls, getFileStatusCalls); - if (!fs.isMetadataStoreConfigured()) { + if (!fs.hasMetadataStore()) { assertEquals(listRequests.toString(), 2, listRequests.diff()); } reset(metadataRequests, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
