HADOOP-14918. Remove the Local Dynamo DB test option. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b089a067 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b089a067 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b089a067 Branch: refs/heads/HDFS-12943 Commit: b089a06793d94d42b7da1b7566e366ceb748e081 Parents: d6ee429 Author: Sean Mackrory <[email protected]> Authored: Wed Jun 20 16:10:36 2018 -0600 Committer: Sean Mackrory <[email protected]> Committed: Wed Jun 20 16:45:08 2018 -0600 ---------------------------------------------------------------------- hadoop-project/pom.xml | 5 - hadoop-tools/hadoop-aws/pom.xml | 42 +- .../org/apache/hadoop/fs/s3a/Constants.java | 11 + .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 1 + .../hadoop/fs/s3a/AbstractS3ATestBase.java | 21 +- .../apache/hadoop/fs/s3a/S3ATestConstants.java | 1 - .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 34 +- .../fs/s3a/commit/staging/StagingTestBase.java | 30 +- .../s3a/s3guard/DynamoDBLocalClientFactory.java | 160 ----- .../fs/s3a/s3guard/MetadataStoreTestBase.java | 4 +- .../s3a/s3guard/TestDynamoDBMetadataStore.java | 589 ------------------- 11 files changed, 73 insertions(+), 825 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8cb5bfc..ed0187b 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2060,10 +2060,5 @@ </profiles> <repositories> - <repository> - <id>dynamodb-local-oregon</id> - <name>DynamoDB Local Release Repository</name> - <url>https://s3-us-west-2.amazonaws.com/dynamodb-local/release</url> - </repository> </repositories> </project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 24ed11d..c6dddb0 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -36,7 +36,6 @@ <downloadSources>true</downloadSources> <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir> - <dynamodb.local.version>1.11.86</dynamodb.local.version> <!-- are scale tests enabled ? --> <fs.s3a.scale.test.enabled>unset</fs.s3a.scale.test.enabled> <!-- Size in MB of huge files. --> @@ -49,6 +48,8 @@ <fs.s3a.s3guard.test.enabled>false</fs.s3a.s3guard.test.enabled> <fs.s3a.s3guard.test.authoritative>false</fs.s3a.s3guard.test.authoritative> <fs.s3a.s3guard.test.implementation>local</fs.s3a.s3guard.test.implementation> + <!-- Set a longer timeout for integration test (in milliseconds) --> + <test.integration.timeout>200000</test.integration.timeout> </properties> @@ -162,6 +163,7 @@ <fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative> <fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation> + <test.default.timeout>${test.integration.timeout}</test.default.timeout> </systemPropertyVariables> <!-- Some tests cannot run in parallel. Tests that cover --> <!-- access to the root directory must run in isolation --> @@ -299,23 +301,10 @@ </properties> </profile> - <!-- Switch to DynamoDBLocal for S3Guard. Has no effect unless S3Guard is enabled --> - <profile> - <id>dynamodblocal</id> - <activation> - <property> - <name>dynamodblocal</name> - </property> - </activation> - <properties> - <fs.s3a.s3guard.test.implementation>dynamodblocal</fs.s3a.s3guard.test.implementation> - </properties> - </profile> - <!-- Switch S3Guard from Authoritative=false to true Has no effect unless S3Guard is enabled --> <profile> - <id>non-auth</id> + <id>auth</id> <activation> <property> <name>auth</name> @@ -346,6 +335,9 @@ <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds> + <systemPropertyVariables> + <test.default.timeout>${test.integration.timeout}</test.default.timeout> + </systemPropertyVariables> </configuration> </plugin> <plugin> @@ -418,26 +410,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>com.amazonaws</groupId> - <artifactId>DynamoDBLocal</artifactId> - <version>${dynamodb.local.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 4c95843..c521936 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -402,6 +402,17 @@ public final class Constants { "fs.s3a.s3guard.ddb.table"; /** + * Test table name to use during DynamoDB integration test. + * + * The table will be modified, and deleted in the end of the tests. + * If this value is not set, the integration tests that would be destructive + * won't run. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TEST_TABLE_NAME_KEY = + "fs.s3a.s3guard.ddb.test.table"; + + /** * Whether to create the DynamoDB table if the table does not exist. */ @InterfaceStability.Unstable http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java index b942ed7..0ab8696 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java @@ -261,6 +261,7 @@ public class DynamoDBMetadataStore implements MetadataStore { @Override @Retries.OnceRaw public void initialize(FileSystem fs) throws IOException { + Preconditions.checkNotNull(fs, "Null filesystem"); Preconditions.checkArgument(fs instanceof S3AFileSystem, "DynamoDBMetadataStore only supports S3A filesystem."); owner = (S3AFileSystem) fs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index 73e71f4..f22af49 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -29,13 +29,10 @@ import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED; /** * An extension of the contract test base set up for S3A tests. @@ -78,23 +75,7 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase */ @Override protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - // patch in S3Guard options - maybeEnableS3Guard(conf); - // set hadoop temp dir to a default value - String testUniqueForkId = - System.getProperty(TEST_UNIQUE_FORK_ID); - String tmpDir = conf.get(Constants.HADOOP_TMP_DIR, "target/build/test"); - if (testUniqueForkId != null) { - // patch temp dir for the specific branch - tmpDir = tmpDir + File.pathSeparatorChar + testUniqueForkId; - conf.set(Constants.HADOOP_TMP_DIR, tmpDir); - } - conf.set(Constants.BUFFER_DIR, tmpDir); - // add this so that even on tests where the FS is shared, - // the FS is always "magic" - conf.setBoolean(MAGIC_COMMITTER_ENABLED, true); - return conf; + return S3ATestUtils.prepareTestConfiguration(super.createConfiguration()); } protected Configuration getConfiguration() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 7f7802d..0f7b418 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -143,7 +143,6 @@ public interface S3ATestConstants { String TEST_S3GUARD_IMPLEMENTATION = TEST_S3GUARD_PREFIX + ".implementation"; String TEST_S3GUARD_IMPLEMENTATION_LOCAL = "local"; String TEST_S3GUARD_IMPLEMENTATION_DYNAMO = "dynamo"; - String TEST_S3GUARD_IMPLEMENTATION_DYNAMODBLOCAL = "dynamodblocal"; String TEST_S3GUARD_IMPLEMENTATION_NONE = "none"; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 4414746..d259bf1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -30,9 +30,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.s3a.commit.CommitConstants; -import org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory; -import org.apache.hadoop.fs.s3a.s3guard.DynamoDBLocalClientFactory; -import org.apache.hadoop.fs.s3a.s3guard.S3Guard; import org.hamcrest.core.Is; import org.junit.Assert; @@ -42,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; +import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -56,6 +54,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestConstants.*; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3AUtils.propagateBucketOptions; import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED; import static org.junit.Assert.*; /** @@ -393,9 +392,6 @@ public final class S3ATestUtils { case TEST_S3GUARD_IMPLEMENTATION_LOCAL: implClass = S3GUARD_METASTORE_LOCAL; break; - case TEST_S3GUARD_IMPLEMENTATION_DYNAMODBLOCAL: - conf.setClass(S3Guard.S3GUARD_DDB_CLIENT_FACTORY_IMPL, - DynamoDBLocalClientFactory.class, DynamoDBClientFactory.class); case TEST_S3GUARD_IMPLEMENTATION_DYNAMO: implClass = S3GUARD_METASTORE_DYNAMO; break; @@ -490,6 +486,32 @@ public final class S3ATestUtils { } /** + * Patch a configuration for testing. + * This includes possibly enabling s3guard, setting up the local + * FS temp dir and anything else needed for test runs. + * @param conf configuration to patch + * @return the now-patched configuration + */ + public static Configuration prepareTestConfiguration(final Configuration conf) { + // patch in S3Guard options + maybeEnableS3Guard(conf); + // set hadoop temp dir to a default value + String testUniqueForkId = + System.getProperty(TEST_UNIQUE_FORK_ID); + String tmpDir = conf.get(HADOOP_TMP_DIR, "target/build/test"); + if (testUniqueForkId != null) { + // patch temp dir for the specific branch + tmpDir = tmpDir + File.pathSeparatorChar + testUniqueForkId; + conf.set(HADOOP_TMP_DIR, tmpDir); + } + conf.set(BUFFER_DIR, tmpDir); + // add this so that even on tests where the FS is shared, + // the FS is always "magic" + conf.setBoolean(MAGIC_COMMITTER_ENABLED, true); + return conf; + } + + /** * Helper class to do diffs of metrics. */ public static final class MetricDiff { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java index 38d5156..d81c747 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java @@ -49,6 +49,7 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.mockito.invocation.InvocationOnMock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -515,6 +516,21 @@ public class StagingTestBase { } /** + * InvocationOnMock.getArgumentAt comes and goes with Mockito versions; this + * helper method is designed to be resilient to change. + * @param invocation invocation to query + * @param index argument index + * @param clazz class of return type + * @param <T> type of return + * @return the argument of the invocation, cast to the given type. + */ + @SuppressWarnings("unchecked") + private static<T> T getArgumentAt(InvocationOnMock invocation, int index, + Class<T> clazz) { + return (T)invocation.getArguments()[index]; + } + + /** * Instantiate mock client with the results and errors requested. * @param results results to accrue * @param errors when (if any) to fail @@ -539,7 +555,7 @@ public class StagingTestBase { "Mock Fail on init " + results.requests.size()); } String uploadId = UUID.randomUUID().toString(); - InitiateMultipartUploadRequest req = invocation.getArgumentAt( + InitiateMultipartUploadRequest req = getArgumentAt(invocation, 0, InitiateMultipartUploadRequest.class); results.requests.put(uploadId, req); results.activeUploads.put(uploadId, req.getKey()); @@ -561,7 +577,7 @@ public class StagingTestBase { throw new AmazonClientException( "Mock Fail on upload " + results.parts.size()); } - UploadPartRequest req = invocation.getArgumentAt( + UploadPartRequest req = getArgumentAt(invocation, 0, UploadPartRequest.class); results.parts.add(req); String etag = UUID.randomUUID().toString(); @@ -588,7 +604,7 @@ public class StagingTestBase { throw new AmazonClientException( "Mock Fail on commit " + results.commits.size()); } - CompleteMultipartUploadRequest req = invocation.getArgumentAt( + CompleteMultipartUploadRequest req = getArgumentAt(invocation, 0, CompleteMultipartUploadRequest.class); results.commits.add(req); results.activeUploads.remove(req.getUploadId()); @@ -608,7 +624,7 @@ public class StagingTestBase { throw new AmazonClientException( "Mock Fail on abort " + results.aborts.size()); } - AbortMultipartUploadRequest req = invocation.getArgumentAt( + AbortMultipartUploadRequest req = getArgumentAt(invocation, 0, AbortMultipartUploadRequest.class); String id = req.getUploadId(); String p = results.activeUploads.remove(id); @@ -630,7 +646,7 @@ public class StagingTestBase { doAnswer(invocation -> { LOG.debug("deleteObject for {}", mockClient); synchronized (lock) { - results.deletes.add(invocation.getArgumentAt( + results.deletes.add(getArgumentAt(invocation, 0, DeleteObjectRequest.class)); return null; } @@ -643,8 +659,8 @@ public class StagingTestBase { LOG.debug("deleteObject for {}", mockClient); synchronized (lock) { results.deletes.add(new DeleteObjectRequest( - invocation.getArgumentAt(0, String.class), - invocation.getArgumentAt(1, String.class) + getArgumentAt(invocation, 0, String.class), + getArgumentAt(invocation, 1, String.class) )); return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java deleted file mode 100644 index 9894ac4..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a.s3guard; - -import java.io.File; -import java.io.IOException; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; -import com.amazonaws.services.dynamodbv2.local.main.ServerRunner; -import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer; -import org.apache.commons.lang3.StringUtils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; -import org.apache.hadoop.net.ServerSocketUtil; - -import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; -import static org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet; -import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory.DefaultDynamoDBClientFactory.getRegion; - -/** - * A DynamoDBClientFactory implementation that creates AmazonDynamoDB clients - * against an in-memory DynamoDBLocal server instance. - * - * You won't be charged bills for issuing any DynamoDB requests. However, the - * DynamoDBLocal is considered a simulator of the DynamoDB web service, so it - * may be stale or different. For example, the throttling is not yet supported - * in DynamoDBLocal. This is for testing purpose only. - * - * To use this for creating DynamoDB client in tests: - * <ol> - * <li> - * As all DynamoDBClientFactory implementations, this should be configured. - * </li> - * <li> - * The singleton DynamoDBLocal server instance is started automatically when - * creating the AmazonDynamoDB client for the first time. It still merits to - * launch the server before all the tests and fail fast if error happens. - * </li> - * <li> - * The server can be stopped explicitly, which is not actually needed in - * tests as JVM termination will do that. - * </li> - * </ol> - * - * @see DefaultDynamoDBClientFactory - */ -public class DynamoDBLocalClientFactory extends Configured - implements DynamoDBClientFactory { - - /** The DynamoDBLocal dynamoDBLocalServer instance for testing. */ - private static DynamoDBProxyServer dynamoDBLocalServer; - private static String ddbEndpoint; - - private static final String SYSPROP_SQLITE_LIB = "sqlite4java.library.path"; - - @Override - public AmazonDynamoDB createDynamoDBClient(String defaultRegion) - throws IOException { - startSingletonServer(); - - final Configuration conf = getConf(); - // use the default credential provider chain - conf.unset(AWS_CREDENTIALS_PROVIDER); - final AWSCredentialsProvider credentials = - createAWSCredentialProviderSet(null, conf); - final ClientConfiguration awsConf = - DefaultS3ClientFactory.createAwsConf(conf); - // fail fast in case of service errors - awsConf.setMaxErrorRetry(3); - - final String region = getRegion(conf, defaultRegion); - LOG.info("Creating DynamoDBLocal client using endpoint {} in region {}", - ddbEndpoint, region); - - return AmazonDynamoDBClientBuilder.standard() - .withCredentials(credentials) - .withClientConfiguration(awsConf) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(ddbEndpoint, region)) - .build(); - } - - /** - * Start a singleton in-memory DynamoDBLocal server if not started yet. - * @throws IOException if any error occurs - */ - public synchronized static void startSingletonServer() throws IOException { - if (dynamoDBLocalServer != null) { - return; - } - - // Set this property if it has not been set elsewhere - if (StringUtils.isEmpty(System.getProperty(SYSPROP_SQLITE_LIB))) { - String projectBuildDir = System.getProperty("project.build.directory"); - if (StringUtils.isEmpty(projectBuildDir)) { - projectBuildDir = "target"; - } - // sqlite4java lib should have been copied to $projectBuildDir/native-libs - System.setProperty(SYSPROP_SQLITE_LIB, - projectBuildDir + File.separator + "native-libs"); - LOG.info("Setting {} -> {}", - SYSPROP_SQLITE_LIB, System.getProperty(SYSPROP_SQLITE_LIB)); - } - - try { - // Start an in-memory local DynamoDB instance - final String port = String.valueOf(ServerSocketUtil.getPort(0, 100)); - ddbEndpoint = "http://localhost:"; + port; - dynamoDBLocalServer = ServerRunner.createServerFromCommandLineArgs( - new String[]{"-inMemory", "-port", port}); - dynamoDBLocalServer.start(); - LOG.info("DynamoDBLocal singleton server was started at {}", ddbEndpoint); - } catch (Exception t) { - String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint - + " " + t; - LOG.error(msg, t); - throw new IOException(msg, t); - } - } - - /** - * Stop the in-memory DynamoDBLocal server if it is started. - * @throws IOException if any error occurs - */ - public synchronized static void stopSingletonServer() throws IOException { - if (dynamoDBLocalServer != null) { - LOG.info("Shutting down the in-memory DynamoDBLocal server"); - try { - dynamoDBLocalServer.stop(); - } catch (Throwable t) { - String msg = "Error stopping DynamoDBLocal server at " + ddbEndpoint; - LOG.error(msg, t); - throw new IOException(msg, t); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java index 806940b..56618cb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java @@ -28,7 +28,6 @@ import java.util.Map; import com.google.common.collect.Sets; import org.junit.After; -import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -43,6 +42,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.Tristate; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.test.HadoopTestBase; /** * Main test class for MetadataStore implementations. @@ -51,7 +51,7 @@ import org.apache.hadoop.io.IOUtils; * If your implementation may return missing results for recently set paths, * override {@link MetadataStoreTestBase#allowMissing()}. */ -public abstract class MetadataStoreTestBase extends Assert { +public abstract class MetadataStoreTestBase extends HadoopTestBase { private static final Logger LOG = LoggerFactory.getLogger(MetadataStoreTestBase.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b089a067/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java deleted file mode 100644 index 5763b83..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java +++ /dev/null @@ -1,589 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a.s3guard; - -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.dynamodbv2.document.DynamoDB; -import com.amazonaws.services.dynamodbv2.document.Item; -import com.amazonaws.services.dynamodbv2.document.PrimaryKey; -import com.amazonaws.services.dynamodbv2.document.Table; -import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription; -import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; -import com.amazonaws.services.dynamodbv2.model.TableDescription; - -import com.google.common.collect.Lists; -import org.apache.commons.collections.CollectionUtils; -import org.apache.hadoop.fs.s3a.Tristate; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.MockS3ClientFactory; -import org.apache.hadoop.fs.s3a.S3AFileStatus; -import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.fs.s3a.S3ClientFactory; -import org.apache.hadoop.security.UserGroupInformation; - -import static org.apache.hadoop.fs.s3a.Constants.*; -import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*; -import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*; -import static org.apache.hadoop.test.LambdaTestUtils.*; - -/** - * Test that {@link DynamoDBMetadataStore} implements {@link MetadataStore}. - * - * In this unit test, we use an in-memory DynamoDBLocal server instead of real - * AWS DynamoDB. An {@link S3AFileSystem} object is created and shared for - * initializing {@link DynamoDBMetadataStore} objects. There are no real S3 - * request issued as the underlying AWS S3Client is mocked. You won't be - * charged bills for AWS S3 or DynamoDB when you run this test. - * - * According to the base class, every test case will have independent contract - * to create a new {@link DynamoDBMetadataStore} instance and initializes it. - * A table will be created for each test by the test contract, and will be - * destroyed after the test case finishes. - */ -public class TestDynamoDBMetadataStore extends MetadataStoreTestBase { - private static final Logger LOG = - LoggerFactory.getLogger(TestDynamoDBMetadataStore.class); - private static final String BUCKET = "TestDynamoDBMetadataStore"; - private static final String S3URI = - URI.create(FS_S3A + "://" + BUCKET + "/").toString(); - public static final PrimaryKey - VERSION_MARKER_PRIMARY_KEY = createVersionMarkerPrimaryKey( - DynamoDBMetadataStore.VERSION_MARKER); - - /** The DynamoDB instance that can issue requests directly to server. */ - private static DynamoDB dynamoDB; - - @Rule - public final Timeout timeout = new Timeout(60 * 1000); - - /** - * Start the in-memory DynamoDBLocal server and initializes s3 file system. - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - DynamoDBLocalClientFactory.startSingletonServer(); - try { - dynamoDB = new DynamoDBMSContract().getMetadataStore().getDynamoDB(); - } catch (AmazonServiceException e) { - final String msg = "Cannot initialize a DynamoDBMetadataStore instance " - + "against the local DynamoDB server. Perhaps the DynamoDBLocal " - + "server is not configured correctly. "; - LOG.error(msg, e); - // fail fast if the DynamoDBLocal server can not work - throw e; - } - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - if (dynamoDB != null) { - dynamoDB.shutdown(); - } - DynamoDBLocalClientFactory.stopSingletonServer(); - } - - /** - * Each contract has its own S3AFileSystem and DynamoDBMetadataStore objects. - */ - private static class DynamoDBMSContract extends AbstractMSContract { - private final S3AFileSystem s3afs; - private final DynamoDBMetadataStore ms = new DynamoDBMetadataStore(); - - DynamoDBMSContract() throws IOException { - this(new Configuration()); - } - - DynamoDBMSContract(Configuration conf) throws IOException { - // using mocked S3 clients - conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, - S3ClientFactory.class); - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, S3URI); - // setting config for creating a DynamoDBClient against local server - conf.set(ACCESS_KEY, "dummy-access-key"); - conf.set(SECRET_KEY, "dummy-secret-key"); - conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true); - conf.setClass(S3Guard.S3GUARD_DDB_CLIENT_FACTORY_IMPL, - DynamoDBLocalClientFactory.class, DynamoDBClientFactory.class); - - // always create new file system object for a test contract - s3afs = (S3AFileSystem) FileSystem.newInstance(conf); - ms.initialize(s3afs); - } - - @Override - public S3AFileSystem getFileSystem() { - return s3afs; - } - - @Override - public DynamoDBMetadataStore getMetadataStore() { - return ms; - } - } - - @Override - public DynamoDBMSContract createContract() throws IOException { - return new DynamoDBMSContract(); - } - - @Override - public DynamoDBMSContract createContract(Configuration conf) throws - IOException { - return new DynamoDBMSContract(conf); - } - - @Override - FileStatus basicFileStatus(Path path, int size, boolean isDir) - throws IOException { - String owner = UserGroupInformation.getCurrentUser().getShortUserName(); - return isDir - ? new S3AFileStatus(true, path, owner) - : new S3AFileStatus(size, getModTime(), path, BLOCK_SIZE, owner); - } - - private DynamoDBMetadataStore getDynamoMetadataStore() throws IOException { - return (DynamoDBMetadataStore) getContract().getMetadataStore(); - } - - private S3AFileSystem getFileSystem() throws IOException { - return (S3AFileSystem) getContract().getFileSystem(); - } - - /** - * This tests that after initialize() using an S3AFileSystem object, the - * instance should have been initialized successfully, and tables are ACTIVE. - */ - @Test - public void testInitialize() throws IOException { - final String tableName = "testInitializeWithFileSystem"; - final S3AFileSystem s3afs = getFileSystem(); - final Configuration conf = s3afs.getConf(); - conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); - try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { - ddbms.initialize(s3afs); - verifyTableInitialized(tableName); - assertNotNull(ddbms.getTable()); - assertEquals(tableName, ddbms.getTable().getTableName()); - String expectedRegion = conf.get(S3GUARD_DDB_REGION_KEY, - s3afs.getBucketLocation(tableName)); - assertEquals("DynamoDB table should be in configured region or the same" + - " region as S3 bucket", - expectedRegion, - ddbms.getRegion()); - } - } - - /** - * This tests that after initialize() using a Configuration object, the - * instance should have been initialized successfully, and tables are ACTIVE. - */ - @Test - public void testInitializeWithConfiguration() throws IOException { - final String tableName = "testInitializeWithConfiguration"; - final Configuration conf = getFileSystem().getConf(); - conf.unset(S3GUARD_DDB_TABLE_NAME_KEY); - String savedRegion = conf.get(S3GUARD_DDB_REGION_KEY, - getFileSystem().getBucketLocation()); - conf.unset(S3GUARD_DDB_REGION_KEY); - try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { - ddbms.initialize(conf); - fail("Should have failed because the table name is not set!"); - } catch (IllegalArgumentException ignored) { - } - // config table name - conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); - try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { - ddbms.initialize(conf); - fail("Should have failed because as the region is not set!"); - } catch (IllegalArgumentException ignored) { - } - // config region - conf.set(S3GUARD_DDB_REGION_KEY, savedRegion); - try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { - ddbms.initialize(conf); - verifyTableInitialized(tableName); - assertNotNull(ddbms.getTable()); - assertEquals(tableName, ddbms.getTable().getTableName()); - assertEquals("Unexpected key schema found!", - keySchema(), - ddbms.getTable().describe().getKeySchema()); - } - } - - /** - * Test that for a large batch write request, the limit is handled correctly. - */ - @Test - public void testBatchWrite() throws IOException { - final int[] numMetasToDeleteOrPut = { - -1, // null - 0, // empty collection - 1, // one path - S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT, // exact limit of a batch request - S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT + 1 // limit + 1 - }; - for (int numOldMetas : numMetasToDeleteOrPut) { - for (int numNewMetas : numMetasToDeleteOrPut) { - doTestBatchWrite(numOldMetas, numNewMetas); - } - } - } - - private void doTestBatchWrite(int numDelete, int numPut) throws IOException { - final String root = S3URI + "/testBatchWrite_" + numDelete + '_' + numPut; - final Path oldDir = new Path(root, "oldDir"); - final Path newDir = new Path(root, "newDir"); - LOG.info("doTestBatchWrite: oldDir={}, newDir={}", oldDir, newDir); - - DynamoDBMetadataStore ms = getDynamoMetadataStore(); - ms.put(new PathMetadata(basicFileStatus(oldDir, 0, true))); - ms.put(new PathMetadata(basicFileStatus(newDir, 0, true))); - - final List<PathMetadata> oldMetas = - numDelete < 0 ? null : new ArrayList<PathMetadata>(numDelete); - for (int i = 0; i < numDelete; i++) { - oldMetas.add(new PathMetadata( - basicFileStatus(new Path(oldDir, "child" + i), i, true))); - } - final List<PathMetadata> newMetas = - numPut < 0 ? null : new ArrayList<PathMetadata>(numPut); - for (int i = 0; i < numPut; i++) { - newMetas.add(new PathMetadata( - basicFileStatus(new Path(newDir, "child" + i), i, false))); - } - - Collection<Path> pathsToDelete = null; - if (oldMetas != null) { - // put all metadata of old paths and verify - ms.put(new DirListingMetadata(oldDir, oldMetas, false)); - assertEquals(0, ms.listChildren(newDir).withoutTombstones().numEntries()); - assertTrue(CollectionUtils.isEqualCollection(oldMetas, - ms.listChildren(oldDir).getListing())); - - pathsToDelete = new ArrayList<>(oldMetas.size()); - for (PathMetadata meta : oldMetas) { - pathsToDelete.add(meta.getFileStatus().getPath()); - } - } - - // move the old paths to new paths and verify - ms.move(pathsToDelete, newMetas); - assertEquals(0, ms.listChildren(oldDir).withoutTombstones().numEntries()); - if (newMetas != null) { - assertTrue(CollectionUtils.isEqualCollection(newMetas, - ms.listChildren(newDir).getListing())); - } - } - - @Test - public void testInitExistingTable() throws IOException { - final DynamoDBMetadataStore ddbms = getDynamoMetadataStore(); - final String tableName = ddbms.getTable().getTableName(); - verifyTableInitialized(tableName); - // create existing table - ddbms.initTable(); - verifyTableInitialized(tableName); - } - - /** - * Test the low level version check code. - */ - @Test - public void testItemVersionCompatibility() throws Throwable { - verifyVersionCompatibility("table", - createVersionMarker(VERSION_MARKER, VERSION, 0)); - } - - /** - * Test that a version marker entry without the version number field - * is rejected as incompatible with a meaningful error message. - */ - @Test - public void testItemLacksVersion() throws Throwable { - intercept(IOException.class, E_NOT_VERSION_MARKER, - new VoidCallable() { - @Override - public void call() throws Exception { - verifyVersionCompatibility("table", - new Item().withPrimaryKey( - createVersionMarkerPrimaryKey(VERSION_MARKER))); - } - }); - } - - /** - * Delete the version marker and verify that table init fails. - */ - @Test - public void testTableVersionRequired() throws Exception { - Configuration conf = getFileSystem().getConf(); - int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES, - S3GUARD_DDB_MAX_RETRIES_DEFAULT); - conf.setInt(S3GUARD_DDB_MAX_RETRIES, 3); - - final DynamoDBMetadataStore ddbms = createContract(conf).getMetadataStore(); - String tableName = conf.get(S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); - Table table = verifyTableInitialized(tableName); - table.deleteItem(VERSION_MARKER_PRIMARY_KEY); - - // create existing table - intercept(IOException.class, E_NO_VERSION_MARKER, - new VoidCallable() { - @Override - public void call() throws Exception { - ddbms.initTable(); - } - }); - - conf.setInt(S3GUARD_DDB_MAX_RETRIES, maxRetries); - } - - /** - * Set the version value to a different number and verify that - * table init fails. - */ - @Test - public void testTableVersionMismatch() throws Exception { - final DynamoDBMetadataStore ddbms = createContract().getMetadataStore(); - String tableName = getFileSystem().getConf() - .get(S3GUARD_DDB_TABLE_NAME_KEY, BUCKET); - Table table = verifyTableInitialized(tableName); - table.deleteItem(VERSION_MARKER_PRIMARY_KEY); - Item v200 = createVersionMarker(VERSION_MARKER, 200, 0); - table.putItem(v200); - - // create existing table - intercept(IOException.class, E_INCOMPATIBLE_VERSION, - new VoidCallable() { - @Override - public void call() throws Exception { - ddbms.initTable(); - } - }); - } - - /** - * Test that initTable fails with IOException when table does not exist and - * table auto-creation is disabled. - */ - @Test - public void testFailNonexistentTable() throws IOException { - final String tableName = "testFailNonexistentTable"; - final S3AFileSystem s3afs = getFileSystem(); - final Configuration conf = s3afs.getConf(); - conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); - conf.unset(S3GUARD_DDB_TABLE_CREATE_KEY); - try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { - ddbms.initialize(s3afs); - fail("Should have failed as table does not exist and table auto-creation" - + " is disabled"); - } catch (IOException ignored) { - } - } - - /** - * Test cases about root directory as it is not in the DynamoDB table. - */ - @Test - public void testRootDirectory() throws IOException { - final DynamoDBMetadataStore ddbms = getDynamoMetadataStore(); - Path rootPath = new Path(S3URI); - verifyRootDirectory(ddbms.get(rootPath), true); - - ddbms.put(new PathMetadata(new S3AFileStatus(true, - new Path(rootPath, "foo"), - UserGroupInformation.getCurrentUser().getShortUserName()))); - verifyRootDirectory(ddbms.get(new Path(S3URI)), false); - } - - private void verifyRootDirectory(PathMetadata rootMeta, boolean isEmpty) { - assertNotNull(rootMeta); - final FileStatus status = rootMeta.getFileStatus(); - assertNotNull(status); - assertTrue(status.isDirectory()); - // UNKNOWN is always a valid option, but true / false should not contradict - if (isEmpty) { - assertNotSame("Should not be marked non-empty", - Tristate.FALSE, - rootMeta.isEmptyDirectory()); - } else { - assertNotSame("Should not be marked empty", - Tristate.TRUE, - rootMeta.isEmptyDirectory()); - } - } - - /** - * Test that when moving nested paths, all its ancestors up to destination - * root will also be created. - * Here is the directory tree before move: - * <pre> - * testMovePopulateAncestors - * âââ a - * â  âââ b - * â  âââ src - * â  âââ dir1 - * â  â  âââ dir2 - * â  âââ file1.txt - * âââ c - * âââ d - * âââ dest - *</pre> - * As part of rename(a/b/src, d/c/dest), S3A will enumerate the subtree at - * a/b/src. This test verifies that after the move, the new subtree at - * 'dest' is reachable from the root (i.e. c/ and c/d exist in the table. - * DynamoDBMetadataStore depends on this property to do recursive delete - * without a full table scan. - */ - @Test - public void testMovePopulatesAncestors() throws IOException { - final DynamoDBMetadataStore ddbms = getDynamoMetadataStore(); - final String testRoot = "/testMovePopulatesAncestors"; - final String srcRoot = testRoot + "/a/b/src"; - final String destRoot = testRoot + "/c/d/e/dest"; - - final Path nestedPath1 = strToPath(srcRoot + "/file1.txt"); - ddbms.put(new PathMetadata(basicFileStatus(nestedPath1, 1024, false))); - final Path nestedPath2 = strToPath(srcRoot + "/dir1/dir2"); - ddbms.put(new PathMetadata(basicFileStatus(nestedPath2, 0, true))); - - // We don't put the destRoot path here, since put() would create ancestor - // entries, and we want to ensure that move() does it, instead. - - // Build enumeration of src / dest paths and do the move() - final Collection<Path> fullSourcePaths = Lists.newArrayList( - strToPath(srcRoot), - strToPath(srcRoot + "/dir1"), - strToPath(srcRoot + "/dir1/dir2"), - strToPath(srcRoot + "/file1.txt") - ); - final Collection<PathMetadata> pathsToCreate = Lists.newArrayList( - new PathMetadata(basicFileStatus(strToPath(destRoot), - 0, true)), - new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1"), - 0, true)), - new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1/dir2"), - 0, true)), - new PathMetadata(basicFileStatus(strToPath(destRoot + "/file1.txt"), - 1024, false)) - ); - - ddbms.move(fullSourcePaths, pathsToCreate); - - // assert that all the ancestors should have been populated automatically - assertCached(testRoot + "/c"); - assertCached(testRoot + "/c/d"); - assertCached(testRoot + "/c/d/e"); - assertCached(destRoot /* /c/d/e/dest */); - - // Also check moved files while we're at it - assertCached(destRoot + "/dir1"); - assertCached(destRoot + "/dir1/dir2"); - assertCached(destRoot + "/file1.txt"); - } - - @Test - public void testProvisionTable() throws IOException { - final DynamoDBMetadataStore ddbms = getDynamoMetadataStore(); - final String tableName = ddbms.getTable().getTableName(); - final ProvisionedThroughputDescription oldProvision = - dynamoDB.getTable(tableName).describe().getProvisionedThroughput(); - ddbms.provisionTable(oldProvision.getReadCapacityUnits() * 2, - oldProvision.getWriteCapacityUnits() * 2); - final ProvisionedThroughputDescription newProvision = - dynamoDB.getTable(tableName).describe().getProvisionedThroughput(); - LOG.info("Old provision = {}, new provision = {}", - oldProvision, newProvision); - assertEquals(oldProvision.getReadCapacityUnits() * 2, - newProvision.getReadCapacityUnits().longValue()); - assertEquals(oldProvision.getWriteCapacityUnits() * 2, - newProvision.getWriteCapacityUnits().longValue()); - } - - @Test - public void testDeleteTable() throws Exception { - final String tableName = "testDeleteTable"; - final S3AFileSystem s3afs = getFileSystem(); - final Configuration conf = s3afs.getConf(); - conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); - try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { - ddbms.initialize(s3afs); - // we can list the empty table - ddbms.listChildren(new Path(S3URI)); - - ddbms.destroy(); - verifyTableNotExist(tableName); - - // delete table once more; be ResourceNotFoundException swallowed silently - ddbms.destroy(); - verifyTableNotExist(tableName); - try { - // we can no longer list the destroyed table - ddbms.listChildren(new Path(S3URI)); - fail("Should have failed after the table is destroyed!"); - } catch (IOException ignored) { - } - } - } - - /** - * This validates the table is created and ACTIVE in DynamoDB. - * - * This should not rely on the {@link DynamoDBMetadataStore} implementation. - * Return the table - */ - private static Table verifyTableInitialized(String tableName) { - final Table table = dynamoDB.getTable(tableName); - final TableDescription td = table.describe(); - assertEquals(tableName, td.getTableName()); - assertEquals("ACTIVE", td.getTableStatus()); - return table; - } - - /** - * This validates the table is not found in DynamoDB. - * - * This should not rely on the {@link DynamoDBMetadataStore} implementation. - */ - private static void verifyTableNotExist(String tableName) throws Exception{ - intercept(ResourceNotFoundException.class, - () -> dynamoDB.getTable(tableName).describe()); - } - -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
