HADOOP-13446. Support running isolated unit tests separate from AWS integration tests. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cda68de9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cda68de9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cda68de9 Branch: refs/heads/branch-2 Commit: cda68de9b913878210118a6637a1d0eb950c26ad Parents: a318ef7 Author: Chris Nauroth <cnaur...@apache.org> Authored: Tue Aug 23 07:19:20 2016 -0700 Committer: Chris Nauroth <cnaur...@apache.org> Committed: Tue Aug 23 07:19:20 2016 -0700 ---------------------------------------------------------------------- hadoop-project/pom.xml | 5 + hadoop-tools/hadoop-aws/pom.xml | 95 +++- .../src/site/markdown/tools/hadoop-aws/index.md | 67 ++- .../fs/contract/s3/ITestS3ContractCreate.java | 35 ++ .../fs/contract/s3/ITestS3ContractDelete.java | 35 ++ .../fs/contract/s3/ITestS3ContractMkdir.java | 35 ++ .../fs/contract/s3/ITestS3ContractOpen.java | 35 ++ .../fs/contract/s3/ITestS3ContractRename.java | 36 ++ .../fs/contract/s3/ITestS3ContractRootDir.java | 50 ++ .../fs/contract/s3/ITestS3ContractSeek.java | 44 ++ .../fs/contract/s3/TestS3ContractCreate.java | 33 -- .../fs/contract/s3/TestS3ContractDelete.java | 32 -- .../fs/contract/s3/TestS3ContractMkdir.java | 33 -- .../fs/contract/s3/TestS3ContractOpen.java | 33 -- .../fs/contract/s3/TestS3ContractRename.java | 33 -- .../fs/contract/s3/TestS3ContractRootDir.java | 50 -- .../fs/contract/s3/TestS3ContractSeek.java | 42 -- .../fs/contract/s3a/ITestS3AContractCreate.java | 35 ++ .../fs/contract/s3a/ITestS3AContractDelete.java | 34 ++ .../fs/contract/s3a/ITestS3AContractDistCp.java | 46 ++ .../s3a/ITestS3AContractGetFileStatus.java | 51 ++ .../fs/contract/s3a/ITestS3AContractMkdir.java | 34 ++ .../fs/contract/s3a/ITestS3AContractOpen.java | 34 ++ .../fs/contract/s3a/ITestS3AContractRename.java | 62 +++ .../contract/s3a/ITestS3AContractRootDir.java | 72 +++ .../fs/contract/s3a/ITestS3AContractSeek.java | 34 ++ .../fs/contract/s3a/TestS3AContractCreate.java | 33 -- .../fs/contract/s3a/TestS3AContractDelete.java | 31 -- .../fs/contract/s3a/TestS3AContractDistCp.java | 46 -- .../s3a/TestS3AContractGetFileStatus.java | 47 -- .../fs/contract/s3a/TestS3AContractMkdir.java | 34 -- .../fs/contract/s3a/TestS3AContractOpen.java | 31 -- .../fs/contract/s3a/TestS3AContractRename.java | 61 --- .../fs/contract/s3a/TestS3AContractRootDir.java | 72 --- .../fs/contract/s3a/TestS3AContractSeek.java | 31 -- .../fs/contract/s3n/ITestS3NContractCreate.java | 41 ++ .../fs/contract/s3n/ITestS3NContractDelete.java | 34 ++ .../fs/contract/s3n/ITestS3NContractMkdir.java | 34 ++ .../fs/contract/s3n/ITestS3NContractOpen.java | 34 ++ .../fs/contract/s3n/ITestS3NContractRename.java | 35 ++ .../contract/s3n/ITestS3NContractRootDir.java | 35 ++ .../fs/contract/s3n/ITestS3NContractSeek.java | 34 ++ .../fs/contract/s3n/TestS3NContractCreate.java | 38 -- .../fs/contract/s3n/TestS3NContractDelete.java | 31 -- .../fs/contract/s3n/TestS3NContractMkdir.java | 34 -- .../fs/contract/s3n/TestS3NContractOpen.java | 31 -- .../fs/contract/s3n/TestS3NContractRename.java | 32 -- .../fs/contract/s3n/TestS3NContractRootDir.java | 35 -- .../fs/contract/s3n/TestS3NContractSeek.java | 31 -- .../s3/ITestInMemoryS3FileSystemContract.java | 35 ++ .../fs/s3/TestInMemoryS3FileSystemContract.java | 32 -- .../ITestBlockingThreadPoolExecutorService.java | 182 +++++++ .../fs/s3a/ITestS3AAWSCredentialsProvider.java | 250 +++++++++ .../fs/s3a/ITestS3ABlockingThreadPool.java | 80 +++ .../apache/hadoop/fs/s3a/ITestS3ABlocksize.java | 96 ++++ .../hadoop/fs/s3a/ITestS3AConfiguration.java | 435 +++++++++++++++ .../hadoop/fs/s3a/ITestS3ACredentialsInURL.java | 155 ++++++ .../hadoop/fs/s3a/ITestS3AEncryption.java | 104 ++++ .../ITestS3AEncryptionAlgorithmPropagation.java | 83 +++ .../s3a/ITestS3AEncryptionFastOutputStream.java | 35 ++ .../hadoop/fs/s3a/ITestS3AFailureHandling.java | 192 +++++++ .../hadoop/fs/s3a/ITestS3AFastOutputStream.java | 74 +++ .../fs/s3a/ITestS3AFileOperationCost.java | 191 +++++++ .../fs/s3a/ITestS3AFileSystemContract.java | 106 ++++ .../fs/s3a/ITestS3ATemporaryCredentials.java | 148 +++++ .../TestBlockingThreadPoolExecutorService.java | 182 ------- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 250 --------- .../fs/s3a/TestS3ABlockingThreadPool.java | 80 --- .../apache/hadoop/fs/s3a/TestS3ABlocksize.java | 93 ---- .../hadoop/fs/s3a/TestS3AConfiguration.java | 429 --------------- .../hadoop/fs/s3a/TestS3ACredentialsInURL.java | 155 ------ .../apache/hadoop/fs/s3a/TestS3AEncryption.java | 104 ---- .../TestS3AEncryptionAlgorithmPropagation.java | 82 --- .../s3a/TestS3AEncryptionFastOutputStream.java | 35 -- .../hadoop/fs/s3a/TestS3AFailureHandling.java | 194 ------- .../hadoop/fs/s3a/TestS3AFastOutputStream.java | 74 --- .../hadoop/fs/s3a/TestS3AFileOperationCost.java | 191 ------- .../fs/s3a/TestS3AFileSystemContract.java | 104 ---- .../fs/s3a/TestS3ATemporaryCredentials.java | 148 ----- .../fs/s3a/fileContext/ITestS3AFileContext.java | 23 + .../ITestS3AFileContextCreateMkdir.java | 35 ++ .../ITestS3AFileContextMainOperations.java | 60 +++ .../ITestS3AFileContextStatistics.java | 61 +++ .../s3a/fileContext/ITestS3AFileContextURI.java | 44 ++ .../fileContext/ITestS3AFileContextUtil.java | 34 ++ .../fs/s3a/fileContext/TestS3AFileContext.java | 23 - .../TestS3AFileContextCreateMkdir.java | 35 -- .../TestS3AFileContextMainOperations.java | 60 --- .../TestS3AFileContextStatistics.java | 60 --- .../s3a/fileContext/TestS3AFileContextURI.java | 43 -- .../s3a/fileContext/TestS3AFileContextUtil.java | 34 -- .../s3a/scale/ITestS3ADeleteFilesOneByOne.java | 44 ++ .../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 135 +++++ .../s3a/scale/ITestS3ADirectoryPerformance.java | 187 +++++++ .../scale/ITestS3AInputStreamPerformance.java | 534 +++++++++++++++++++ .../s3a/scale/TestS3ADeleteFilesOneByOne.java | 40 -- .../fs/s3a/scale/TestS3ADeleteManyFiles.java | 135 ----- .../s3a/scale/TestS3ADirectoryPerformance.java | 187 ------- .../scale/TestS3AInputStreamPerformance.java | 534 ------------------- .../org/apache/hadoop/fs/s3a/yarn/ITestS3A.java | 88 +++ .../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 148 +++++ .../org/apache/hadoop/fs/s3a/yarn/TestS3A.java | 85 --- .../fs/s3a/yarn/TestS3AMiniYarnCluster.java | 144 ----- ...ITestInMemoryNativeS3FileSystemContract.java | 33 ++ .../ITestJets3tNativeFileSystemStore.java | 133 +++++ .../ITestJets3tNativeS3FileSystemContract.java | 33 ++ .../TestInMemoryNativeS3FileSystemContract.java | 30 -- .../TestJets3tNativeFileSystemStore.java | 129 ----- .../TestJets3tNativeS3FileSystemContract.java | 30 -- 109 files changed, 4808 insertions(+), 4632 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9c91a96..ebd1986 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1045,6 +1045,11 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>${maven-failsafe-plugin.version}</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> <version>${maven-install-plugin.version}</version> </plugin> http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index aada612..493e94e 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -46,7 +46,7 @@ </file> </activation> <properties> - <maven.test.skip>true</maven.test.skip> + <skipITs>true</skipITs> </properties> </profile> <profile> @@ -57,11 +57,16 @@ </file> </activation> <properties> - <maven.test.skip>false</maven.test.skip> + <skipITs>false</skipITs> </properties> </profile> <profile> <id>parallel-tests</id> + <activation> + <property> + <name>parallel-tests</name> + </property> + </activation> <build> <plugins> <plugin> @@ -96,12 +101,32 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <forkCount>${testsThreadCount}</forkCount> + <reuseForks>false</reuseForks> + <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> + <systemPropertyVariables> + <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> + <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> + <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> + + <!-- Due to a Maven quirk, setting this to just --> + <!-- surefire.forkNumber won't do the parameter --> + <!-- substitution. Putting a prefix in front of it like --> + <!-- "fork-" makes it work. --> + <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> <executions> <execution> - <id>default-test</id> - <phase>test</phase> + <id>default-integration-test</id> <goals> - <goal>test</goal> + <goal>integration-test</goal> + <goal>verify</goal> </goals> <configuration> <forkCount>${testsThreadCount}</forkCount> @@ -129,32 +154,35 @@ <!-- Exclude all of these tests from parallel execution, --> <!-- and instead run them sequentially in a separate --> <!-- Surefire execution step later. --> + <includes> + <include>**/ITest*.java</include> + </includes> <excludes> - <exclude>**/TestJets3tNativeS3FileSystemContract.java</exclude> - <exclude>**/TestS3ABlockingThreadPool.java</exclude> - <exclude>**/TestS3AFastOutputStream.java</exclude> - <exclude>**/TestS3AFileSystemContract.java</exclude> - <exclude>**/TestS3AMiniYarnCluster.java</exclude> - <exclude>**/Test*Root*.java</exclude> + <exclude>**/ITestJets3tNativeS3FileSystemContract.java</exclude> + <exclude>**/ITestS3ABlockingThreadPool.java</exclude> + <exclude>**/ITestS3AFastOutputStream.java</exclude> + <exclude>**/ITestS3AFileSystemContract.java</exclude> + <exclude>**/ITestS3AMiniYarnCluster.java</exclude> + <exclude>**/ITest*Root*.java</exclude> </excludes> </configuration> </execution> <execution> - <id>sequential-tests</id> - <phase>test</phase> + <id>sequential-integration-tests</id> <goals> - <goal>test</goal> + <goal>integration-test</goal> + <goal>verify</goal> </goals> <configuration> <!-- Do a sequential run for tests that cannot handle --> <!-- parallel execution. --> <includes> - <include>**/TestJets3tNativeS3FileSystemContract.java</include> - <include>**/TestS3ABlockingThreadPool.java</include> - <include>**/TestS3AFastOutputStream.java</include> - <include>**/TestS3AFileSystemContract.java</include> - <include>**/TestS3AMiniYarnCluster.java</include> - <include>**/Test*Root*.java</include> + <include>**/ITestJets3tNativeS3FileSystemContract.java</include> + <include>**/ITestS3ABlockingThreadPool.java</include> + <include>**/ITestS3AFastOutputStream.java</include> + <include>**/ITestS3AFileSystemContract.java</include> + <include>**/ITestS3AMiniYarnCluster.java</include> + <include>**/ITest*Root*.java</include> </includes> </configuration> </execution> @@ -163,6 +191,33 @@ </plugins> </build> </profile> + <profile> + <id>sequential-tests</id> + <activation> + <property> + <name>!parallel-tests</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + <configuration> + <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> </profiles> <build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 16b97c9..2959a3f 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1316,11 +1316,18 @@ works with S3 to something lower. See [AWS documentation](http://docs.aws.amazon ## Testing the S3 filesystem clients -Due to eventual consistency, tests may fail without reason. Transient -failures, which no longer occur upon rerunning the test, should thus be ignored. +This module includes both unit tests, which can run in isolation without +connecting to the S3 service, and integration tests, which require a working +connection to S3 to interact with a bucket. Unit test suites follow the naming +convention `Test*.java`. Integration tests follow the naming convention +`ITest*.java`. -To test the S3* filesystem clients, you need to provide two files -which pass in authentication details to the test runner +Due to eventual consistency, integration tests may fail without reason. +Transient failures, which no longer occur upon rerunning the test, should thus +be ignored. + +To integration test the S3* filesystem clients, you need to provide two files +which pass in authentication details to the test runner. 1. `auth-keys.xml` 1. `core-site.xml` @@ -1340,7 +1347,8 @@ need to apply a specific, non-default property change during the tests. The presence of this file triggers the testing of the S3 classes. -Without this file, *none of the tests in this module will be executed* +Without this file, *none of the integration tests in this module will be +executed*. The XML file must contain all the ID/key information needed to connect each of the filesystem clients to the object stores, and a URL for @@ -1527,23 +1535,50 @@ source code tree, it is not going to get accidentally committed. After completing the configuration, execute the test run through Maven. - mvn clean test + mvn clean verify -It's also possible to execute multiple test suites in parallel by enabling the -`parallel-tests` Maven profile. The tests spend most of their time blocked on -network I/O with the S3 service, so running in parallel tends to complete full -test runs faster. +It's also possible to execute multiple test suites in parallel by passing the +`parallel-tests` property on the command line. The tests spend most of their +time blocked on network I/O with the S3 service, so running in parallel tends to +complete full test runs faster. - mvn -Pparallel-tests clean test + mvn -Dparallel-tests clean verify Some tests must run with exclusive access to the S3 bucket, so even with the -`parallel-tests` profile enabled, several test suites will run in serial in a -separate Maven execution step after the parallel tests. +`parallel-tests` property, several test suites will run in serial in a separate +Maven execution step after the parallel tests. + +By default, `parallel-tests` runs 4 test suites concurrently. This can be tuned +by passing the `testsThreadCount` property. + + mvn -Dparallel-tests -DtestsThreadCount=8 clean verify + +To run just unit tests, which do not require S3 connectivity or AWS credentials, +use any of the above invocations, but switch the goal to `test` instead of +`verify`. + + mvn clean test + + mvn -Dparallel-tests clean test + + mvn -Dparallel-tests -DtestsThreadCount=8 clean test + +To run only a specific named subset of tests, pass the `test` property for unit +tests or the `it.test` property for integration tests. + + mvn clean test -Dtest=TestS3AInputPolicies + + mvn clean verify -Dit.test=ITestS3AFileContextStatistics -By default, the `parallel-tests` profile runs 4 test suites concurrently. This -can be tuned by passing the `testsThreadCount` argument. + mvn clean verify -Dtest=TestS3A* -Dit.test=ITestS3A* - mvn -Pparallel-tests -DtestsThreadCount=8 clean test +Note that when running a specific subset of tests, the patterns passed in `test` +and `it.test` override the configuration of which tests need to run in isolation +in a separate serial phase (mentioned above). This can cause unpredictable +results, so the recommendation is to avoid passing `parallel-tests` in +combination with `test` or `it.test`. If you know that you are specifying only +tests that can run safely in parallel, then it will work. For wide patterns, +like `ITestS3A*` shown above, it may cause unpredictable test failures. ### Testing against different regions http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractCreate.java new file mode 100644 index 0000000..e003f28 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractCreate.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3 contract tests creating files. + */ +@Deprecated +public class ITestS3ContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractDelete.java new file mode 100644 index 0000000..307e521 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractDelete.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3 contract tests covering deletes. + */ +@Deprecated +public class ITestS3ContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractMkdir.java new file mode 100644 index 0000000..3368393 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractMkdir.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test dir operations on S3. + */ +@Deprecated +public class ITestS3ContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractOpen.java new file mode 100644 index 0000000..c8cc998 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractOpen.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3 contract tests opening files. + */ +@Deprecated +public class ITestS3ContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRename.java new file mode 100644 index 0000000..e4dcb13 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRename.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3 contract tests covering rename. + */ +@Deprecated +public class ITestS3ContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRootDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRootDir.java new file mode 100644 index 0000000..8fd91e4 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractRootDir.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +import org.junit.Ignore; +import org.junit.Test; + +/** + * root dir operations against an S3 bucket. + */ +@Deprecated +public class ITestS3ContractRootDir extends AbstractContractRootDirectoryTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } + + @Override + @Test + @Ignore + public void testRmEmptyRootDirNonRecursive() { + } + + @Override + @Test + @Ignore + public void testRmRootRecursive() { + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractSeek.java new file mode 100644 index 0000000..67b16b9 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/ITestS3ContractSeek.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +import org.junit.Ignore; +import org.junit.Test; + +/** + * S3 contract tests covering file seek. + */ +@Deprecated +public class ITestS3ContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3Contract(conf); + } + + @Override + @Test + @Ignore + public void testReadFullyZeroByteFile() { + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java deleted file mode 100644 index 4e98db9..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractCreateTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.ContractTestUtils; - -@Deprecated -public class TestS3ContractCreate extends AbstractContractCreateTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3Contract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java deleted file mode 100644 index 37b4ed2..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -@Deprecated -public class TestS3ContractDelete extends AbstractContractDeleteTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3Contract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java deleted file mode 100644 index 1e6ac7f..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.ContractTestUtils; - -@Deprecated -public class TestS3ContractMkdir extends AbstractContractMkdirTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3Contract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java deleted file mode 100644 index 2af61de..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractOpenTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.ContractTestUtils; - -@Deprecated -public class TestS3ContractOpen extends AbstractContractOpenTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3Contract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java deleted file mode 100644 index ba45478..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractRenameTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -@Deprecated -public class TestS3ContractRename extends AbstractContractRenameTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3Contract(conf); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java deleted file mode 100644 index 4a22304..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -import org.junit.Ignore; -import org.junit.Test; - -/** - * root dir operations against an S3 bucket - */ -@Deprecated -public class TestS3ContractRootDir extends AbstractContractRootDirectoryTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3Contract(conf); - } - - @Override - @Test - @Ignore - public void testRmEmptyRootDirNonRecursive() { - } - - @Override - @Test - @Ignore - public void testRmRootRecursive() { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java deleted file mode 100644 index b15c0a2..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractSeekTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.ContractTestUtils; - -import org.junit.Ignore; -import org.junit.Test; - -@Deprecated -public class TestS3ContractSeek extends AbstractContractSeekTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3Contract(conf); - } - - @Override - @Test - @Ignore - public void testReadFullyZeroByteFile() { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java new file mode 100644 index 0000000..d2a858f --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3A contract tests creating files. + */ +public class ITestS3AContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java new file mode 100644 index 0000000..a47dcae --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3A contract tests covering deletes. + */ +public class ITestS3AContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java new file mode 100644 index 0000000..28278fe --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD; +import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; + +/** + * Contract test suite covering S3A integration with DistCp. + */ +public class ITestS3AContractDistCp extends AbstractContractDistCpTest { + + private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB + + @Override + protected Configuration createConfiguration() { + Configuration newConf = super.createConfiguration(); + newConf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_SETTING); + newConf.setLong(MULTIPART_SIZE, MULTIPART_SETTING); + return newConf; + } + + @Override + protected S3AContract createContract(Configuration conf) { + return new S3AContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java new file mode 100644 index 0000000..c7ed5a3 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.s3a.Constants; +import org.apache.hadoop.fs.s3a.S3ATestUtils; + +/** + * S3A contract tests covering getFileStatus. + */ +public class ITestS3AContractGetFileStatus + extends AbstractContractGetFileStatusTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } + + @Override + public void teardown() throws Exception { + getLog().info("FS details {}", getFileSystem()); + super.teardown(); + } + + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + S3ATestUtils.disableFilesystemCaching(conf); + // aggressively low page size forces tests to go multipage + conf.setInt(Constants.MAX_PAGING_KEYS, 2); + return conf; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java new file mode 100644 index 0000000..d953e7e --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test dir operations on S3A. + */ +public class ITestS3AContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java new file mode 100644 index 0000000..a7bdc0d --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3A contract tests opening files. + */ +public class ITestS3AContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java new file mode 100644 index 0000000..5dba03d --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; + +/** + * S3A contract tests covering rename. + */ +public class ITestS3AContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } + + @Override + public void testRenameDirIntoExistingDir() throws Throwable { + describe("Verify renaming a dir into an existing dir puts the files" + +" from the source dir into the existing dir" + +" and leaves existing files alone"); + FileSystem fs = getFileSystem(); + String sourceSubdir = "source"; + Path srcDir = path(sourceSubdir); + Path srcFilePath = new Path(srcDir, "source-256.txt"); + byte[] srcDataset = dataset(256, 'a', 'z'); + writeDataset(fs, srcFilePath, srcDataset, srcDataset.length, 1024, false); + Path destDir = path("dest"); + + Path destFilePath = new Path(destDir, "dest-512.txt"); + byte[] destDateset = dataset(512, 'A', 'Z'); + writeDataset(fs, destFilePath, destDateset, destDateset.length, 1024, + false); + assertIsFile(destFilePath); + + boolean rename = fs.rename(srcDir, destDir); + assertFalse("s3a doesn't support rename to non-empty directory", rename); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java new file mode 100644 index 0000000..8383a77 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * root dir operations against an S3 bucket. + */ +public class ITestS3AContractRootDir extends + AbstractContractRootDirectoryTest { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestS3AContractRootDir.class); + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } + + @Override + public void testListEmptyRootDirectory() throws IOException { + for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) { + try { + super.testListEmptyRootDirectory(); + break; + } catch (AssertionError | FileNotFoundException e) { + if (attempt < maxAttempts) { + LOG.info("Attempt {} of {} for empty root directory test failed. " + + "This is likely caused by eventual consistency of S3 " + + "listings. Attempting retry.", attempt, maxAttempts); + try { + Thread.sleep(1000); + } catch (InterruptedException e2) { + Thread.currentThread().interrupt(); + fail("Test interrupted."); + break; + } + } else { + LOG.error( + "Empty root directory test failed {} attempts. Failing test.", + maxAttempts); + throw e; + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java new file mode 100644 index 0000000..1572fbc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3a; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * S3A contract tests covering file seek. + */ +public class ITestS3AContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java deleted file mode 100644 index 035252b..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractCreateTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.ContractTestUtils; - -public class TestS3AContractCreate extends AbstractContractCreateTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java deleted file mode 100644 index 733a517..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestS3AContractDelete extends AbstractContractDeleteTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java deleted file mode 100644 index 7eb0afa..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD; -import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; - -/** - * Contract test suite covering S3A integration with DistCp. - */ -public class TestS3AContractDistCp extends AbstractContractDistCpTest { - - private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB - - @Override - protected Configuration createConfiguration() { - Configuration newConf = super.createConfiguration(); - newConf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_SETTING); - newConf.setLong(MULTIPART_SIZE, MULTIPART_SETTING); - return newConf; - } - - @Override - protected S3AContract createContract(Configuration conf) { - return new S3AContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java deleted file mode 100644 index 5937d49..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs.contract.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; -import org.apache.hadoop.fs.s3a.Constants; -import org.apache.hadoop.fs.s3a.S3ATestUtils; - -public class TestS3AContractGetFileStatus extends AbstractContractGetFileStatusTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - - @Override - public void teardown() throws Exception { - getLog().info("FS details {}", getFileSystem()); - super.teardown(); - } - - @Override - protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - S3ATestUtils.disableFilesystemCaching(conf); - // aggressively low page size forces tests to go multipage - conf.setInt(Constants.MAX_PAGING_KEYS, 2); - return conf; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java deleted file mode 100644 index a312782..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -/** - * Test dir operations on S3 - */ -public class TestS3AContractMkdir extends AbstractContractMkdirTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java deleted file mode 100644 index f735deb..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractOpenTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestS3AContractOpen extends AbstractContractOpenTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java deleted file mode 100644 index af1ed37..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractRenameTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.junit.Test; - -import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; -import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; - -public class TestS3AContractRename extends AbstractContractRenameTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - - @Override - public void testRenameDirIntoExistingDir() throws Throwable { - describe("Verify renaming a dir into an existing dir puts the files" - +" from the source dir into the existing dir" - +" and leaves existing files alone"); - FileSystem fs = getFileSystem(); - String sourceSubdir = "source"; - Path srcDir = path(sourceSubdir); - Path srcFilePath = new Path(srcDir, "source-256.txt"); - byte[] srcDataset = dataset(256, 'a', 'z'); - writeDataset(fs, srcFilePath, srcDataset, srcDataset.length, 1024, false); - Path destDir = path("dest"); - - Path destFilePath = new Path(destDir, "dest-512.txt"); - byte[] destDateset = dataset(512, 'A', 'Z'); - writeDataset(fs, destFilePath, destDateset, destDateset.length, 1024, - false); - assertIsFile(destFilePath); - - boolean rename = fs.rename(srcDir, destDir); - assertFalse("s3a doesn't support rename to non-empty directory", rename); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java deleted file mode 100644 index 1cf1eb8..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import java.io.FileNotFoundException; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * root dir operations against an S3 bucket. - */ -public class TestS3AContractRootDir extends - AbstractContractRootDirectoryTest { - - private static final Logger LOG = - LoggerFactory.getLogger(TestS3AContractRootDir.class); - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - - @Override - public void testListEmptyRootDirectory() throws IOException { - for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) { - try { - super.testListEmptyRootDirectory(); - break; - } catch (AssertionError | FileNotFoundException e) { - if (attempt < maxAttempts) { - LOG.info("Attempt {} of {} for empty root directory test failed. " - + "This is likely caused by eventual consistency of S3 " - + "listings. Attempting retry.", attempt, maxAttempts); - try { - Thread.sleep(1000); - } catch (InterruptedException e2) { - Thread.currentThread().interrupt(); - fail("Test interrupted."); - break; - } - } else { - LOG.error( - "Empty root directory test failed {} attempts. Failing test.", - maxAttempts); - throw e; - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/cda68de9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java deleted file mode 100644 index d677ec4..0000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.contract.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractSeekTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestS3AContractSeek extends AbstractContractSeekTest { - - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org