HADOOP-14553. Add (parallelized) integration tests to hadoop-azure Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d2d97fa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d2d97fa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d2d97fa Branch: refs/heads/YARN-1011 Commit: 2d2d97fa7d4224369b3c13bc4a45e8cc9e29afb1 Parents: 11390c2 Author: Steve Loughran <ste...@apache.org> Authored: Fri Sep 15 17:03:01 2017 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Sep 15 17:03:01 2017 +0100 ---------------------------------------------------------------------- .../hadoop/fs/FileSystemContractBaseTest.java | 11 +- .../fs/contract/AbstractContractOpenTest.java | 4 +- .../fs/contract/AbstractContractSeekTest.java | 2 +- hadoop-tools/hadoop-azure/pom.xml | 251 ++++++ .../fs/azure/AzureNativeFileSystemStore.java | 2 +- .../hadoop-azure/src/site/markdown/index.md | 94 +- .../src/site/markdown/testing_azure.md | 576 ++++++++++++ .../hadoop/fs/azure/AbstractWasbTestBase.java | 136 ++- .../fs/azure/AbstractWasbTestWithTimeout.java | 73 ++ .../fs/azure/AzureBlobStorageTestAccount.java | 42 +- .../azure/ITestAzureConcurrentOutOfBandIo.java | 179 ++++ ...zureConcurrentOutOfBandIoWithSecureMode.java | 33 + .../ITestAzureFileSystemErrorConditions.java | 243 +++++ .../fs/azure/ITestBlobDataValidation.java | 244 ++++++ .../fs/azure/ITestBlobTypeSpeedDifference.java | 163 ++++ .../fs/azure/ITestBlockBlobInputStream.java | 874 ++++++++++++++++++ .../hadoop/fs/azure/ITestContainerChecks.java | 194 ++++ ...estFileSystemOperationExceptionHandling.java | 283 ++++++ ...TestFileSystemOperationExceptionMessage.java | 79 ++ ...perationsExceptionHandlingMultiThreaded.java | 366 ++++++++ .../ITestFileSystemOperationsWithThreads.java | 821 +++++++++++++++++ ...stNativeAzureFSAuthWithBlobSpecificKeys.java | 40 + .../ITestNativeAzureFSAuthorizationCaching.java | 53 ++ .../azure/ITestNativeAzureFSPageBlobLive.java | 43 + .../azure/ITestNativeAzureFileSystemAppend.java | 350 ++++++++ ...ativeAzureFileSystemAtomicRenameDirList.java | 55 ++ ...veAzureFileSystemAuthorizationWithOwner.java | 122 +++ ...ITestNativeAzureFileSystemClientLogging.java | 136 +++ ...estNativeAzureFileSystemConcurrencyLive.java | 185 ++++ ...stNativeAzureFileSystemContractEmulator.java | 65 ++ .../ITestNativeAzureFileSystemContractLive.java | 108 +++ ...tiveAzureFileSystemContractPageBlobLive.java | 114 +++ .../azure/ITestNativeAzureFileSystemLive.java | 236 +++++ .../ITestOutOfBandAzureBlobOperationsLive.java | 185 ++++ .../ITestReadAndSeekPageBlobAfterWrite.java | 341 ++++++++ .../fs/azure/ITestWasbRemoteCallHelper.java | 568 ++++++++++++ .../fs/azure/ITestWasbUriAndConfiguration.java | 610 +++++++++++++ .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 103 ++- .../fs/azure/NativeAzureFileSystemBaseTest.java | 115 ++- .../hadoop/fs/azure/RunningLiveWasbTests.txt | 22 - .../azure/TestAzureConcurrentOutOfBandIo.java | 195 ----- ...zureConcurrentOutOfBandIoWithSecureMode.java | 50 -- .../TestAzureFileSystemErrorConditions.java | 244 ------ .../hadoop/fs/azure/TestBlobDataValidation.java | 237 ----- .../hadoop/fs/azure/TestBlobMetadata.java | 7 +- .../fs/azure/TestBlobOperationDescriptor.java | 3 - .../fs/azure/TestBlobTypeSpeedDifference.java | 160 ---- .../fs/azure/TestBlockBlobInputStream.java | 875 ------------------- .../fs/azure/TestClientThrottlingAnalyzer.java | 5 +- .../hadoop/fs/azure/TestContainerChecks.java | 185 ---- ...estFileSystemOperationExceptionHandling.java | 269 ------ ...TestFileSystemOperationExceptionMessage.java | 79 -- ...perationsExceptionHandlingMultiThreaded.java | 330 ------- .../TestFileSystemOperationsWithThreads.java | 821 ----------------- ...stNativeAzureFSAuthWithBlobSpecificKeys.java | 44 - .../TestNativeAzureFSAuthorizationCaching.java | 60 -- .../fs/azure/TestNativeAzureFSPageBlobLive.java | 43 - .../azure/TestNativeAzureFileSystemAppend.java | 362 -------- ...ativeAzureFileSystemAtomicRenameDirList.java | 50 -- .../TestNativeAzureFileSystemAuthorization.java | 53 +- ...veAzureFileSystemAuthorizationWithOwner.java | 122 --- ...TestNativeAzureFileSystemBlockLocations.java | 8 +- .../TestNativeAzureFileSystemClientLogging.java | 140 --- .../TestNativeAzureFileSystemConcurrency.java | 29 +- ...estNativeAzureFileSystemConcurrencyLive.java | 184 ---- ...stNativeAzureFileSystemContractEmulator.java | 48 - .../TestNativeAzureFileSystemContractLive.java | 80 -- ...TestNativeAzureFileSystemContractMocked.java | 3 + ...tiveAzureFileSystemContractPageBlobLive.java | 93 -- .../TestNativeAzureFileSystemFileNameCheck.java | 28 +- .../fs/azure/TestNativeAzureFileSystemLive.java | 242 ----- .../azure/TestNativeAzureFileSystemMocked.java | 4 + .../TestNativeAzureFileSystemUploadLogic.java | 78 +- .../azure/TestOutOfBandAzureBlobOperations.java | 8 +- .../TestOutOfBandAzureBlobOperationsLive.java | 203 ----- .../TestReadAndSeekPageBlobAfterWrite.java | 355 -------- .../azure/TestShellDecryptionKeyProvider.java | 15 +- .../apache/hadoop/fs/azure/TestWasbFsck.java | 9 +- .../fs/azure/TestWasbRemoteCallHelper.java | 569 ------------ .../fs/azure/TestWasbUriAndConfiguration.java | 617 ------------- .../ITestAzureNativeContractAppend.java | 41 + .../ITestAzureNativeContractCreate.java | 34 + .../ITestAzureNativeContractDelete.java | 33 + .../ITestAzureNativeContractDistCp.java | 47 + .../ITestAzureNativeContractGetFileStatus.java | 35 + .../contract/ITestAzureNativeContractMkdir.java | 33 + .../contract/ITestAzureNativeContractOpen.java | 34 + .../ITestAzureNativeContractRename.java | 34 + .../contract/ITestAzureNativeContractSeek.java | 34 + .../contract/NativeAzureFileSystemContract.java | 19 +- .../contract/TestAzureNativeContractAppend.java | 37 - .../contract/TestAzureNativeContractCreate.java | 30 - .../contract/TestAzureNativeContractDelete.java | 30 - .../contract/TestAzureNativeContractDistCp.java | 33 - .../TestAzureNativeContractGetFileStatus.java | 30 - .../contract/TestAzureNativeContractMkdir.java | 30 - .../contract/TestAzureNativeContractOpen.java | 30 - .../contract/TestAzureNativeContractRename.java | 30 - .../contract/TestAzureNativeContractSeek.java | 30 - .../integration/AbstractAzureScaleTest.java | 66 ++ .../azure/integration/AzureTestConstants.java | 180 ++++ .../fs/azure/integration/AzureTestUtils.java | 479 ++++++++++ .../integration/CleanupTestContainers.java | 87 ++ .../azure/integration/ITestAzureHugeFiles.java | 456 ++++++++++ .../hadoop/fs/azure/integration/Sizes.java | 43 + .../ITestAzureFileSystemInstrumentation.java | 586 +++++++++++++ .../TestAzureFileSystemInstrumentation.java | 579 ------------ 107 files changed, 10227 insertions(+), 7901 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index b49dd53..a4ccee3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@ -61,7 +61,16 @@ public abstract class FileSystemContractBaseTest { protected byte[] data = dataset(getBlockSize() * 2, 0, 255); @Rule - public Timeout globalTimeout = new Timeout(30000); + public Timeout globalTimeout = new Timeout(getGlobalTimeout()); + + /** + * Get the timeout in milliseconds for each test case. + * @return a time in milliseconds. + */ + protected int getGlobalTimeout() { + return 30 * 1000; + } + @Rule public ExpectedException thrown = ExpectedException.none(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java index f9b16f4..ccf188f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java @@ -122,7 +122,7 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas Path path = path("testopenfiletwice.txt"); byte[] block = dataset(TEST_FILE_LEN, 0, 255); //this file now has a simple rule: offset => value - createFile(getFileSystem(), path, false, block); + createFile(getFileSystem(), path, true, block); //open first FSDataInputStream instream1 = getFileSystem().open(path); FSDataInputStream instream2 = null; @@ -150,7 +150,7 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas int base = 0x40; // 64 byte[] block = dataset(len, base, base + len); //this file now has a simple rule: offset => (value | 0x40) - createFile(getFileSystem(), path, false, block); + createFile(getFileSystem(), path, true, block); //open first instream = getFileSystem().open(path); assertEquals(base, instream.read()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java index 3e71682..7af3cb0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java @@ -341,7 +341,7 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas int filesize = 10 * 1024; byte[] buf = dataset(filesize, 0, 255); Path randomSeekFile = path("testrandomseeks.bin"); - createFile(getFileSystem(), randomSeekFile, false, buf); + createFile(getFileSystem(), randomSeekFile, true, buf); Random r = new Random(); // Record the sequence of seeks and reads which trigger a failure. http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 0c5ac63..b479872 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -34,6 +34,15 @@ <properties> <file.encoding>UTF-8</file.encoding> <downloadSources>true</downloadSources> + <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir> + <!-- are scale tests enabled ? --> + <fs.azure.scale.test.enabled>unset</fs.azure.scale.test.enabled> + <!-- Size in MB of huge files. --> + <fs.azure.scale.test.huge.filesize>unset</fs.azure.scale.test.huge.filesize> + <!-- Size in MB of the partion size in huge file uploads. --> + <fs.azure.scale.test.huge.partitionsize>unset</fs.azure.scale.test.huge.partitionsize> + <!-- Timeout in seconds for scale tests.--> + <fs.azure.scale.test.timeout>7200</fs.azure.scale.test.timeout> </properties> <build> @@ -224,4 +233,246 @@ </dependency> </dependencies> + + <profiles> + <profile> + <id>parallel-tests</id> + <activation> + <property> + <name>parallel-tests</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>create-parallel-tests-dirs</id> + <phase>test-compile</phase> + <configuration> + <target> + <script language="javascript"><![CDATA[ + var baseDirs = [ + project.getProperty("test.build.data"), + project.getProperty("test.build.dir"), + project.getProperty("hadoop.tmp.dir") + ]; + for (var i in baseDirs) { + for (var j = 1; j <= ${testsThreadCount}; ++j) { + var mkdir = project.createTask("mkdir"); + mkdir.setDir(new java.io.File(baseDirs[i], j)); + mkdir.perform(); + } + } + ]]></script> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <executions> + <execution> + <id>default-test</id> + <goals> + <goal>test</goal> + </goals> + <configuration> + <forkCount>1</forkCount> + <forkCount>${testsThreadCount}</forkCount> + <reuseForks>false</reuseForks> + <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> + <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds> + <systemPropertyVariables> + <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> + <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> + <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> + <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id> + <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled> + <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize> + <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize> + <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout> + </systemPropertyVariables> + <includes> + <include>**/Test*.java</include> + </includes> + <excludes> + <exclude>**/TestRollingWindowAverage*.java</exclude> + </excludes> + </configuration> + </execution> + <execution> + <id>serialized-test</id> + <goals> + <goal>test</goal> + </goals> + <configuration> + <forkCount>1</forkCount> + <reuseForks>false</reuseForks> + <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> + <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds> + <systemPropertyVariables> + <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> + <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> + <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> + <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id> + <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled> + <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize> + <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize> + <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout> + </systemPropertyVariables> + <includes> + <include>**/TestRollingWindowAverage*.java</include> + </includes> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <id>default-integration-test</id> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + <configuration> + <forkCount>${testsThreadCount}</forkCount> + <reuseForks>false</reuseForks> + <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> + <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds> + <systemPropertyVariables> + <!-- Tell tests that they are being executed in parallel --> + <test.parallel.execution>true</test.parallel.execution> + <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> + <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> + <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> + + <!-- Due to a Maven quirk, setting this to just --> + <!-- surefire.forkNumber won't do the parameter --> + <!-- substitution. Putting a prefix in front of it like --> + <!-- "fork-" makes it work. --> + <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id> + <!-- Propagate scale parameters --> + <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled> + <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize> + <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize> + <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout> + </systemPropertyVariables> + <!-- Some tests cannot run in parallel. Tests that cover --> + <!-- access to the root directory must run in isolation --> + <!-- from anything else that could modify the bucket. --> + <!-- azure tests that cover multi-part upload must run in --> + <!-- isolation, because the file system is configured to --> + <!-- purge existing multi-part upload data on --> + <!-- initialization. MiniYARNCluster has not yet been --> + <!-- changed to handle parallel test execution gracefully. --> + <!-- Exclude all of these tests from parallel execution, --> + <!-- and instead run them sequentially in a separate --> + <!-- Surefire execution step later. --> + <includes> + <include>**/ITest*.java</include> + </includes> + <excludes> + <exclude>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</exclude> + <exclude>**/ITestFileSystemOperationsWithThreads.java</exclude> + <exclude>**/ITestOutOfBandAzureBlobOperationsLive.java</exclude> + <exclude>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</exclude> + <exclude>**/ITestNativeAzureFileSystemConcurrencyLive.java</exclude> + <exclude>**/ITestNativeAzureFileSystemLive.java</exclude> + <exclude>**/ITestNativeAzureFSPageBlobLive.java</exclude> + <exclude>**/ITestWasbRemoteCallHelper.java</exclude> + <exclude>**/ITestBlockBlobInputStream.java</exclude> + </excludes> + </configuration> + </execution> + <!-- Do a sequential run for tests that cannot handle --> + <!-- parallel execution. --> + <execution> + <id>sequential-integration-tests</id> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + <configuration> + <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds> + <systemPropertyVariables> + <test.parallel.execution>false</test.parallel.execution> + <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled> + <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize> + <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize> + <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout> + </systemPropertyVariables> + <includes> + <include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include> + <include>**/ITestFileSystemOperationsWithThreads.java</include> + <include>**/ITestOutOfBandAzureBlobOperationsLive.java</include> + <include>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</include> + <include>**/ITestNativeAzureFileSystemConcurrencyLive.java</include> + <include>**/ITestNativeAzureFileSystemLive.java</include> + <include>**/ITestWasbRemoteCallHelper.java</include> + <include>**/ITestBlockBlobInputStream.java</include> + </includes> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>sequential-tests</id> + <activation> + <property> + <name>!parallel-tests</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + <configuration> + <systemPropertyVariables> + <!-- Propagate scale parameters --> + <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled> + <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize> + <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout> + </systemPropertyVariables> + <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <!-- Turn on scale tests--> + <profile> + <id>scale</id> + <activation> + <property> + <name>scale</name> + </property> + </activation> + <properties> + <fs.azure.scale.test.enabled>true</fs.azure.scale.test.enabled> + </properties> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 639862f..f1031b4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -346,7 +346,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private String delegationToken; /** The error message template when container is not accessible. */ - static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for " + public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for " + "account %s in the configuration, and its container %s is not " + "accessible using anonymous credentials. Please check if the container " + "exists first. If it is not publicly available, you have to provide " http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index 466bf0b..876d7cc 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -519,96 +519,8 @@ The maximum number of entries that that cache can hold can be customized using t <value>true</value> </property> ``` -## Testing the hadoop-azure Module -The hadoop-azure module includes a full suite of unit tests. Most of the tests -will run without additional configuration by running `mvn test`. This includes -tests against mocked storage, which is an in-memory emulation of Azure Storage. - -A selection of tests can run against the -[Azure Storage Emulator](http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx) -which is a high-fidelity emulation of live Azure Storage. The emulator is -sufficient for high-confidence testing. The emulator is a Windows executable -that runs on a local machine. - -To use the emulator, install Azure SDK 2.3 and start the storage emulator. Then, -edit `src/test/resources/azure-test.xml` and add the following property: - -```xml -<property> - <name>fs.azure.test.emulator</name> - <value>true</value> -</property> -``` - -There is a known issue when running tests with the emulator. You may see the -following failure message: - - com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format. - -To resolve this, restart the Azure Emulator. Ensure it v3.2 or later. - -It's also possible to run tests against a live Azure Storage account by saving a -file to `src/test/resources/azure-auth-keys.xml` and setting -the name of the storage account and its access key. - -For example: - -```xml -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<configuration> - <property> - <name>fs.azure.test.account.name</name> - <value>{ACCOUNTNAME}.blob.core.windows.net</value> - </property> - <property> - <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name> - <value>{ACCOUNT ACCESS KEY}</value> - </property> -</configuration> -``` - -To run contract tests, set the WASB file system URI in `src/test/resources/azure-auth-keys.xml` -and the account access key. For example: - -```xml -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<configuration> - <property> - <name>fs.contract.test.fs.wasb</name> - <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value> - <description>The name of the azure file system for testing.</description> - </property> - <property> - <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name> - <value>{ACCOUNT ACCESS KEY}</value> - </property> -</configuration> -``` - -Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml` is like following: - -```xml -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<configuration> - <property> - <name>fs.azure.test.account.name</name> - <value>{ACCOUNTNAME}.blob.core.windows.net</value> - </property> - <property> - <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name> - <value>{ACCOUNT ACCESS KEY}</value> - </property> - <property> - <name>fs.contract.test.fs.wasb</name> - <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value> - </property> -</configuration> -``` - -DO NOT ADD `azure-auth-keys.xml` TO REVISION CONTROL. The keys to your Azure -Storage account are a secret and must not be shared. +## Further Reading +* [Testing the Azure WASB client](testing_azure.html). +* MSDN article, [Understanding Block Blobs, Append Blobs, and Page Blobs](https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs) http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md new file mode 100644 index 0000000..b58e68b --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md @@ -0,0 +1,576 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Testing the Azure WASB client + +<!-- MACRO{toc|fromDepth=0|toDepth=5} --> + +This module includes both unit tests, which can run in isolation without +connecting to the Azure Storage service, and integration tests, which require a working +connection to interact with a container. Unit test suites follow the naming +convention `Test*.java`. Integration tests follow the naming convention +`ITest*.java`. + +## Policy for submitting patches which affect the `hadoop-azure` module. + +The Apache Jenkins infrastucture does not run any cloud integration tests, +due to the need to keep credentials secure. + +### The submitter of any patch is required to run all the integration tests and declare which Azure region they used. + +This is important: **patches which do not include this declaration will be ignored** + +This policy has proven to be the only mechanism to guarantee full regression +testing of code changes. Why the declaration of region? Two reasons + +1. It helps us identify regressions which only surface against specific endpoints. +1. It forces the submitters to be more honest about their testing. It's easy +to lie, "yes, I tested this". To say "yes, I tested this against Azure US-west" +is a more specific lie and harder to make. And, if you get caught out: you +lose all credibility with the project. + +You don't need to test from a VM within the Azure infrastructure, all you need +are credentials. + +It's neither hard nor expensive to run the tests; if you can't, +there's no guarantee your patch works. The reviewers have enough to do, and +don't have the time to do these tests, especially as every failure will simply +make for a slow iterative development. + +Please: run the tests. And if you don't, we are sorry for declining your +patch, but we have to. + + +### What if there's an intermittent failure of a test? + +Some of the tests do fail intermittently, especially in parallel runs. +If this happens, try to run the test on its own to see if the test succeeds. + +If it still fails, include this fact in your declaration. We know some tests +are intermittently unreliable. + +### What if the tests are timing out or failing over my network connection? + +The tests are designed to be configurable for different +timeouts. If you are seeing problems and this configuration isn't working, +that's a sign of the configuration mechanism isn't complete. If it's happening +in the production code, that could be a sign of a problem which may surface +over long-haul connections. Please help us identify and fix these problems +— especially as you are the one best placed to verify the fixes work. + +## Setting up the tests + +## Testing the `hadoop-azure` Module + +The `hadoop-azure` module includes a full suite of unit tests. Many of the tests +will run without additional configuration by running `mvn test`. This includes +tests against mocked storage, which is an in-memory emulation of Azure Storage. + +The integration tests are designed to test directly against an Azure storage +service, and require an account and credentials in order to run. + +This is done by creating the file to `src/test/resources/azure-auth-keys.xml` +and setting the name of the storage account and its access key. + +For example: + +```xml +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + <property> + <name>fs.azure.test.account.name</name> + <value>{ACCOUNTNAME}.blob.core.windows.net</value> + </property> + <property> + <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name> + <value>{ACCOUNT ACCESS KEY}</value> + </property> +</configuration> +``` + +To run contract tests, set the WASB file system URI in `src/test/resources/azure-auth-keys.xml` +and the account access key. For example: + +```xml +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + <property> + <name>fs.contract.test.fs.wasb</name> + <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value> + <description>The name of the azure file system for testing.</description> + </property> + <property> + <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name> + <value>{ACCOUNT ACCESS KEY}</value> + </property> +</configuration> +``` + +Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml` is like following: + +```xml +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + <property> + <name>fs.azure.test.account.name</name> + <value>{ACCOUNTNAME}.blob.core.windows.net</value> + </property> + <property> + <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name> + <value>{ACCOUNT ACCESS KEY}</value> + </property> + <property> + <name>fs.contract.test.fs.wasb</name> + <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value> + </property> +</configuration> +``` + +DO NOT ADD `azure-auth-keys.xml` TO REVISION CONTROL. The keys to your Azure +Storage account are a secret and must not be shared. + + +## Running the Tests + +After completing the configuration, execute the test run through Maven. + +```bash +mvn -T 1C clean verify +``` + +It's also possible to execute multiple test suites in parallel by passing the +`parallel-tests` property on the command line. The tests spend most of their +time blocked on network I/O, so running in parallel tends to +complete full test runs faster. + +```bash +mvn -T 1C -Dparallel-tests clean verify +``` + +Some tests must run with exclusive access to the storage container, so even with the +`parallel-tests` property, several test suites will run in serial in a separate +Maven execution step after the parallel tests. + +By default, `parallel-tests` runs 4 test suites concurrently. This can be tuned +by passing the `testsThreadCount` property. + +```bash +mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean verify +``` + +<!--- +To run just unit tests, which do not require Azure connectivity or credentials, +use any of the above invocations, but switch the goal to `test` instead of +`verify`. +--> + +```bash +mvn -T 1C clean test + +mvn -T 1C -Dparallel-tests clean test + +mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean test +``` + +To run only a specific named subset of tests, pass the `test` property for unit +tests or the `it.test` property for integration tests. + +```bash +mvn -T 1C clean test -Dtest=TestRollingWindowAverage + +mvn -T 1C clean verify -Dscale -Dit.test=ITestFileSystemOperationExceptionMessage -Dtest=none + +mvn -T 1C clean verify -Dtest=none -Dit.test=ITest* + +``` + +Note + +1. When running a specific subset of tests, the patterns passed in `test` +and `it.test` override the configuration of which tests need to run in isolation +in a separate serial phase (mentioned above). This can cause unpredictable +results, so the recommendation is to avoid passing `parallel-tests` in +combination with `test` or `it.test`. If you know that you are specifying only +tests that can run safely in parallel, then it will work. For wide patterns, +like `ITest*` shown above, it may cause unpredictable test failures. + +2. The command line shell may try to expand the "*" and sometimes the "#" symbols +in test patterns. In such situations, escape the character it with a "\\" prefix. +Example: + + mvn -T 1C clean verify -Dtest=none -Dit.test=ITest\* + + +## Viewing the results + +Integration test results and logs are stored in `target/failsafe-reports/`. +An HTML report can be generated during site generation, or with the `surefire-report` +plugin: + +```bash + +# for the unit tests +mvn -T 1C surefire-report:report-only + +# for the integration tests +mvn -T 1C surefire-report:failsafe-report-only + +# all reports for this module +mvn -T 1C site:site +``` + +## Scale Tests + +There are a set of tests designed to measure the scalability and performance +at scale of the filesystem client, *Scale Tests*. Tests include: creating +and traversing directory trees, uploading large files, renaming them, +deleting them, seeking through the files, performing random IO, and others. +This makes them a foundational part of the benchmarking. + +By their very nature they are slow. And, as their execution time is often +limited by bandwidth between the computer running the tests and the Azure endpoint, +parallel execution does not speed these tests up. + +### Enabling the Scale Tests + +The tests are enabled if the `scale` property is set in the maven build +this can be done regardless of whether or not the parallel test profile +is used + +```bash +mvn -T 1C verify -Dscale + +mvn -T 1C verify -Dparallel-tests -Dscale -DtestsThreadCount=8 +``` + +The most bandwidth intensive tests (those which upload data) always run +sequentially; those which are slow due to HTTPS setup costs or server-side +actions are included in the set of parallelized tests. + + +### Scale test tuning options + + +Some of the tests can be tuned from the maven build or from the +configuration file used to run the tests. + +```bash +mvn -T 1C verify -Dparallel-tests -Dscale -DtestsThreadCount=8 -Dfs.azure.scale.test.huge.filesize=128M +``` + +The algorithm is + +1. The value is queried from the configuration file, using a default value if +it is not set. +1. The value is queried from the JVM System Properties, where it is passed +down by maven. +1. If the system property is null, an empty string, or it has the value `unset`, +then the configuration value is used. The `unset` option is used to +[work round a quirk in maven property propagation](http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven). + +Only a few properties can be set this way; more will be added. + +| Property | Meaninging | +|-----------|-------------| +| `fs.azure.scale.test.huge.filesize`| Size for huge file uploads | +| `fs.azure.scale.test.huge.huge.partitionsize`| Size for partitions in huge file uploads | + +The file and partition sizes are numeric values with a k/m/g/t/p suffix depending +on the desired size. For example: 128M, 128m, 2G, 2G, 4T or even 1P. + +#### Scale test configuration options + +Some scale tests perform multiple operations (such as creating many directories). + +The exact number of operations to perform is configurable in the option +`scale.test.operation.count` + +```xml +<property> + <name>scale.test.operation.count</name> + <value>10</value> +</property> +``` + +Larger values generate more load, and are recommended when testing locally, +or in batch runs. + +Smaller values results in faster test runs, especially when the object +store is a long way away. + +Operations which work on directories have a separate option: this controls +the width and depth of tests creating recursive directories. Larger +values create exponentially more directories, with consequent performance +impact. + +```xml +<property> + <name>scale.test.directory.count</name> + <value>2</value> +</property> +``` + +DistCp tests targeting Azure support a configurable file size. The default is +10 MB, but the configuration value is expressed in KB so that it can be tuned +smaller to achieve faster test runs. + +```xml +<property> + <name>scale.test.distcp.file.size.kb</name> + <value>10240</value> +</property> +``` + +Azure-specific scale test properties are + +##### `fs.azure.scale.test.huge.filesize`: size in MB for "Huge file tests". + +The Huge File tests validate Azure storages's ability to handle large files âthe property +`fs.azure.scale.test.huge.filesize` declares the file size to use. + +```xml +<property> + <name>fs.azure.scale.test.huge.filesize</name> + <value>200M</value> +</property> +``` + +Tests at this scale are slow: they are best executed from hosts running in +the cloud infrastructure where the storage endpoint is based. + +## Using the emulator + +A selection of tests can run against the +[Azure Storage Emulator](http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx) +which is a high-fidelity emulation of live Azure Storage. The emulator is +sufficient for high-confidence testing. The emulator is a Windows executable +that runs on a local machine. + +To use the emulator, install Azure SDK 2.3 and start the storage emulator. Then, +edit `src/test/resources/azure-test.xml` and add the following property: + +```xml +<property> + <name>fs.azure.test.emulator</name> + <value>true</value> +</property> +``` + +There is a known issue when running tests with the emulator. You may see the +following failure message: + + com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format. + +To resolve this, restart the Azure Emulator. Ensure it is v3.2 or later. + + +## Debugging Test failures + +Logging at debug level is the standard way to provide more diagnostics output; +after setting this rerun the tests + +```properties +log4j.logger.org.apache.hadoop.fs.azure=DEBUG +``` + +## Adding new tests + +New tests are always welcome. Bear in mind that we need to keep costs +and test time down, which is done by + +* Not duplicating tests. +* Being efficient in your use of Hadoop API calls. +* Isolating large/slow tests into the "scale" test group. +* Designing all tests to execute in parallel (where possible). +* Adding new probes and predicates into existing tests, albeit carefully. + +*No duplication*: if an operation is tested elsewhere, don't repeat it. This +applies as much for metadata operations as it does for bulk IO. If a new +test case is added which completely obsoletes an existing test, it is OK +to cut the previous one âafter showing that coverage is not worsened. + +*Efficient*: prefer the `getFileStatus()` and examining the results, rather than +call to `exists()`, `isFile()`, etc. + +*Fail with useful information:* provide as much diagnostics as possible +on a failure. Using `org.apache.hadoop.fs.contract.ContractTestUtils` to make +assertions about the state of a filesystem helps here. + +*Isolating Scale tests*. Any test doing large amounts of IO MUST extend the +class `AbstractAzureScaleTest`, so only running if `scale` is defined on a build, +supporting test timeouts configurable by the user. Scale tests should also +support configurability as to the actual size of objects/number of operations, +so that behavior at different scale can be verified. + +*Designed for parallel execution*. A key need here is for each test suite to work +on isolated parts of the filesystem. Subclasses of `AbstractWasbTestBase` +SHOULD use the `path()`, `methodpath()` and `blobpath()` methods, +to build isolated paths. Tests MUST NOT assume that they have exclusive access +to a bucket. + +*Extending existing tests where appropriate*. This recommendation goes +against normal testing best practise of "test one thing per method". +Because it is so slow to create directory trees or upload large files, we do +not have that luxury. All the tests against real endpoints are integration +tests where sharing test setup and teardown saves time and money. + +A standard way to do this is to extend existing tests with some extra predicates, +rather than write new tests. When doing this, make sure that the new predicates +fail with meaningful diagnostics, so any new problems can be easily debugged +from test logs. + + +### Requirements of new Tests + + +This is what we expect from new tests; they're an extension of the normal +Hadoop requirements, based on the need to work with remote servers whose +use requires the presence of secret credentials, where tests may be slow, +and where finding out why something failed from nothing but the test output +is critical. + +#### Subclasses Existing Shared Base Blasses + +There are a set of base classes which should be extended for Azure tests and +integration tests. + +##### `org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout` + +This extends the junit `Assert` class with thread names and timeouts, +the default timeout being set in `AzureTestConstants.AZURE_TEST_TIMEOUT` to +ten minutes. The thread names are set to aid analyzing the stack trace of +a test: a `jstack` call can be used to + +##### `org.apache.hadoop.fs.azure.AbstractWasbTestBase` + +The base class for tests which use `AzureBlobStorageTestAccount` to create +mock or live Azure clients; in test teardown it tries to clean up store state. + +1. This class requires subclasses to implement `createTestAccount()` to create +a mock or real test account. + +1. The configuration used to create a test account *should* be that from +`createConfiguration()`; this can be extended in subclasses to tune the settings. + + +##### `org.apache.hadoop.fs.azure.integration.AbstractAzureScaleTest` + +This extends `AbstractWasbTestBase` for scale tests; those test which +only run when `-Dscale` is used to select the "scale" profile. +These tests have a timeout of 30 minutes, so as to support slow test runs. + +Having shared base classes help reduces future maintenance. Please +use them. + +#### Secure + +Don't ever log credentials. The credential tests go out of their way to +not provide meaningful logs or assertion messages precisely to avoid this. + +#### Efficient of Time and Money + +This means efficient in test setup/teardown, and, ideally, making use of +existing public datasets to save setup time and tester cost. + + +The reference example is `ITestAzureHugeFiles`:. This marks the test suite as +`@FixMethodOrder(MethodSorters.NAME_ASCENDING)` then orders the test cases such +that each test case expects the previous test to have completed (here: uploaded a file, +renamed a file, ...). This provides for independent tests in the reports, yet still +permits an ordered sequence of operations. Do note the use of `Assume.assume()` +to detect when the preconditions for a single test case are not met, hence, +the tests become skipped, rather than fail with a trace which is really a false alarm. + + +### Works Over Long-haul Links + +As well as making file size and operation counts scaleable, this includes +making test timeouts adequate. The Scale tests make this configurable; it's +hard coded to ten minutes in `AbstractAzureIntegrationTest()`; subclasses can +change this by overriding `getTestTimeoutMillis()`. + +Equally importantly: support proxies, as some testers need them. + + +### Provides Diagnostics and timing information + +1. Create logs, log things. +1. you can use `AbstractWasbTestBase.describe(format-string, args)` here; it +adds some newlines so as to be easier to spot. +1. Use `ContractTestUtils.NanoTimer` to measure the duration of operations, +and log the output. + +#### Fails Meaningfully + +The `ContractTestUtils` class contains a whole set of assertions for making +statements about the expected state of a filesystem, e.g. +`assertPathExists(FS, path)`, `assertPathDoesNotExists(FS, path)`, and others. +These do their best to provide meaningful diagnostics on failures (e.g. directory +listings, file status, ...), so help make failures easier to understand. + +At the very least, *do not use `assertTrue()` or `assertFalse()` without +including error messages*. + + +### Cleans Up Afterwards + +Keeps costs down. + +1. Do not only cleanup if a test case completes successfully; test suite +teardown must do it. +1. That teardown code must check for the filesystem and other fields being +null before the cleanup. Why? If test setup fails, the teardown methods still +get called. + +### Works Reliably + +We really appreciate this — you will too. + + +## Tips + +### How to keep your credentials really safe + +Although the `auth-keys.xml` file is marged as ignored in git and subversion, +it is still in your source tree, and there's always that risk that it may +creep out. + +You can avoid this by keeping your keys outside the source tree and +using an absolute XInclude reference to it. + +```xml +<configuration> + + <include xmlns="http://www.w3.org/2001/XInclude"; + href="file:///users/qe/.auth-keys.xml" /> + +</configuration> +``` + +### Cleaning up Containers + +The Azure tests create containers with the prefix `"wasbtests-"` and delete +them after the test runs. If a test run is interrupted, these containers +may not get deleted. There is a special test case which can be manually invoked +to list and delete these, `CleanupTestContainers` + +```bash +mvn test -Dtest=CleanupTestContainers +``` + +This will delete the containers; the output log of the test run will +provide the details and summary of the operation. http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java index d04a19c..0d3a06c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java @@ -18,15 +18,21 @@ package org.apache.hadoop.fs.azure; -import static org.junit.Assume.assumeNotNull; +import java.io.IOException; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.junit.After; import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.integration.AzureTestConstants; +import org.apache.hadoop.io.IOUtils; + +import static org.junit.Assume.assumeNotNull; +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.*; + /** * Abstract test class that provides basic setup and teardown of testing Azure * Storage account. Each subclass defines a different set of test cases to run @@ -34,41 +40,137 @@ import org.slf4j.LoggerFactory; * to run those tests. The returned account might integrate with Azure Storage * directly or it might be a mock implementation. */ -public abstract class AbstractWasbTestBase { +public abstract class AbstractWasbTestBase extends AbstractWasbTestWithTimeout + implements AzureTestConstants { protected static final Logger LOG = LoggerFactory.getLogger(AbstractWasbTestBase.class); - @VisibleForTesting protected NativeAzureFileSystem fs; - private AzureBlobStorageTestAccount testAccount; + protected AzureBlobStorageTestAccount testAccount; @Before public void setUp() throws Exception { - testAccount = createTestAccount(); - if (testAccount != null) { - fs = testAccount.getFileSystem(); - } - assumeNotNull(testAccount); + AzureBlobStorageTestAccount account = createTestAccount(); + assumeNotNull(account); + bindToTestAccount(account); } @After public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - fs = null; - } + describe("closing test account and filesystem"); + testAccount = cleanupTestAccount(testAccount); + IOUtils.closeStream(fs); + fs = null; } - public Configuration getConfiguration() { - return new Configuration(); + /** + * Create the configuration to use when creating a test account. + * Subclasses can override this to tune the test account configuration. + * @return a configuration. + */ + public Configuration createConfiguration() { + return AzureBlobStorageTestAccount.createTestConfiguration(); } + /** + * Create the test account. + * Subclasses must implement this. + * @return the test account. + * @throws Exception + */ protected abstract AzureBlobStorageTestAccount createTestAccount() throws Exception; + /** + * Get the test account. + * @return the current test account. + */ protected AzureBlobStorageTestAccount getTestAccount() { return testAccount; } + + /** + * Get the filesystem + * @return the current filesystem. + */ + protected NativeAzureFileSystem getFileSystem() { + return fs; + } + + /** + * Get the configuration used to create the filesystem + * @return the configuration of the test FS + */ + protected Configuration getConfiguration() { + return getFileSystem().getConf(); + } + + /** + * Bind to a new test account; closing any existing one. + * This updates the test account returned in {@link #getTestAccount()} + * and the filesystem in {@link #getFileSystem()}. + * @param account new test account + */ + protected void bindToTestAccount(AzureBlobStorageTestAccount account) { + // clean any existing test account + cleanupTestAccount(testAccount); + IOUtils.closeStream(fs); + testAccount = account; + if (testAccount != null) { + fs = testAccount.getFileSystem(); + } + } + + /** + * Return a path to a blob which will be unique for this fork. + * @param filepath filepath + * @return a path under the default blob directory + * @throws IOException + */ + protected Path blobPath(String filepath) throws IOException { + return blobPathForTests(getFileSystem(), filepath); + } + + /** + * Create a path under the test path provided by + * the FS contract. + * @param filepath path string in + * @return a path qualified by the test filesystem + * @throws IOException IO problems + */ + protected Path path(String filepath) throws IOException { + return pathForTests(getFileSystem(), filepath); + } + + /** + * Return a path bonded to this method name, unique to this fork during + * parallel execution. + * @return a method name unique to (fork, method). + * @throws IOException IO problems + */ + protected Path methodPath() throws IOException { + return path(methodName.getMethodName()); + } + + /** + * Return a blob path bonded to this method name, unique to this fork during + * parallel execution. + * @return a method name unique to (fork, method). + * @throws IOException IO problems + */ + protected Path methodBlobPath() throws IOException { + return blobPath(methodName.getMethodName()); + } + + /** + * Describe a test in the logs. + * @param text text to print + * @param args arguments to format in the printing + */ + protected void describe(String text, Object... args) { + LOG.info("\n\n{}: {}\n", + methodName.getMethodName(), + String.format(text, args)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java new file mode 100644 index 0000000..b7076a4 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; + +import org.apache.hadoop.fs.azure.integration.AzureTestConstants; + +/** + * Base class for any Wasb test with timeouts & named threads. + * This class does not attempt to bind to Azure. + */ +public class AbstractWasbTestWithTimeout extends Assert { + + /** + * The name of the current method. + */ + @Rule + public TestName methodName = new TestName(); + /** + * Set the timeout for every test. + * This is driven by the value returned by {@link #getTestTimeoutMillis()}. + */ + @Rule + public Timeout testTimeout = new Timeout(getTestTimeoutMillis()); + + /** + * Name the junit thread for the class. This will overridden + * before the individual test methods are run. + */ + @BeforeClass + public static void nameTestThread() { + Thread.currentThread().setName("JUnit"); + } + + /** + * Name the thread to the current test method. + */ + @Before + public void nameThread() { + Thread.currentThread().setName("JUnit-" + methodName.getMethodName()); + } + + /** + * Override point: the test timeout in milliseconds. + * @return a timeout in milliseconds + */ + protected int getTestTimeoutMillis() { + return AzureTestConstants.AZURE_TEST_TIMEOUT; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java index 7fa59ce..5b36c87 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java @@ -21,12 +21,15 @@ package org.apache.hadoop.fs.azure; import com.microsoft.azure.storage.*; import com.microsoft.azure.storage.blob.*; import com.microsoft.azure.storage.core.Base64; -import org.apache.commons.configuration2.SubsetConfiguration; +import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.configuration2.SubsetConfiguration; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.integration.AzureTestConstants; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem; import org.apache.hadoop.metrics2.AbstractMetric; @@ -35,6 +38,8 @@ import org.apache.hadoop.metrics2.MetricsSink; import org.apache.hadoop.metrics2.MetricsTag; import org.apache.hadoop.metrics2.impl.TestMetricsConfig; +import java.io.File; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.*; @@ -46,10 +51,10 @@ import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECU /** * Helper class to create WASB file systems backed by either a mock in-memory - * implementation or a real Azure Storage account. See RunningLiveWasbTests.txt - * for instructions on how to connect to a real Azure Storage account. + * implementation or a real Azure Storage account. */ -public final class AzureBlobStorageTestAccount { +public final class AzureBlobStorageTestAccount implements AutoCloseable, + AzureTestConstants { private static final Logger LOG = LoggerFactory.getLogger( AzureBlobStorageTestAccount.class); @@ -166,6 +171,7 @@ public final class AzureBlobStorageTestAccount { return new Path("/" + DEFAULT_PAGE_BLOB_DIRECTORY); } + @Deprecated public static Path pageBlobPath(String fileName) { return new Path(pageBlobPath(), fileName); } @@ -201,6 +207,9 @@ public final class AzureBlobStorageTestAccount { * @return */ private boolean wasGeneratedByMe(MetricsRecord currentRecord) { + Assert.assertNotNull("null filesystem", fs); + Assert.assertNotNull("null filesystemn instance ID", + fs.getInstrumentation().getFileSystemInstanceId()); String myFsId = fs.getInstrumentation().getFileSystemInstanceId().toString(); for (MetricsTag currentTag : currentRecord.tags()) { if (currentTag.name().equalsIgnoreCase("wasbFileSystemId")) { @@ -247,13 +256,16 @@ public final class AzureBlobStorageTestAccount { getBlobReference(blobKey).releaseLease(accessCondition); } - private static void saveMetricsConfigFile() { + private static void saveMetricsConfigFile() throws IOException { if (!metricsConfigSaved) { + String testFilename = TestMetricsConfig.getTestFilename( + "hadoop-metrics2-azure-file-system"); + File dest = new File(testFilename).getCanonicalFile(); + dest.getParentFile().mkdirs(); new org.apache.hadoop.metrics2.impl.ConfigBuilder() .add("azure-file-system.sink.azuretestcollector.class", StandardCollector.class.getName()) - .save(TestMetricsConfig.getTestFilename( - "hadoop-metrics2-azure-file-system.properties")); + .save(testFilename); metricsConfigSaved = true; } } @@ -314,9 +326,8 @@ public final class AzureBlobStorageTestAccount { Configuration conf = createTestConfiguration(); if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) { // Not configured to test against the storage emulator. - LOG.warn("Skipping emulator Azure test because configuration doesn't " - + "indicate that it's running. Please see RunningLiveWasbTests.txt " - + "for guidance."); + LOG.warn("Skipping emulator Azure test because configuration " + + "doesn't indicate that it's running."); return null; } CloudStorageAccount account = @@ -482,8 +493,7 @@ public final class AzureBlobStorageTestAccount { credentials = StorageCredentialsAnonymous.ANONYMOUS; } else { LOG.warn("Skipping live Azure test because of missing key for" - + " account '" + accountName + "'. " - + "Please see RunningLiveWasbTests.txt for guidance."); + + " account '" + accountName + "'."); return null; } } else { @@ -517,8 +527,7 @@ public final class AzureBlobStorageTestAccount { throws URISyntaxException, KeyProviderException { String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); if (testAccountName == null) { - LOG.warn("Skipping live Azure test because of missing test account. " - + "Please see RunningLiveWasbTests.txt for guidance."); + LOG.warn("Skipping live Azure test because of missing test account"); return null; } return createStorageAccount(testAccountName, conf, false); @@ -863,6 +872,11 @@ public final class AzureBlobStorageTestAccount { } } + @Override + public void close() throws Exception { + cleanup(); + } + public NativeAzureFileSystem getFileSystem() { return fs; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java new file mode 100644 index 0000000..7e733dc --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.azure.integration.AzureTestUtils; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.permission.PermissionStatus; + +/** + * Handle OOB IO into a shared container. + */ +public class ITestAzureConcurrentOutOfBandIo extends AbstractWasbTestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestAzureConcurrentOutOfBandIo.class); + + // Class constants. + static final int DOWNLOAD_BLOCK_SIZE = 8 * 1024 * 1024; + static final int UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024; + static final int BLOB_SIZE = 32 * 1024 * 1024; + + // Number of blocks to be written before flush. + static final int NUMBER_OF_BLOCKS = 2; + + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.createOutOfBandStore( + UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE); + } + + class DataBlockWriter implements Runnable { + + Thread runner; + AzureBlobStorageTestAccount writerStorageAccount; + String key; + boolean done = false; + + /** + * Constructor captures the test account. + * + * @param testAccount + */ + public DataBlockWriter(AzureBlobStorageTestAccount testAccount, String key) { + writerStorageAccount = testAccount; + this.key = key; + } + + /** + * Start writing blocks to Azure storage. + */ + public void startWriting() { + runner = new Thread(this); // Create the block writer thread. + runner.start(); // Start the block writer thread. + } + + /** + * Stop writing blocks to Azure storage. + */ + public void stopWriting() { + done = true; + } + + /** + * Implementation of the runnable interface. The run method is a tight loop + * which repeatedly updates the blob with a 4 MB block. + */ + public void run() { + byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE]; + + OutputStream outputStream = null; + + try { + for (int i = 0; !done; i++) { + // Write two 4 MB blocks to the blob. + // + outputStream = writerStorageAccount.getStore().storefile( + key, + new PermissionStatus("", "", FsPermission.getDefault()), + key); + + Arrays.fill(dataBlockWrite, (byte) (i % 256)); + for (int j = 0; j < NUMBER_OF_BLOCKS; j++) { + outputStream.write(dataBlockWrite); + } + + outputStream.flush(); + outputStream.close(); + } + } catch (AzureException e) { + LOG.error("DatablockWriter thread encountered a storage exception." + + e.getMessage(), e); + } catch (IOException e) { + LOG.error("DatablockWriter thread encountered an I/O exception." + + e.getMessage(), e); + } + } + } + + @Test + public void testReadOOBWrites() throws Exception { + + byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE]; + byte[] dataBlockRead = new byte[UPLOAD_BLOCK_SIZE]; + + // Write to blob to make sure it exists. + // + // Write five 4 MB blocks to the blob. To ensure there is data in the blob before + // reading. This eliminates the race between the reader and writer threads. + String key = "WASB_String" + AzureTestUtils.getForkID() + ".txt"; + OutputStream outputStream = testAccount.getStore().storefile( + key, + new PermissionStatus("", "", FsPermission.getDefault()), + key); + Arrays.fill(dataBlockWrite, (byte) 255); + for (int i = 0; i < NUMBER_OF_BLOCKS; i++) { + outputStream.write(dataBlockWrite); + } + + outputStream.flush(); + outputStream.close(); + + // Start writing blocks to Azure store using the DataBlockWriter thread. + DataBlockWriter writeBlockTask = new DataBlockWriter(testAccount, key); + writeBlockTask.startWriting(); + int count = 0; + + for (int i = 0; i < 5; i++) { + try(InputStream inputStream = testAccount.getStore().retrieve(key)) { + count = 0; + int c = 0; + + while (c >= 0) { + c = inputStream.read(dataBlockRead, 0, UPLOAD_BLOCK_SIZE); + if (c < 0) { + break; + } + + // Counting the number of bytes. + count += c; + } + } catch (IOException e) { + System.out.println(e.getCause().toString()); + e.printStackTrace(); + fail(); + } + } + + // Stop writing blocks. + writeBlockTask.stopWriting(); + + // Validate that a block was read. + assertEquals(NUMBER_OF_BLOCKS * UPLOAD_BLOCK_SIZE, count); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java new file mode 100644 index 0000000..2b0ea56 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +/** + * Extends ITestAzureConcurrentOutOfBandIo in order to run testReadOOBWrites with secure mode + * (fs.azure.secure.mode) both enabled and disabled. + */ +public class ITestAzureConcurrentOutOfBandIoWithSecureMode + extends ITestAzureConcurrentOutOfBandIo { + + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.createOutOfBandStore( + UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE, true); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java new file mode 100644 index 0000000..49e6730 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.Callable; + +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageEvent; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext; +import org.apache.hadoop.test.GenericTestUtils; + +import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assume.assumeNotNull; + +/** + * Error handling. + */ +public class ITestAzureFileSystemErrorConditions extends + AbstractWasbTestWithTimeout { + private static final int ALL_THREE_FILE_SIZE = 1024; + + @Test + public void testNoInitialize() throws Exception { + intercept(AssertionError.class, + new Callable<FileMetadata>() { + @Override + public FileMetadata call() throws Exception { + return new AzureNativeFileSystemStore() + .retrieveMetadata("foo"); + } + }); + } + + /** + * Try accessing an unauthorized or non-existent (treated the same) container + * from WASB. + */ + @Test + public void testAccessUnauthorizedPublicContainer() throws Exception { + final String container = "nonExistentContainer"; + final String account = "hopefullyNonExistentAccount"; + Path noAccessPath = new Path( + "wasb://" + container + "@" + account + "/someFile"); + NativeAzureFileSystem.suppressRetryPolicy(); + try { + FileSystem.get(noAccessPath.toUri(), new Configuration()) + .open(noAccessPath); + assertTrue("Should've thrown.", false); + } catch (AzureException ex) { + GenericTestUtils.assertExceptionContains( + String.format(NO_ACCESS_TO_CONTAINER_MSG, account, container), ex); + } finally { + NativeAzureFileSystem.resumeRetryPolicy(); + } + } + + @Test + public void testAccessContainerWithWrongVersion() throws Exception { + AzureNativeFileSystemStore store = new AzureNativeFileSystemStore(); + MockStorageInterface mockStorage = new MockStorageInterface(); + store.setAzureStorageInteractionLayer(mockStorage); + try (FileSystem fs = new NativeAzureFileSystem(store)) { + Configuration conf = new Configuration(); + AzureBlobStorageTestAccount.setMockAccountKey(conf); + HashMap<String, String> metadata = new HashMap<String, String>(); + metadata.put(AzureNativeFileSystemStore.VERSION_METADATA_KEY, + "2090-04-05"); // It's from the future! + mockStorage.addPreExistingContainer( + AzureBlobStorageTestAccount.getMockContainerUri(), metadata); + + AzureException ex = intercept(AzureException.class, + new Callable<FileStatus[]>() { + @Override + public FileStatus[] call() throws Exception { + fs.initialize(new URI(AzureBlobStorageTestAccount.MOCK_WASB_URI), + conf); + return fs.listStatus(new Path("/")); + } + }); + GenericTestUtils.assertExceptionContains( + "unsupported version: 2090-04-05.", ex); + } + } + + private interface ConnectionRecognizer { + boolean isTargetConnection(HttpURLConnection connection); + } + + private class TransientErrorInjector extends StorageEvent<SendingRequestEvent> { + private final ConnectionRecognizer connectionRecognizer; + private boolean injectedErrorOnce = false; + + public TransientErrorInjector(ConnectionRecognizer connectionRecognizer) { + this.connectionRecognizer = connectionRecognizer; + } + + @Override + public void eventOccurred(SendingRequestEvent eventArg) { + HttpURLConnection connection + = (HttpURLConnection) eventArg.getConnectionObject(); + if (!connectionRecognizer.isTargetConnection(connection)) { + return; + } + if (!injectedErrorOnce) { + connection.setReadTimeout(1); + connection.disconnect(); + injectedErrorOnce = true; + } + } + } + + private void injectTransientError(NativeAzureFileSystem fs, + final ConnectionRecognizer connectionRecognizer) { + fs.getStore().addTestHookToOperationContext(new TestHookOperationContext() { + @Override + public OperationContext modifyOperationContext(OperationContext original) { + original.getSendingRequestEventHandler().addListener( + new TransientErrorInjector(connectionRecognizer)); + return original; + } + }); + } + + @Test + public void testTransientErrorOnDelete() throws Exception { + // Need to do this test against a live storage account + AzureBlobStorageTestAccount testAccount = + AzureBlobStorageTestAccount.create(); + assumeNotNull(testAccount); + try { + NativeAzureFileSystem fs = testAccount.getFileSystem(); + injectTransientError(fs, new ConnectionRecognizer() { + @Override + public boolean isTargetConnection(HttpURLConnection connection) { + return connection.getRequestMethod().equals("DELETE"); + } + }); + Path testFile = new Path("/a/b"); + assertTrue(fs.createNewFile(testFile)); + assertTrue(fs.rename(testFile, new Path("/x"))); + } finally { + testAccount.cleanup(); + } + } + + private void writeAllThreeFile(NativeAzureFileSystem fs, Path testFile) + throws IOException { + byte[] buffer = new byte[ALL_THREE_FILE_SIZE]; + Arrays.fill(buffer, (byte) 3); + try(OutputStream stream = fs.create(testFile)) { + stream.write(buffer); + } + } + + private void readAllThreeFile(NativeAzureFileSystem fs, Path testFile) + throws IOException { + byte[] buffer = new byte[ALL_THREE_FILE_SIZE]; + InputStream inStream = fs.open(testFile); + assertEquals(buffer.length, + inStream.read(buffer, 0, buffer.length)); + inStream.close(); + for (int i = 0; i < buffer.length; i++) { + assertEquals(3, buffer[i]); + } + } + + @Test + public void testTransientErrorOnCommitBlockList() throws Exception { + // Need to do this test against a live storage account + AzureBlobStorageTestAccount testAccount = + AzureBlobStorageTestAccount.create(); + assumeNotNull(testAccount); + try { + NativeAzureFileSystem fs = testAccount.getFileSystem(); + injectTransientError(fs, new ConnectionRecognizer() { + @Override + public boolean isTargetConnection(HttpURLConnection connection) { + return connection.getRequestMethod().equals("PUT") + && connection.getURL().getQuery() != null + && connection.getURL().getQuery().contains("blocklist"); + } + }); + Path testFile = new Path("/a/b"); + writeAllThreeFile(fs, testFile); + readAllThreeFile(fs, testFile); + } finally { + testAccount.cleanup(); + } + } + + @Test + public void testTransientErrorOnRead() throws Exception { + // Need to do this test against a live storage account + AzureBlobStorageTestAccount testAccount = + AzureBlobStorageTestAccount.create(); + assumeNotNull(testAccount); + try { + NativeAzureFileSystem fs = testAccount.getFileSystem(); + Path testFile = new Path("/a/b"); + writeAllThreeFile(fs, testFile); + injectTransientError(fs, new ConnectionRecognizer() { + @Override + public boolean isTargetConnection(HttpURLConnection connection) { + return connection.getRequestMethod().equals("GET"); + } + }); + readAllThreeFile(fs, testFile); + } finally { + testAccount.cleanup(); + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
