http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java deleted file mode 100644 index f6ab94d..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeNotNull; - -import java.io.FileNotFoundException; -import java.util.EnumSet; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; -import org.junit.After; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Test; - -import com.microsoft.azure.storage.blob.BlobOutputStream; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlockBlob; - -/** - * Tests that WASB creates containers only if needed. - */ -public class TestContainerChecks { - private AzureBlobStorageTestAccount testAccount; - private boolean runningInSASMode = false; - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - } - } - - @Before - public void setMode() { - runningInSASMode = AzureBlobStorageTestAccount.createTestConfiguration(). - getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, false); - } - - @Test - public void testContainerExistAfterDoesNotExist() throws Exception { - testAccount = AzureBlobStorageTestAccount.create("", - EnumSet.noneOf(CreateOptions.class)); - assumeNotNull(testAccount); - CloudBlobContainer container = testAccount.getRealContainer(); - FileSystem fs = testAccount.getFileSystem(); - - // Starting off with the container not there - assertFalse(container.exists()); - - // A list shouldn't create the container and will set file system store - // state to DoesNotExist - try { - fs.listStatus(new Path("/")); - assertTrue("Should've thrown.", false); - } catch (FileNotFoundException ex) { - assertTrue("Unexpected exception: " + ex, - ex.getMessage().contains("does not exist.")); - } - assertFalse(container.exists()); - - // Create a container outside of the WASB FileSystem - container.create(); - // Add a file to the container outside of the WASB FileSystem - CloudBlockBlob blob = testAccount.getBlobReference("foo"); - BlobOutputStream outputStream = blob.openOutputStream(); - outputStream.write(new byte[10]); - outputStream.close(); - - // Make sure the file is visible - assertTrue(fs.exists(new Path("/foo"))); - assertTrue(container.exists()); - } - - @Test - public void testContainerCreateAfterDoesNotExist() throws Exception { - testAccount = AzureBlobStorageTestAccount.create("", - EnumSet.noneOf(CreateOptions.class)); - assumeNotNull(testAccount); - CloudBlobContainer container = testAccount.getRealContainer(); - FileSystem fs = testAccount.getFileSystem(); - - // Starting off with the container not there - assertFalse(container.exists()); - - // A list shouldn't create the container and will set file system store - // state to DoesNotExist - try { - assertNull(fs.listStatus(new Path("/"))); - assertTrue("Should've thrown.", false); - } catch (FileNotFoundException ex) { - assertTrue("Unexpected exception: " + ex, - ex.getMessage().contains("does not exist.")); - } - assertFalse(container.exists()); - - // Create a container outside of the WASB FileSystem - container.create(); - - // Write should succeed - assertTrue(fs.createNewFile(new Path("/foo"))); - assertTrue(container.exists()); - } - - @Test - public void testContainerCreateOnWrite() throws Exception { - testAccount = AzureBlobStorageTestAccount.create("", - EnumSet.noneOf(CreateOptions.class)); - assumeNotNull(testAccount); - CloudBlobContainer container = testAccount.getRealContainer(); - FileSystem fs = testAccount.getFileSystem(); - - // Starting off with the container not there - assertFalse(container.exists()); - - // A list shouldn't create the container. - try { - fs.listStatus(new Path("/")); - assertTrue("Should've thrown.", false); - } catch (FileNotFoundException ex) { - assertTrue("Unexpected exception: " + ex, - ex.getMessage().contains("does not exist.")); - } - assertFalse(container.exists()); - - // Neither should a read. - try { - fs.open(new Path("/foo")); - assertFalse("Should've thrown.", true); - } catch (FileNotFoundException ex) { - } - assertFalse(container.exists()); - - // Neither should a rename - assertFalse(fs.rename(new Path("/foo"), new Path("/bar"))); - assertFalse(container.exists()); - - // But a write should. - assertTrue(fs.createNewFile(new Path("/foo"))); - assertTrue(container.exists()); - } - - @Test - public void testContainerChecksWithSas() throws Exception { - - Assume.assumeFalse(runningInSASMode); - testAccount = AzureBlobStorageTestAccount.create("", - EnumSet.of(CreateOptions.UseSas)); - assumeNotNull(testAccount); - CloudBlobContainer container = testAccount.getRealContainer(); - FileSystem fs = testAccount.getFileSystem(); - - // The container shouldn't be there - assertFalse(container.exists()); - - // A write should just fail - try { - fs.createNewFile(new Path("/foo")); - assertFalse("Should've thrown.", true); - } catch (AzureException ex) { - } - assertFalse(container.exists()); - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java deleted file mode 100644 index 9ac25dd..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.FileNotFoundException; - -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; - - -public class TestFileSystemOperationExceptionHandling - extends AbstractWasbTestBase { - - private FSDataInputStream inputStream = null; - - private static Path testPath = new Path("testfile.dat"); - - private static Path testFolderPath = new Path("testfolder"); - - /* - * Helper method that creates a InputStream to validate exceptions - * for various scenarios - */ - private void setupInputStreamToTest(AzureBlobStorageTestAccount testAccount) - throws Exception { - - FileSystem fs = testAccount.getFileSystem(); - - // Step 1: Create a file and write dummy data. - Path testFilePath1 = new Path("test1.dat"); - Path testFilePath2 = new Path("test2.dat"); - FSDataOutputStream outputStream = fs.create(testFilePath1); - String testString = "This is a test string"; - outputStream.write(testString.getBytes()); - outputStream.close(); - - // Step 2: Open a read stream on the file. - inputStream = fs.open(testFilePath1); - - // Step 3: Rename the file - fs.rename(testFilePath1, testFilePath2); - } - - /* - * Tests a basic single threaded read scenario for Page blobs. - */ - @Test(expected=FileNotFoundException.class) - public void testSingleThreadedPageBlobReadScenario() throws Throwable { - AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(); - setupInputStreamToTest(testAccount); - byte[] readBuffer = new byte[512]; - inputStream.read(readBuffer); - } - - /* - * Tests a basic single threaded seek scenario for Page blobs. - */ - @Test(expected=FileNotFoundException.class) - public void testSingleThreadedPageBlobSeekScenario() throws Throwable { - AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(); - setupInputStreamToTest(testAccount); - inputStream.seek(5); - } - - /* - * Test a basic single thread seek scenario for Block blobs. - */ - @Test(expected=FileNotFoundException.class) - public void testSingleThreadBlockBlobSeekScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - setupInputStreamToTest(testAccount); - inputStream.seek(5); - inputStream.read(); - } - - /* - * Tests a basic single threaded read scenario for Block blobs. - */ - @Test(expected=FileNotFoundException.class) - public void testSingledThreadBlockBlobReadScenario() throws Throwable{ - AzureBlobStorageTestAccount testAccount = createTestAccount(); - setupInputStreamToTest(testAccount); - byte[] readBuffer = new byte[512]; - inputStream.read(readBuffer); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic single threaded setPermission scenario - */ - public void testSingleThreadedBlockBlobSetPermissionScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), testPath); - fs.delete(testPath, true); - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic single threaded setPermission scenario - */ - public void testSingleThreadedPageBlobSetPermissionScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - fs.delete(testPath, true); - fs.setOwner(testPath, "testowner", "testgroup"); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic single threaded setPermission scenario - */ - public void testSingleThreadedBlockBlobSetOwnerScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), testPath); - fs.delete(testPath, true); - fs.setOwner(testPath, "testowner", "testgroup"); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic single threaded setPermission scenario - */ - public void testSingleThreadedPageBlobSetOwnerScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - fs.delete(testPath, true); - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - - @Test(expected=FileNotFoundException.class) - /* - * Test basic single threaded listStatus scenario - */ - public void testSingleThreadedBlockBlobListStatusScenario() throws Throwable { - ExceptionHandlingTestHelper.createTestFolder(createTestAccount(), testFolderPath); - fs.delete(testFolderPath, true); - fs.listStatus(testFolderPath); - } - - @Test(expected=FileNotFoundException.class) - /* - * Test basica single threaded listStatus scenario - */ - public void testSingleThreadedPageBlobListStatusScenario() throws Throwable { - ExceptionHandlingTestHelper.createTestFolder(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testFolderPath); - fs.delete(testFolderPath, true); - fs.listStatus(testFolderPath); - } - - @Test - /* - * Test basic single threaded listStatus scenario - */ - public void testSingleThreadedBlockBlobRenameScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), - testPath); - Path dstPath = new Path("dstFile.dat"); - fs.delete(testPath, true); - boolean renameResult = fs.rename(testPath, dstPath); - Assert.assertFalse(renameResult); - } - - @Test - /* - * Test basic single threaded listStatus scenario - */ - public void testSingleThreadedPageBlobRenameScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - Path dstPath = new Path("dstFile.dat"); - fs.delete(testPath, true); - boolean renameResult = fs.rename(testPath, dstPath); - Assert.assertFalse(renameResult); - } - - @Test - /* - * Test basic single threaded listStatus scenario - */ - public void testSingleThreadedBlockBlobDeleteScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), - testPath); - fs.delete(testPath, true); - boolean deleteResult = fs.delete(testPath, true); - Assert.assertFalse(deleteResult); - } - - @Test - /* - * Test basic single threaded listStatus scenario - */ - public void testSingleThreadedPageBlobDeleteScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - fs.delete(testPath, true); - boolean deleteResult = fs.delete(testPath, true); - Assert.assertFalse(deleteResult); - } - - @Test(expected=FileNotFoundException.class) - /* - * Test basic single threaded listStatus scenario - */ - public void testSingleThreadedBlockBlobOpenScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), - testPath); - fs.delete(testPath, true); - inputStream = fs.open(testPath); - } - - @Test(expected=FileNotFoundException.class) - /* - * Test basic single threaded listStatus scenario - */ - public void testSingleThreadedPageBlobOpenScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - fs.delete(testPath, true); - inputStream = fs.open(testPath); - } - - @After - public void tearDown() throws Exception { - if (inputStream != null) { - inputStream.close(); - } - - if (fs != null && fs.exists(testPath)) { - fs.delete(testPath, true); - } - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java deleted file mode 100644 index e619817..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; -import java.net.URI; -import java.util.UUID; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.test.GenericTestUtils; - -import org.junit.Assert; -import org.junit.Test; - -import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG; - - -public class TestFileSystemOperationExceptionMessage extends - NativeAzureFileSystemBaseTest { - - @Test - public void testAnonymouseCredentialExceptionMessage() throws Throwable{ - - Configuration conf = AzureBlobStorageTestAccount.createTestConfiguration(); - String testStorageAccount = conf.get("fs.azure.test.account.name"); - conf = new Configuration(); - conf.set("fs.AbstractFileSystem.wasb.impl", "org.apache.hadoop.fs.azure.Wasb"); - conf.set("fs.azure.skip.metrics", "true"); - - String testContainer = UUID.randomUUID().toString(); - String wasbUri = String.format("wasb://%s@%s", - testContainer, testStorageAccount); - - fs = new NativeAzureFileSystem(); - try { - fs.initialize(new URI(wasbUri), conf); - } catch (Exception ex) { - - Throwable innerException = ex.getCause(); - while (innerException != null - && !(innerException instanceof AzureException)) { - innerException = innerException.getCause(); - } - - if (innerException != null) { - String exceptionMessage = innerException.getMessage(); - if (exceptionMessage == null - || exceptionMessage.length() == 0) { - Assert.fail();} - else { - GenericTestUtils.assertExceptionContains(String.format( - NO_ACCESS_TO_CONTAINER_MSG, testStorageAccount, testContainer), - ex); - } - } else { - Assert.fail(); - } - } - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java deleted file mode 100644 index 1cd18ee..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java +++ /dev/null @@ -1,330 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.FileNotFoundException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.junit.After; -import org.junit.Test; - -public class TestFileSystemOperationsExceptionHandlingMultiThreaded - extends AbstractWasbTestBase { - - FSDataInputStream inputStream = null; - - private static Path testPath = new Path("testfile.dat"); - private static Path testFolderPath = new Path("testfolder"); - - - /* - * Helper method to creates an input stream to test various scenarios. - */ - private void getInputStreamToTest(FileSystem fs, Path testPath) throws Throwable { - - FSDataOutputStream outputStream = fs.create(testPath); - String testString = "This is a test string"; - outputStream.write(testString.getBytes()); - outputStream.close(); - - inputStream = fs.open(testPath); - } - - /* - * Test to validate correct exception is thrown for Multithreaded read - * scenario for block blobs - */ - @Test(expected=FileNotFoundException.class) - public void testMultiThreadedBlockBlobReadScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - - byte[] readBuffer = new byte[512]; - inputStream.read(readBuffer); - } - - /* - * Test to validate correct exception is thrown for Multithreaded seek - * scenario for block blobs - */ - - @Test(expected=FileNotFoundException.class) - public void testMultiThreadBlockBlobSeekScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - - inputStream.seek(5); - inputStream.read(); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedPageBlobSetPermissionScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedBlockBlobSetPermissionScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ)); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedPageBlobOpenScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - inputStream = fs.open(testPath); - inputStream.close(); - } - - inputStream = fs.open(testPath); - inputStream.close(); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setPermission scenario - */ - public void testMultiThreadedBlockBlobOpenScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - - while (t.isAlive()) { - inputStream = fs.open(testPath); - inputStream.close(); - } - inputStream = fs.open(testPath); - inputStream.close(); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setOwner scenario - */ - public void testMultiThreadedBlockBlobSetOwnerScenario() throws Throwable { - - ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setOwner(testPath, "testowner", "testgroup"); - } - fs.setOwner(testPath, "testowner", "testgroup"); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded setOwner scenario - */ - public void testMultiThreadedPageBlobSetOwnerScenario() throws Throwable { - ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testPath); - Thread t = new Thread(new DeleteThread(fs, testPath)); - t.start(); - while (t.isAlive()) { - fs.setOwner(testPath, "testowner", "testgroup"); - } - fs.setOwner(testPath, "testowner", "testgroup"); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded listStatus scenario - */ - public void testMultiThreadedBlockBlobListStatusScenario() throws Throwable { - - ExceptionHandlingTestHelper.createTestFolder(createTestAccount(), testFolderPath); - Thread t = new Thread(new DeleteThread(fs, testFolderPath)); - t.start(); - while (t.isAlive()) { - fs.listStatus(testFolderPath); - } - fs.listStatus(testFolderPath); - } - - @Test(expected=FileNotFoundException.class) - /* - * Tests basic multi threaded listStatus scenario - */ - public void testMultiThreadedPageBlobListStatusScenario() throws Throwable { - - ExceptionHandlingTestHelper.createTestFolder(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(), - testFolderPath); - Thread t = new Thread(new DeleteThread(fs, testFolderPath)); - t.start(); - while (t.isAlive()) { - fs.listStatus(testFolderPath); - } - fs.listStatus(testFolderPath); - } - - /* - * Test to validate correct exception is thrown for Multithreaded read - * scenario for page blobs - */ - - @Test(expected=FileNotFoundException.class) - public void testMultiThreadedPageBlobReadScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - byte[] readBuffer = new byte[512]; - inputStream.read(readBuffer); - } - - /* - * Test to validate correct exception is thrown for Multithreaded seek - * scenario for page blobs - */ - - @Test(expected=FileNotFoundException.class) - public void testMultiThreadedPageBlobSeekScenario() throws Throwable { - - AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(); - fs = testAccount.getFileSystem(); - Path testFilePath1 = new Path("test1.dat"); - - getInputStreamToTest(fs, testFilePath1); - Thread renameThread = new Thread(new RenameThread(fs, testFilePath1)); - renameThread.start(); - - renameThread.join(); - inputStream.seek(5); - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } - - @After - public void tearDown() throws Exception { - - if (inputStream != null) { - inputStream.close(); - } - - if (fs != null && fs.exists(testPath)) { - fs.delete(testPath, true); - } - } -} - -/* - * Helper thread that just renames the test file. - */ -class RenameThread implements Runnable { - - private FileSystem fs; - private Path testPath; - private Path renamePath = new Path("test2.dat"); - - public RenameThread(FileSystem fs, Path testPath) { - this.fs = fs; - this.testPath = testPath; - } - - @Override - public void run(){ - try { - fs.rename(testPath, renamePath); - }catch (Exception e) { - // Swallowing the exception as the - // correctness of the test is controlled - // by the other thread - } - } -} - -class DeleteThread implements Runnable { - private FileSystem fs; - private Path testPath; - - public DeleteThread(FileSystem fs, Path testPath) { - this.fs = fs; - this.testPath = testPath; - } - - @Override - public void run() { - try { - fs.delete(testPath, true); - } catch (Exception e) { - // Swallowing the exception as the - // correctness of the test is controlled - // by the other thread - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java deleted file mode 100644 index fd3690c..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java +++ /dev/null @@ -1,821 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.net.URI; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending; -import org.apache.hadoop.test.GenericTestUtils.LogCapturer; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -/** - * Tests the Native Azure file system (WASB) using parallel threads for rename and delete operations. - */ -public class TestFileSystemOperationsWithThreads extends AbstractWasbTestBase { - - private final int renameThreads = 10; - private final int deleteThreads = 20; - private int iterations = 1; - private LogCapturer logs = null; - - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Before - public void setUp() throws Exception { - super.setUp(); - Configuration conf = fs.getConf(); - - // By default enable parallel threads for rename and delete operations. - // Also enable flat listing of blobs for these operations. - conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, renameThreads); - conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, deleteThreads); - conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, true); - - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - // Capture logs - logs = LogCapturer.captureLogs(new Log4JLogger(org.apache.log4j.Logger - .getRootLogger())); - } - - /* - * Helper method to create sub directory and different types of files - * for multiple iterations. - */ - private void createFolder(FileSystem fs, String root) throws Exception { - fs.mkdirs(new Path(root)); - for (int i = 0; i < this.iterations; i++) { - fs.mkdirs(new Path(root + "/" + i)); - fs.createNewFile(new Path(root + "/" + i + "/fileToRename")); - fs.createNewFile(new Path(root + "/" + i + "/file/to/rename")); - fs.createNewFile(new Path(root + "/" + i + "/file+to%rename")); - fs.createNewFile(new Path(root + "/fileToRename" + i)); - } - } - - /* - * Helper method to do rename operation and validate all files in source folder - * doesn't exists and similar files exists in new folder. - */ - private void validateRenameFolder(FileSystem fs, String source, String dest) throws Exception { - // Create source folder with files. - createFolder(fs, source); - Path sourceFolder = new Path(source); - Path destFolder = new Path(dest); - - // rename operation - assertTrue(fs.rename(sourceFolder, destFolder)); - assertTrue(fs.exists(destFolder)); - - for (int i = 0; i < this.iterations; i++) { - // Check destination folder and files exists. - assertTrue(fs.exists(new Path(dest + "/" + i))); - assertTrue(fs.exists(new Path(dest + "/" + i + "/fileToRename"))); - assertTrue(fs.exists(new Path(dest + "/" + i + "/file/to/rename"))); - assertTrue(fs.exists(new Path(dest + "/" + i + "/file+to%rename"))); - assertTrue(fs.exists(new Path(dest + "/fileToRename" + i))); - - // Check source folder and files doesn't exists. - assertFalse(fs.exists(new Path(source + "/" + i))); - assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename"))); - assertFalse(fs.exists(new Path(source + "/fileToRename" + i))); - } - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameSmallFolderWithThreads() throws Exception { - - validateRenameFolder(fs, "root", "rootnew"); - - // With single iteration, we would have created 7 blobs. - int expectedThreadsCreated = Math.min(7, renameThreads); - - // Validate from logs that threads are created. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + expectedThreadsCreated); - - // Validate thread executions - for (int i = 0; i < expectedThreadsCreated; i++) { - assertInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - - // Also ensure that we haven't spawned extra threads. - if (expectedThreadsCreated < renameThreads) { - for (int i = expectedThreadsCreated; i < renameThreads; i++) { - assertNotInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - } - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameLargeFolderWithThreads() throws Exception { - - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateRenameFolder(fs, "root", "rootnew"); - - // Validate from logs that threads are created. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + renameThreads); - - // Validate thread executions - for (int i = 0; i < renameThreads; i++) { - assertInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for rename operation with threads disabled and flat listing enabled. - */ - @Test - public void testRenameLargeFolderDisableThreads() throws Exception { - Configuration conf = fs.getConf(); - - // Number of threads set to 0 or 1 disables threads. - conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 0); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateRenameFolder(fs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Rename operation as thread count 0"); - - // Validate no thread executions - for (int i = 0; i < renameThreads; i++) { - String term = "AzureBlobRenameThread-" - + Thread.currentThread().getName() - + "-" + i; - assertNotInLog(content, term); - } - } - - /** - * Assert that a log contains the given term. - * @param content log output - * @param term search term - */ - protected void assertInLog(String content, String term) { - assertTrue("Empty log", !content.isEmpty()); - if (!content.contains(term)) { - String message = "No " + term + " found in logs"; - LOG.error(message); - System.err.println(content); - fail(message); - } - } - - /** - * Assert that a log does not contain the given term. - * @param content log output - * @param term search term - */ - protected void assertNotInLog(String content, String term) { - assertTrue("Empty log", !content.isEmpty()); - if (content.contains(term)) { - String message = term + " found in logs"; - LOG.error(message); - System.err.println(content); - fail(message); - } - } - - /* - * Test case for rename operation with threads and flat listing disabled. - */ - @Test - public void testRenameSmallFolderDisableThreadsDisableFlatListing() throws Exception { - Configuration conf = fs.getConf(); - conf = fs.getConf(); - - // Number of threads set to 0 or 1 disables threads. - conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 1); - conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - validateRenameFolder(fs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Rename operation as thread count 1"); - - // Validate no thread executions - for (int i = 0; i < renameThreads; i++) { - assertNotInLog(content, - "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Helper method to do delete operation and validate all files in source folder - * doesn't exists after delete operation. - */ - private void validateDeleteFolder(FileSystem fs, String source) throws Exception { - // Create folder with files. - createFolder(fs, "root"); - Path sourceFolder = new Path(source); - - // Delete operation - assertTrue(fs.delete(sourceFolder, true)); - assertFalse(fs.exists(sourceFolder)); - - for (int i = 0; i < this.iterations; i++) { - // check that source folder and files doesn't exists - assertFalse(fs.exists(new Path(source + "/" + i))); - assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename"))); - assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename"))); - assertFalse(fs.exists(new Path(source + "/fileToRename" + i))); - } - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteSmallFolderWithThreads() throws Exception { - - validateDeleteFolder(fs, "root"); - - // With single iteration, we would have created 7 blobs. - int expectedThreadsCreated = Math.min(7, deleteThreads); - - // Validate from logs that threads are enabled. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + expectedThreadsCreated); - - // Validate thread executions - for (int i = 0; i < expectedThreadsCreated; i++) { - assertInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - - // Also ensure that we haven't spawned extra threads. - if (expectedThreadsCreated < deleteThreads) { - for (int i = expectedThreadsCreated; i < deleteThreads; i++) { - assertNotInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteLargeFolderWithThreads() throws Exception { - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateDeleteFolder(fs, "root"); - - // Validate from logs that threads are enabled. - String content = logs.getOutput(); - assertInLog(content, "ms with threads: " + deleteThreads); - - // Validate thread executions - for (int i = 0; i < deleteThreads; i++) { - assertInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for delete operation with threads disabled and flat listing enabled. - */ - @Test - public void testDeleteLargeFolderDisableThreads() throws Exception { - Configuration conf = fs.getConf(); - conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 0); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - // Populate source folder with large number of files and directories. - this.iterations = 10; - validateDeleteFolder(fs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Delete operation as thread count 0"); - - // Validate no thread executions - for (int i = 0; i < deleteThreads; i++) { - assertNotInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for rename operation with threads and flat listing disabled. - */ - @Test - public void testDeleteSmallFolderDisableThreadsDisableFlatListing() throws Exception { - Configuration conf = fs.getConf(); - - // Number of threads set to 0 or 1 disables threads. - conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 1); - conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - validateDeleteFolder(fs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Disabling threads for Delete operation as thread count 1"); - - // Validate no thread executions - for (int i = 0; i < deleteThreads; i++) { - assertNotInLog(content, - "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i); - } - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolExceptionFailure() throws Exception { - - // Spy azure file system object and raise exception for new thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception()); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - validateDeleteFolder(mockFs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, "Failed to create thread pool with threads"); - assertInLog(content, "Serializing the Delete operation"); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolExecuteFailure() throws Exception { - - // Mock thread pool executor to throw exception for all requests. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - validateDeleteFolder(mockFs, "root"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Rejected execution of thread for Delete operation on blob"); - assertInLog(content, "Serializing the Delete operation"); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolExecuteSingleThreadFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - // Create a thread executor and link it to mocked thread pool executor object. - ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // Mock thread executor to throw exception for all requests. - Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - validateDeleteFolder(mockFs, "root"); - - // Validate from logs that threads are enabled and unused threads. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads 7"); - assertInLog(content, - "6 threads not used for Delete operation on blob"); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteThreadPoolTerminationFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - ((NativeAzureFileSystem) fs).getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)); - - // Create a thread executor and link it to mocked thread pool executor object. - // Mock thread executor to throw exception for terminating threads. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException()); - - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", - path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor); - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - boolean exception = false; - try { - mockFs.delete(sourceFolder, true); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and delete operation is failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads"); - assertInLog(content, "Threads got interrupted Delete blob operation"); - assertInLog(content, - "Delete failed as operation on subfolders and files failed."); - } - - /* - * Validate that when a directory is deleted recursively, the operation succeeds - * even if a child directory delete fails because the directory does not exist. - * This can happen if a child directory is deleted by an external agent while - * the parent is in progress of being deleted recursively. - */ - @Test - public void testRecursiveDirectoryDeleteWhenChildDirectoryDeleted() - throws Exception { - testRecusiveDirectoryDelete(true); - } - - /* - * Validate that when a directory is deleted recursively, the operation succeeds - * even if a file delete fails because it does not exist. - * This can happen if a file is deleted by an external agent while - * the parent directory is in progress of being deleted. - */ - @Test - public void testRecursiveDirectoryDeleteWhenDeletingChildFileReturnsFalse() - throws Exception { - testRecusiveDirectoryDelete(false); - } - - private void testRecusiveDirectoryDelete(boolean useDir) throws Exception { - String childPathToBeDeletedByExternalAgent = (useDir) - ? "root/0" - : "root/0/fileToRename"; - // Spy azure file system object and return false for deleting one file - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path( - childPathToBeDeletedByExternalAgent))); - - Answer<Boolean> answer = new Answer<Boolean>() { - public Boolean answer(InvocationOnMock invocation) throws Throwable { - String path = (String) invocation.getArguments()[0]; - boolean isDir = (boolean) invocation.getArguments()[1]; - boolean realResult = fs.deleteFile(path, isDir); - assertTrue(realResult); - boolean fakeResult = false; - return fakeResult; - } - }; - - Mockito.when(mockFs.deleteFile(path, useDir)).thenAnswer(answer); - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - - assertTrue(mockFs.delete(sourceFolder, true)); - assertFalse(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled, that a child directory was - // deleted by an external caller, and the parent delete operation still - // succeeds. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads"); - assertInLog(content, String.format("Attempt to delete non-existent %s %s", - useDir ? "directory" : "file", path)); - } - - /* - * Test case for delete operation with multiple threads and flat listing enabled. - */ - @Test - public void testDeleteSingleDeleteException() throws Exception { - - // Spy azure file system object and raise exception for deleting one file - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0"))); - Mockito.doThrow(new IOException()).when(mockFs).deleteFile(path, true); - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - - boolean exception = false; - try { - mockFs.delete(sourceFolder, true); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and delete operation failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Delete operation with threads"); - assertInLog(content, - "Encountered Exception for Delete operation for file " + path); - assertInLog(content, - "Terminating execution of Delete operation now as some other thread already got exception or operation failed"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolExceptionFailure() throws Exception { - - // Spy azure file system object and raise exception for new thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - ((NativeAzureFileSystem) fs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception()); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.doReturn(mockThreadPoolExecutor).when(mockFs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS); - - validateRenameFolder(mockFs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, "Failed to create thread pool with threads"); - assertInLog(content, "Serializing the Rename operation"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolExecuteFailure() throws Exception { - - // Mock thread pool executor to throw exception for all requests. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor); - - validateRenameFolder(mockFs, "root", "rootnew"); - - // Validate from logs that threads are disabled. - String content = logs.getOutput(); - assertInLog(content, - "Rejected execution of thread for Rename operation on blob"); - assertInLog(content, "Serializing the Rename operation"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolExecuteSingleThreadFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor); - - // Create a thread executor and link it to mocked thread pool executor object. - ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7)); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - // Mock thread executor to throw exception for all requests. - Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - - validateRenameFolder(mockFs, "root", "rootnew"); - - // Validate from logs that threads are enabled and unused threads exists. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Rename operation with threads 7"); - assertInLog(content, - "6 threads not used for Rename operation on blob"); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameThreadPoolTerminationFailure() throws Exception { - - // Spy azure file system object and return mocked thread pool - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Spy a thread pool executor and link it to azure file system object. - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root"))); - AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy( - mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)); - - // With single iteration, we would have created 7 blobs resulting 7 threads. - Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename", - path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor); - - // Mock thread executor to throw exception for all requests. - ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class)); - Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException()); - Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor); - - - createFolder(mockFs, "root"); - Path sourceFolder = new Path("root"); - Path destFolder = new Path("rootnew"); - boolean exception = false; - try { - mockFs.rename(sourceFolder, destFolder); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and rename operation is failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Rename operation with threads"); - assertInLog(content, "Threads got interrupted Rename blob operation"); - assertInLog(content, - "Rename failed as operation on subfolders and files failed."); - } - - /* - * Test case for rename operation with multiple threads and flat listing enabled. - */ - @Test - public void testRenameSingleRenameException() throws Exception { - - // Spy azure file system object and raise exception for deleting one file - Path sourceFolder = new Path("root"); - Path destFolder = new Path("rootnew"); - - // Spy azure file system object and populate rename pending spy object. - NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs); - - // Populate data now only such that rename pending spy object would see this data. - createFolder(mockFs, "root"); - - String srcKey = mockFs.pathToKey(mockFs.makeAbsolute(sourceFolder)); - String dstKey = mockFs.pathToKey(mockFs.makeAbsolute(destFolder)); - - FolderRenamePending mockRenameFs = Mockito.spy(mockFs.prepareAtomicFolderRename(srcKey, dstKey)); - Mockito.when(mockFs.prepareAtomicFolderRename(srcKey, dstKey)).thenReturn(mockRenameFs); - String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0"))); - Mockito.doThrow(new IOException()).when(mockRenameFs).renameFile(Mockito.any(FileMetadata.class)); - - boolean exception = false; - try { - mockFs.rename(sourceFolder, destFolder); - } catch (IOException e){ - exception = true; - } - - assertTrue(exception); - assertTrue(mockFs.exists(sourceFolder)); - - // Validate from logs that threads are enabled and delete operation failed. - String content = logs.getOutput(); - assertInLog(content, - "Using thread pool for Rename operation with threads"); - assertInLog(content, - "Encountered Exception for Rename operation for file " + path); - assertInLog(content, - "Terminating execution of Rename operation now as some other thread already got exception or operation failed"); - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java deleted file mode 100644 index 6149154..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; - -import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS; - -/** - * Test class to hold all WASB authorization tests that use blob-specific keys - * to access storage. - */ -public class TestNativeAzureFSAuthWithBlobSpecificKeys - extends TestNativeAzureFileSystemAuthorizationWithOwner { - - @Override - public Configuration getConfiguration() { - Configuration conf = super.getConfiguration(); - conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false"); - return conf; - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = getConfiguration(); - return AzureBlobStorageTestAccount.create(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java deleted file mode 100644 index 84558f8..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.junit.Test; - -import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE; - -/** - * Test class to hold all WASB authorization caching related tests. - */ -public class TestNativeAzureFSAuthorizationCaching - extends TestNativeAzureFileSystemAuthorizationWithOwner { - - private static final int DUMMY_TTL_VALUE = 5000; - - @Override - public Configuration getConfiguration() { - Configuration conf = super.getConfiguration(); - conf.set(KEY_AUTH_SERVICE_CACHING_ENABLE, "true"); - return conf; - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = getConfiguration(); - return AzureBlobStorageTestAccount.create(conf); - } - - /** - * Test to verify cache behavior -- assert that PUT overwrites value if present - */ - @Test - public void testCachePut() throws Throwable { - CachingAuthorizer<String, Integer> cache = new CachingAuthorizer<>(DUMMY_TTL_VALUE, "TEST"); - cache.init(getConfiguration()); - cache.put("TEST", 1); - cache.put("TEST", 3); - int result = cache.get("TEST"); - ContractTestUtils.assertTrue("Cache returned unexpected result", result == 3); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java deleted file mode 100644 index 208cff3..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; - -/** - * Run the base Azure file system tests strictly on page blobs to make sure fundamental - * operations on page blob files and folders work as expected. - * These operations include create, delete, rename, list, and so on. - */ -public class TestNativeAzureFSPageBlobLive extends - NativeAzureFileSystemBaseTest { - - @Override - protected AzureBlobStorageTestAccount createTestAccount() - throws Exception { - Configuration conf = new Configuration(); - - // Configure the page blob directories key so every file created is a page blob. - conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/"); - - // Configure the atomic rename directories key so every folder will have - // atomic rename applied. - conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/"); - return AzureBlobStorageTestAccount.create(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java deleted file mode 100644 index a2b35cb..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java +++ /dev/null @@ -1,362 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.IOException; -import java.net.URI; -import java.util.Arrays; - -import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestNativeAzureFileSystemAppend extends AbstractWasbTestBase { - - private static final String TEST_FILE = "test.dat"; - private static final Path TEST_PATH = new Path(TEST_FILE); - - private AzureBlobStorageTestAccount testAccount = null; - - @Before - public void setUp() throws Exception { - super.setUp(); - testAccount = createTestAccount(); - fs = testAccount.getFileSystem(); - Configuration conf = fs.getConf(); - conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, true); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - } - - /* - * Helper method that creates test data of size provided by the - * "size" parameter. - */ - private static byte[] getTestData(int size) { - byte[] testData = new byte[size]; - System.arraycopy(RandomStringUtils.randomAlphabetic(size).getBytes(), 0, testData, 0, size); - return testData; - } - - // Helper method to create file and write fileSize bytes of data on it. - private byte[] createBaseFileWithData(int fileSize, Path testPath) throws Throwable { - - FSDataOutputStream createStream = null; - try { - createStream = fs.create(testPath); - byte[] fileData = null; - - if (fileSize != 0) { - fileData = getTestData(fileSize); - createStream.write(fileData); - } - return fileData; - } finally { - if (createStream != null) { - createStream.close(); - } - } - } - - /* - * Helper method to verify a file data equal to "dataLength" parameter - */ - private boolean verifyFileData(int dataLength, byte[] testData, int testDataIndex, - FSDataInputStream srcStream) { - - try { - - byte[] fileBuffer = new byte[dataLength]; - byte[] testDataBuffer = new byte[dataLength]; - - int fileBytesRead = srcStream.read(fileBuffer); - - if (fileBytesRead < dataLength) { - return false; - } - - System.arraycopy(testData, testDataIndex, testDataBuffer, 0, dataLength); - - if (!Arrays.equals(fileBuffer, testDataBuffer)) { - return false; - } - - return true; - - } catch (Exception ex) { - return false; - } - - } - - /* - * Helper method to verify Append on a testFile. - */ - private boolean verifyAppend(byte[] testData, Path testFile) { - - FSDataInputStream srcStream = null; - try { - - srcStream = fs.open(testFile); - int baseBufferSize = 2048; - int testDataSize = testData.length; - int testDataIndex = 0; - - while (testDataSize > baseBufferSize) { - - if (!verifyFileData(baseBufferSize, testData, testDataIndex, srcStream)) { - return false; - } - testDataIndex += baseBufferSize; - testDataSize -= baseBufferSize; - } - - if (!verifyFileData(testDataSize, testData, testDataIndex, srcStream)) { - return false; - } - - return true; - } catch(Exception ex) { - return false; - } finally { - if (srcStream != null) { - try { - srcStream.close(); - } catch(IOException ioe) { - // Swallowing - } - } - } - } - - /* - * Test case to verify if an append on small size data works. This tests - * append E2E - */ - @Test - public void testSingleAppend() throws Throwable{ - - FSDataOutputStream appendStream = null; - try { - int baseDataSize = 50; - byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH); - - int appendDataSize = 20; - byte[] appendDataBuffer = getTestData(appendDataSize); - appendStream = fs.append(TEST_PATH, 10); - appendStream.write(appendDataBuffer); - appendStream.close(); - byte[] testData = new byte[baseDataSize + appendDataSize]; - System.arraycopy(baseDataBuffer, 0, testData, 0, baseDataSize); - System.arraycopy(appendDataBuffer, 0, testData, baseDataSize, appendDataSize); - - Assert.assertTrue(verifyAppend(testData, TEST_PATH)); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - /* - * Test case to verify append to an empty file. - */ - @Test - public void testSingleAppendOnEmptyFile() throws Throwable { - - FSDataOutputStream appendStream = null; - - try { - createBaseFileWithData(0, TEST_PATH); - - int appendDataSize = 20; - byte[] appendDataBuffer = getTestData(appendDataSize); - appendStream = fs.append(TEST_PATH, 10); - appendStream.write(appendDataBuffer); - appendStream.close(); - - Assert.assertTrue(verifyAppend(appendDataBuffer, TEST_PATH)); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - /* - * Test to verify that we can open only one Append stream on a File. - */ - @Test - public void testSingleAppenderScenario() throws Throwable { - - FSDataOutputStream appendStream1 = null; - FSDataOutputStream appendStream2 = null; - IOException ioe = null; - try { - createBaseFileWithData(0, TEST_PATH); - appendStream1 = fs.append(TEST_PATH, 10); - boolean encounteredException = false; - try { - appendStream2 = fs.append(TEST_PATH, 10); - } catch(IOException ex) { - encounteredException = true; - ioe = ex; - } - - appendStream1.close(); - - Assert.assertTrue(encounteredException); - GenericTestUtils.assertExceptionContains("Unable to set Append lease on the Blob", ioe); - } finally { - if (appendStream1 != null) { - appendStream1.close(); - } - - if (appendStream2 != null) { - appendStream2.close(); - } - } - } - - /* - * Tests to verify multiple appends on a Blob. - */ - @Test - public void testMultipleAppends() throws Throwable { - - int baseDataSize = 50; - byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH); - - int appendDataSize = 100; - int targetAppendCount = 50; - byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)]; - int testDataIndex = 0; - System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize); - testDataIndex += baseDataSize; - - int appendCount = 0; - - FSDataOutputStream appendStream = null; - - try { - while (appendCount < targetAppendCount) { - - byte[] appendDataBuffer = getTestData(appendDataSize); - appendStream = fs.append(TEST_PATH, 30); - appendStream.write(appendDataBuffer); - appendStream.close(); - - System.arraycopy(appendDataBuffer, 0, testData, testDataIndex, appendDataSize); - testDataIndex += appendDataSize; - appendCount++; - } - - Assert.assertTrue(verifyAppend(testData, TEST_PATH)); - - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - /* - * Test to verify we multiple appends on the same stream. - */ - @Test - public void testMultipleAppendsOnSameStream() throws Throwable { - - int baseDataSize = 50; - byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH); - int appendDataSize = 100; - int targetAppendCount = 50; - byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)]; - int testDataIndex = 0; - System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize); - testDataIndex += baseDataSize; - int appendCount = 0; - - FSDataOutputStream appendStream = null; - - try { - - while (appendCount < targetAppendCount) { - - appendStream = fs.append(TEST_PATH, 50); - - int singleAppendChunkSize = 20; - int appendRunSize = 0; - while (appendRunSize < appendDataSize) { - - byte[] appendDataBuffer = getTestData(singleAppendChunkSize); - appendStream.write(appendDataBuffer); - System.arraycopy(appendDataBuffer, 0, testData, - testDataIndex + appendRunSize, singleAppendChunkSize); - - appendRunSize += singleAppendChunkSize; - } - - appendStream.close(); - testDataIndex += appendDataSize; - appendCount++; - } - - Assert.assertTrue(verifyAppend(testData, TEST_PATH)); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - @Test(expected=UnsupportedOperationException.class) - /* - * Test to verify the behavior when Append Support configuration flag is set to false - */ - public void testFalseConfigurationFlagBehavior() throws Throwable { - - fs = testAccount.getFileSystem(); - Configuration conf = fs.getConf(); - conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - - FSDataOutputStream appendStream = null; - - try { - createBaseFileWithData(0, TEST_PATH); - appendStream = fs.append(TEST_PATH, 10); - } finally { - if (appendStream != null) { - appendStream.close(); - } - } - } - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - return AzureBlobStorageTestAccount.create(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java deleted file mode 100644 index 602c1f7..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.IOException; -import java.net.URI; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; - -public class TestNativeAzureFileSystemAtomicRenameDirList - extends AbstractWasbTestBase { - private AzureBlobStorageTestAccount testAccount; - - // HBase-site config controlling HBase root dir - private static final String HBASE_ROOT_DIR_CONF_STRING = "hbase.rootdir"; - private static final String HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS = "wasb://somedifferentfilesystem.blob.core.windows.net/hbase"; - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - testAccount = AzureBlobStorageTestAccount.create(); - return testAccount; - } - - @Test - public void testAzureNativeStoreIsAtomicRenameKeyDoesNotThrowNPEOnInitializingWithNonDefaultURI () throws IOException { - NativeAzureFileSystem azureFs = (NativeAzureFileSystem)fs; - AzureNativeFileSystemStore azureStore = azureFs.getStore(); - Configuration conf = fs.getConf(); - conf.set(HBASE_ROOT_DIR_CONF_STRING, HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS); - URI uri = fs.getUri(); - fs.initialize(uri, conf); - azureStore.isAtomicRenameKey("anyrandomkey"); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org