[
https://issues.apache.org/jira/browse/HADOOP-19232?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17913576#comment-17913576
]
ASF GitHub Bot commented on HADOOP-19232:
-----------------------------------------
anmolanmol1234 commented on code in PR #7272:
URL: https://github.com/apache/hadoop/pull/7272#discussion_r1917814590
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java:
##########
@@ -504,4 +538,875 @@ private <E extends Throwable> void
validateCreateFileException(final Class<E> ex
private AbfsRestOperationException getMockAbfsRestOperationException(int
status) {
return new AbfsRestOperationException(status, "", "", new Exception());
}
-}
+
+
+ /**
+ * Attempts to test multiple flush calls.
+ */
+ @Test
+ public void testMultipleFlush() throws Throwable {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
+ try (FSDataOutputStream out = fs.create(testPath)) {
+ out.write('1');
+ out.hsync();
+ out.write('2');
+ out.hsync();
+ }
+ }
+
+ /**
+ * Delete the blob before flush and verify that an exception should be
thrown.
+ */
+ @Test
+ public void testDeleteBeforeFlush() throws Throwable {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
+ try (FSDataOutputStream out = fs.create(testPath)) {
+ out.write('1');
+ fs.delete(testPath, false);
+ out.hsync();
+ // this will cause the next write to failAll
+ } catch (IOException fnfe) {
+ //appendblob outputStream does not generate suppressed exception on
close as it is
+ //single threaded code
+ if
(!fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(testPath).toString())) {
+ // the exception raised in close() must be in the caught exception's
+ // suppressed list
+ Throwable[] suppressed = fnfe.getSuppressed();
+ assertEquals("suppressed count", 1, suppressed.length);
+ Throwable inner = suppressed[0];
+ if (!(inner instanceof IOException)) {
+ throw inner;
+ }
+ GenericTestUtils.assertExceptionContains(fnfe.getMessage(),
inner.getCause(), inner.getCause().getMessage());
+ }
+ }
+ }
+
+ /**
+ * Creating subdirectory on existing file path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/d"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d/e")));
+
+ Assertions.assertThat(fs.exists(new Path("a/b/c"))).isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/d"))).isTrue();
+ // Asserting directory created still exists as explicit.
+ Assertions.assertThat(
+ DirectoryStateHelper.isExplicitDirectory(new Path("a/b/d"), fs,
+ getTestTracingContext(fs, true)))
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Try creating file same as an existing directory.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateDirectoryAndFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c"))).isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b/c")));
+ // Asserting that directory still exists as explicit
+ Assertions.assertThat(
+ DirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creating same file without specifying overwrite.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ fs.create(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ }
+
+ /**
+ * Creating same file with overwrite flag set to false.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameFileWithOverwriteFalse() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b/c"), false));
+ }
+
+ /**
+ * Creation of already existing subpath should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSubPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b")));
+ }
+
+ /**
+ * Creating path with parent explicit.
+ */
+ @Test
+ public void testCreatePathParentExplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ fs.create(new Path("a/b/c/d"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d")))
+ .describedAs("Path does not exist")
+ .isTrue();
+
+ // asserting that parent stays explicit
+ Assertions.assertThat(
+ DirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Test create on implicit directory with explicit parent.
+ * @throws Exception
+ */
+ @Test
+ public void testParentExplicitPathImplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getClientHandler().getIngressClient()
instanceof AbfsBlobClient);
+ fs.mkdirs(new Path("/explicitParent"));
+ String sourcePathName = "/explicitParent/implicitDir";
+ Path sourcePath = new Path(sourcePathName);
+ createAzCopyFolder(sourcePath);
+
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, true));
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, false));
+
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(sourcePath.getParent(),
fs, getTestTracingContext(fs, true)))
+ .describedAs("Parent directory should be explicit.")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(sourcePath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Path should be implicit.")
+ .isTrue();
+ }
+
+ /**
+ * Test create on implicit directory with implicit parent
+ * @throws Exception
+ */
+ @Test
+ public void testParentImplicitPathImplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+ Assume.assumeTrue(fs.getAbfsStore().getClientHandler().getIngressClient()
instanceof AbfsBlobClient);
+ String parentPathName = "/implicitParent";
+ Path parentPath = new Path(parentPathName);
+ String sourcePathName = "/implicitParent/implicitDir";
+ Path sourcePath = new Path(sourcePathName);
+
+ createAzCopyFolder(parentPath);
+ createAzCopyFolder(sourcePath);
+
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, true));
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, false));
+
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(parentPath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Parent directory is implicit.")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(sourcePath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Path should also be implicit.")
+ .isTrue();
+ }
+
+ /**
+ * Tests create file when file exists already
+ * Verifies using eTag for overwrite = true/false
+ */
+ @Test
+ public void testCreateFileExistsImplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+ String parentPathName = "/implicitParent";
+ Path parentPath = new Path(parentPathName);
+ createAzCopyFolder(parentPath);
+
+ String fileName = "/implicitParent/testFile";
+ Path filePath = new Path(fileName);
+ fs.create(filePath);
+ String eTag = extractFileEtag(fileName);
+
+ // testing createFile on already existing file path
+ fs.create(filePath, true);
+
+ String eTagAfterCreateOverwrite = extractFileEtag(fileName);
+
+ Assertions.assertThat(eTag.equals(eTagAfterCreateOverwrite))
+ .describedAs("New file eTag after create overwrite should be different
from old")
+ .isFalse();
+
+ intercept(IOException.class, () ->
+ fs.create(filePath, false));
+
+ String eTagAfterCreate = extractFileEtag(fileName);
+
+ Assertions.assertThat(eTagAfterCreateOverwrite.equals(eTagAfterCreate))
+ .describedAs("File eTag should not change as creation fails")
+ .isTrue();
+
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(parentPath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Parent path should also change to explicit.")
+ .isTrue();
+ }
+
+ /**
+ * Tests create file when the parent is an existing file
+ * should fail.
+ * @throws Exception FileAlreadyExists for blob and IOException for dfs.
+ */
+ @Test
+ public void testCreateFileParentFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+
+ String parentName = "/testParentFile";
+ Path parent = new Path(parentName);
+ fs.create(parent);
+
+ String childName = "/testParentFile/testChildFile";
+ Path child = new Path(childName);
+ IOException e = intercept(IOException.class, () ->
+ fs.create(child, false));
+
+ // asserting that parent stays explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path(parentName)),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not a file")
+ .isFalse();
+ }
+
+ /**
+ * Creating directory on existing file path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateMkdirs() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d")));
+ }
+
+ /**
+ * Test mkdirs.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirs() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b"));
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.mkdirs(new Path("a/b/c/e"));
+
+ Assertions.assertThat(fs.exists(new Path("a/b")))
+ .describedAs("Path a/b does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d")))
+ .describedAs("Path a/b/c/d does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/e")))
+ .describedAs("Path a/b/c/e does not exist")
+ .isTrue();
+
+ // Asserting that directories created as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a/b is not an explicit directory")
+ .isTrue();
+ FileStatus status1 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c/d")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status1.isDirectory())
+ .describedAs("Path a/b/c/d is not an explicit directory")
+ .isTrue();
+ FileStatus status2 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c/e")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status2.isDirectory())
+ .describedAs("Path a/b/c/e is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creating subpath of directory path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsCreateSubPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b")));
+
+ // Asserting that directories created as explicit
+ FileStatus status2 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status2.isDirectory())
+ .describedAs("Path a/b/c is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Test creation of directory by level.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsByLevel() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a"));
+ fs.mkdirs(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/c/d/e"));
+
+ Assertions.assertThat(fs.exists(new Path("a")))
+ .describedAs("Path a does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d/e")))
+ .describedAs("Path a/b/c/d/e does not exist")
+ .isTrue();
+
+ // Asserting that directories created as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a is not an explicit directory")
+ .isTrue();
+ FileStatus status1 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status1.isDirectory())
+ .describedAs("Path a/b/c is not an explicit directory")
+ .isTrue();
+ FileStatus status2 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c/d/e")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status2.isDirectory())
+ .describedAs("Path a/b/c/d/e is not an explicit directory")
+ .isTrue();
+ }
+
+ /*
+ Delete part of a path and validate sub path exists.
+ */
+ @Test
+ public void testMkdirsWithDelete() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b"));
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.delete(new Path("a/b/c/d"));
+ fs.getFileStatus(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ }
+
+ /**
+ * Verify mkdir and rename of parent.
+ */
+ @Test
+ public void testMkdirsWithRename() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.create(new Path("e/file"));
+ fs.delete(new Path("a/b/c/d"));
+ Assertions.assertThat(fs.rename(new Path("e"), new Path("a/b/c/d")))
+ .describedAs("Failed to rename path e to a/b/c/d")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d/file")))
+ .describedAs("Path a/b/c/d/file does not exist")
+ .isTrue();
+ }
+
+ /**
+ * Create a file with name /dir1 and then mkdirs for /dir1/dir2 should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testFileCreateMkdirsRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.setWorkingDirectory(new Path("/"));
+ final Path p1 = new Path("dir1");
+ fs.create(p1);
+ intercept(IOException.class, () -> fs.mkdirs(new Path("dir1/dir2")));
+ }
+
+ /**
+ * Create a file with name /dir1 and then mkdirs for /dir1/dir2 should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testFileCreateMkdirsNonRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path p1 = new Path("dir1");
+ fs.create(p1);
+ intercept(IOException.class, () -> fs.mkdirs(new Path("dir1/dir2")));
+ }
+
+ /**
+ * Creation of same directory without overwrite flag should pass.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/c"));
+
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ // Asserting that directories created as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a/b/c is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of same directory without overwrite flag should pass.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSamePathDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a")));
+ }
+
+ /**
+ * Creation of directory with root as parent
+ */
+ @Test
+ public void testMkdirOnRootAsParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("a");
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory on root
+ */
+ @Test
+ public void testMkdirOnRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/");
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(path);
+
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("/")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory on path with unicode chars
+ */
+ @Test
+ public void testMkdirUnicode() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/dir\u0031");
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(path),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory on same path with parallel threads.
+ */
+ @Test
+ public void testMkdirParallelRequests() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/dir1");
+
+ ExecutorService es = Executors.newFixedThreadPool(3);
+
+ List<CompletableFuture<Void>> tasks = new ArrayList<>();
+
+ for (int i = 0; i < 3; i++) {
+ CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
+ try {
+ fs.mkdirs(path);
+ } catch (IOException e) {
+ throw new CompletionException(e);
+ }
+ }, es);
+ tasks.add(future);
+ }
+
+ // Wait for all the tasks to complete
+ CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).join();
+
+ // Assert that the directory created by mkdir exists as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(path),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+
+ /**
+ * Creation of directory with overwrite set to false should not fail
according to DFS code.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameDirectoryOverwriteFalse() throws Exception {
+ Configuration configuration = getRawConfiguration();
+ configuration.setBoolean(FS_AZURE_ENABLE_MKDIR_OVERWRITE, false);
+ AzureBlobFileSystem fs1 = (AzureBlobFileSystem)
FileSystem.newInstance(configuration);
+ fs1.mkdirs(new Path("a/b/c"));
+ fs1.mkdirs(new Path("a/b/c"));
+
+ // Asserting that directories created as explicit
+ FileStatus status = fs1.getAbfsStore().getFileStatus(fs1.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs1, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Try creating directory same as an existing file.
+ */
+ @Test
+ public void testCreateDirectoryAndFileRecreation() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ fs.create(new Path("a/b/c/d"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Directory a/b/c does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d")))
+ .describedAs("File a/b/c/d does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d")));
+ }
+
+ @Test
+ public void testCreateNonRecursiveForAtomicDirectoryFile() throws Exception {
+ AzureBlobFileSystem fileSystem = getFileSystem();
+ fileSystem.setWorkingDirectory(new Path("/"));
+ fileSystem.mkdirs(new Path("/hbase/dir"));
+ fileSystem.createFile(new Path("/hbase/dir/file"))
+ .overwrite(false)
+ .replication((short) 1)
+ .bufferSize(1024)
+ .blockSize(1024)
+ .build();
+ Assertions.assertThat(fileSystem.exists(new Path("/hbase/dir/file")))
+ .describedAs("File /hbase/dir/file does not exist")
+ .isTrue();
+ }
+
+ @Test
+ public void testMkdirOnNonExistingPathWithImplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+ createAzCopyFolder(implicitPath);
+
+ // Creating a directory on non-existing path inside an implicit directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as explicit directory
+ * @throws Exception
+ */
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as explicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingExplicitDirWithImplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating implicit directory to be used as parent
+ createAzCopyFolder(implicitPath);
+
+ // Creating an explicit directory on the path first
+ fs.mkdirs(path);
+
+ // Creating a directory on existing explicit directory inside an implicit
directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as explicit.
+ * And the directory to be created existing as implicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingImplicitDirWithExplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path explicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating an explicit directory to be used a parent
+ fs.mkdirs(explicitPath);
+
+ createAzCopyFolder(path);
+
+ // Creating a directory on existing implicit directory inside an explicit
directory
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(explicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Explicit parent directory does not exist as explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as implicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingImplicitDirWithImplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir3");
+ final Path path = new Path("dir3/dir4");
+
+ createAzCopyFolder(implicitPath);
+
+ // Creating an implicit directory on path
+ createAzCopyFolder(path);
+
+ // Creating a directory on existing implicit directory inside an implicit
directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as file
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingFileWithImplicitParentDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ createAzCopyFolder(implicitPath);
+
+ // Creating a file on path
+ fs.create(path);
+
+ // Creating a directory on existing file inside an implicit directory
+ // Asserting that the mkdir fails
+ LambdaTestUtils.intercept(FileAlreadyExistsException.class, () -> {
+ fs.mkdirs(path);
+ });
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the file still exists at path.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("File still exists at path")
+ .isFalse();
+ }
+
+ /**
+ * 1. a/b/c as implicit.
+ * 2. Create marker for b.
+ * 3. Do mkdir on a/b/c/d.
+ * 4. Verify all b,c,d have marker but a is implicit.
+ */
+ @Test
+ public void testImplicitExplicitFolder() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ final AzureBlobFileSystem fs = (AzureBlobFileSystem)
FileSystem.newInstance(configuration);
+ final Path implicitPath = new Path("a/b/c");
+
+ createAzCopyFolder(implicitPath);
+
+ Path path = makeQualified(new Path("a/b"));
+ AbfsBlobClient blobClient = (AbfsBlobClient)
fs.getAbfsStore().getClient(AbfsServiceType.BLOB);
+ blobClient.createPath(path.toUri().getPath(), false, true,
+ null, false, null, null, getTestTracingContext(fs, true),
+ true);
+
+ fs.mkdirs(new Path("a/b/c/d"));
+
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(new
Path("a"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a' should be implicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c/d"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c/d' should be explicit")
+ .isTrue();
+ }
+
+ /**
+ * 1. a/b/c implicit.
+ * 2. Marker for a and c.
+ * 3. mkdir on a/b/c/d.
+ * 4. Verify a,c,d are explicit but b is implicit.
+ */
+ @Test
+ public void testImplicitExplicitFolder1() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ final AzureBlobFileSystem fs = (AzureBlobFileSystem)
FileSystem.newInstance(configuration);
+ final Path implicitPath = new Path("a/b/c");
+
+ createAzCopyFolder(implicitPath);
+
+ Path path = makeQualified(new Path("a"));
+ AbfsBlobClient blobClient = (AbfsBlobClient)
fs.getAbfsStore().getClient(AbfsServiceType.BLOB);
+ blobClient.createPath(path.toUri().getPath(), false, true,
+ null, false, null, null, getTestTracingContext(fs, true), true);
+
+ Path newPath = makeQualified(new Path("a/b/c"));
+ blobClient.createPath(newPath.toUri().getPath(), false, true,
+ null, false, null, null, getTestTracingContext(fs, true), true);
+
+ fs.mkdirs(new Path("a/b/c/d"));
+
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(new
Path("a/b"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b' should be implicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c/d"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c/d' should be explicit")
+ .isTrue();
+ }
+
+ /**
+ * Extracts the eTag for an existing file
+ * @param fileName file Path in String from container root
+ * @return String etag for the file
+ * @throws IOException
+ */
+ private String extractFileEtag(String fileName) throws IOException {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AbfsClient client = fs.getAbfsClient();
+ final TracingContext testTracingContext = getTestTracingContext(fs, false);
+ AbfsRestOperation op;
+ op = client.getPathStatus(fileName, true, testTracingContext, null);
+ return AzureBlobFileSystemStore.extractEtagHeader(op.getResult());
+ }
+}
Review Comment:
taken
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java:
##########
@@ -370,4 +397,4 @@ public void testAcquireRetry() throws Exception {
tracingContext);
});
}
-}
+}
Review Comment:
taken
> ABFS: [FnsOverBlob] Implementing Ingress Support with various Fallback
> Handling
> -------------------------------------------------------------------------------
>
> Key: HADOOP-19232
> URL: https://issues.apache.org/jira/browse/HADOOP-19232
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.4.0
> Reporter: Anuj Modi
> Assignee: Anmol Asrani
> Priority: Major
> Labels: pull-request-available
>
> Scope of this task is to refactor the AbfsOutputStream class to handle the
> ingress for DFS and Blob endpoint effectively.
> More details will be added soon.
> Perquisites for this Patch:
> 1. [HADOOP-19187] ABFS: [FnsOverBlob]Making AbfsClient Abstract for
> supporting both DFS and Blob Endpoint - ASF JIRA (apache.org)
> 2. [HADOOP-19226] ABFS: [FnsOverBlob]Implementing Azure Rest APIs on Blob
> Endpoint for AbfsBlobClient - ASF JIRA (apache.org)
> 3. [HADOOP-19207] ABFS: [FnsOverBlob]Response Handling of Blob Endpoint APIs
> and Metadata APIs - ASF JIRA (apache.org)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
