[
https://issues.apache.org/jira/browse/HADOOP-19474?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17938573#comment-17938573
]
ASF GitHub Bot commented on HADOOP-19474:
-----------------------------------------
anmolanmol1234 commented on code in PR #7421:
URL: https://github.com/apache/hadoop/pull/7421#discussion_r2013968726
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java:
##########
@@ -343,4 +319,174 @@ public void testRenameTrailingPeriodFile() throws
IOException {
assertTrue("Attempt to create file that ended with a dot should"
+ " throw IllegalArgumentException", exceptionThrown);
}
+
+
+
+ /**
+ * Test to verify that listStatus returns the correct file status all types
+ * of paths viz. implicit, explicit, file.
+ * @throws Exception if there is an error or test assertions fails.
+ */
+ @Test
+ public void testListStatusWithImplicitExplicitChildren() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.setWorkingDirectory(new Path(ROOT_PATH));
+ Path root = new Path(ROOT_PATH);
+
+ // Create an implicit directory under root
+ Path dir = new Path("a");
+ Path fileInsideDir = new Path("a/file");
+ createAzCopyFolder(dir);
+
+ // Assert that implicit directory is returned
+ FileStatus[] fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(1);
+ assertImplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir));
+
+ // Create a marker blob for the directory.
+ fs.create(fileInsideDir);
+
+ // Assert that only one entry of explicit directory is returned
+ fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(1);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir));
+
+ // Create a file under root
+ Path file1 = new Path("b");
+ fs.create(file1);
+
+ // Assert that two entries are returned in alphabetic order.
+ fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(2);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir));
+ assertFileFileStatus(fileStatuses[1], fs.makeQualified(file1));
+
+ // Create another implicit directory under root.
+ Path dir2 = new Path("c");
+ createAzCopyFolder(dir2);
+
+ // Assert that three entries are returned in alphabetic order.
+ fileStatuses = fs.listStatus(root);
+ Assertions.assertThat(fileStatuses.length).isEqualTo(3);
+ assertExplicitDirectoryFileStatus(fileStatuses[0], fs.makeQualified(dir));
+ assertFileFileStatus(fileStatuses[1], fs.makeQualified(file1));
+ assertImplicitDirectoryFileStatus(fileStatuses[2], fs.makeQualified(dir2));
+ }
+
+ /**
+ * Test to verify that listStatus returns the correct file status when
called on an implicit path
+ * @throws Exception if there is an error or test assertions fails.
+ */
+ @Test
+ public void testListStatusOnImplicitDirectoryPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path implicitPath = new Path("/implicitDir");
+ createAzCopyFolder(implicitPath);
+
+ FileStatus[] statuses = fs.listStatus(implicitPath);
+ Assertions.assertThat(statuses.length).isGreaterThanOrEqualTo(1);
+ assertImplicitDirectoryFileStatus(statuses[0],
fs.makeQualified(statuses[0].getPath()));
+
+ FileStatus[] statuses1 = fs.listStatus(new
Path(statuses[0].getPath().toString()));
+ Assertions.assertThat(statuses1.length).isGreaterThanOrEqualTo(1);
+ assertFileFileStatus(statuses1[0],
fs.makeQualified(statuses1[0].getPath()));
+ }
+
+ @Test
+ public void testListStatusOnEmptyDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path emptyDir = new Path("/emptyDir");
+ fs.mkdirs(emptyDir);
+
+ FileStatus[] statuses = fs.listStatus(emptyDir);
+ Assertions.assertThat(statuses.length).isEqualTo(0);
+ }
+
+ @Test
+ public void testContinuationTokenAcrossListStatus() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path path = new Path("/testContinuationToken");
+ fs.mkdirs(path);
+ fs.create(new Path(path + "/file1"));
+ fs.create(new Path(path + "/file2"));
+
+ fs.listStatus(path);
+
+ ListResponseData listResponseData = fs.getAbfsStore().getClient().listPath(
+ "/testContinuationToken", false, 1, null, getTestTracingContext(fs,
true),
+ fs.getAbfsStore().getUri());
+
+ Assertions.assertThat(listResponseData.getContinuationToken()).isNotNull();
+ Assertions.assertThat(listResponseData.getFileStatusList()).hasSize(1);
+
+ ListResponseData listResponseData1 =
fs.getAbfsStore().getClient().listPath(
+ "/testContinuationToken", false, 1,
listResponseData.getContinuationToken(), getTestTracingContext(fs, true),
+ fs.getAbfsStore().getUri());
+
+ Assertions.assertThat(listResponseData1.getContinuationToken()).isNull();
+ Assertions.assertThat(listResponseData1.getFileStatusList()).hasSize(1);
+ }
+
+ @Test
+ public void testInvalidContinuationToken() throws Exception {
+ assumeHnsDisabled();
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path path = new Path("/testInvalidContinuationToken");
+ fs.mkdirs(path);
+ fs.create(new Path(path + "/file1"));
+ fs.create(new Path(path + "/file2"));
+
+ intercept(AbfsRestOperationException.class,
+ () -> fs.getAbfsStore().getClient().listPath(
+ "/testInvalidContinuationToken", false, 1, "invalidToken",
+ getTestTracingContext(fs, true), fs.getAbfsStore().getUri()));
+ }
+
+ @Test
+ public void testEmptyContinuationToken() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path path = new Path("/testInvalidContinuationToken");
+ fs.mkdirs(path);
+ fs.create(new Path(path + "/file1"));
+ fs.create(new Path(path + "/file2"));
+
+ ListResponseData listResponseData = fs.getAbfsStore().getClient().listPath(
+ "/testInvalidContinuationToken", false, 1, "",
+ getTestTracingContext(fs, true), fs.getAbfsStore().getUri());
+
+ Assertions.assertThat(listResponseData.getContinuationToken()).isNotNull();
+ Assertions.assertThat(listResponseData.getFileStatusList()).hasSize(1);
+ }
+
+ private void assertFileFileStatus(final FileStatus fileStatus,
Review Comment:
typo in name
> ABFS: [FnsOverBlob] Listing Optimizations to avoid multiple iteration over
> list response.
> -----------------------------------------------------------------------------------------
>
> Key: HADOOP-19474
> URL: https://issues.apache.org/jira/browse/HADOOP-19474
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.5.0, 3.4.1
> Reporter: Anuj Modi
> Assignee: Anuj Modi
> Priority: Major
> Labels: pull-request-available
>
> On blob endpoint, there are a couple of handling that is needed to be done on
> client side.
> This involves:
> # Parsing of xml response and converting them to VersionedFileStatus list
> # Removing duplicate entries for non-empty explicit directories coming due
> to presence of the marker files
> # Trigerring Rename recovery on the previously failed rename indicated by
> the presence of pending json file.
> Currently all three are done in a separate iteration over whole list. This is
> to pbring all those things to a common place so that single iteration over
> list reposne can handle all three.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
