[
https://issues.apache.org/jira/browse/HADOOP-19604?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18007803#comment-18007803
]
ASF GitHub Bot commented on HADOOP-19604:
-----------------------------------------
anmolanmol1234 commented on code in PR #7777:
URL: https://github.com/apache/hadoop/pull/7777#discussion_r2213186309
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java:
##########
@@ -201,4 +325,1650 @@ public void testSetWorkingDirectory() throws Exception {
assertEquals(path3, wasb.getWorkingDirectory());
assertEquals(path3, abfs.getWorkingDirectory());
}
+
+ // Scenario wise testing
+
+ //Scenario 1: - Create and write via WASB, read via ABFS
+ @Test
+ public void testScenario1() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+ // Check file status
+ ContractTestUtils.assertIsFile(wasb, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 2: - Create and write via WASB, read via ABFS and then write the
same file via ABFS
+ @Test
+ public void testScenario2() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+ // Check file status
+ ContractTestUtils.assertIsFile(wasb, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Write
+ try (FSDataOutputStream abfsOutputStream = abfs.append(path)) {
+ abfsOutputStream.write(TEST_CONTEXT1.getBytes());
+ abfsOutputStream.flush();
+ abfsOutputStream.hsync();
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 3: - Create and write via ABFS and the read via WASB
+ @Test
+ public void testScenario3() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(wasb.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + wasb,
+ TEST_CONTEXT, line);
+ }
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 4:- Create via WASB, write via ABFS and then write via WASB
+ @Test
+ public void testScenario4() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ wasb.create(path, true);
+ try (FSDataOutputStream abfsOutputStream = abfs.append(path)) {
+ abfsOutputStream.write(TEST_CONTEXT.getBytes());
+ abfsOutputStream.flush();
+ abfsOutputStream.hsync();
+ }
+
+ try (FSDataOutputStream nativeFsStream = wasb.append(path)) {
+ nativeFsStream.write(TEST_CONTEXT1.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 5:- Create via ABFS, write via WASB, read via ABFS (Checksum
validation disabled)
+ @Test
+ public void testScenario5() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, false);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ abfs.create(path, true);
+ try (FSDataOutputStream nativeFsStream = wasb.append(path)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 6: - Create via ABFS, write via WASB, read via ABFS (Checksum
validation enabled)
+ @Test
+ public void testScenario6() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ assumeBlobServiceType();
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ abfs.create(path, true);
+ try (FSDataOutputStream nativeFsStream = wasb.append(path)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ // Scenario 7 :- Create via WASB and then create overwrite true using ABFS
+ @Test
+ public void testScenario7() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+ abfs.create(path, true);
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ // Scenario 8 :- Create via WASB and then create overwrite false using ABFS
+ @Test
+ public void testScenario8() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+ try {
+ abfs.create(path, false);
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("AlreadyExists"));
Review Comment:
taken
> ABFS: Fix WASB ABFS compatibility issues
> ----------------------------------------
>
> Key: HADOOP-19604
> URL: https://issues.apache.org/jira/browse/HADOOP-19604
> Project: Hadoop Common
> Issue Type: Sub-task
> Affects Versions: 3.4.1
> Reporter: Anmol Asrani
> Assignee: Anmol Asrani
> Priority: Major
> Labels: pull-request-available
> Fix For: 3.4.1
>
>
> Fix WASB ABFS compatibility issues. Fix issues such as:-
> # BlockId computation to be consistent across clients for PutBlock and
> PutBlockList
> # Restrict url encoding of certain json metadata during setXAttr calls.
> # Maintain the md5 hash of whole block to validate data integrity during
> flush.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
