anmolanmol1234 commented on code in PR #7777:
URL: https://github.com/apache/hadoop/pull/7777#discussion_r2213200151
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java:
##########
@@ -201,4 +325,1650 @@ public void testSetWorkingDirectory() throws Exception {
assertEquals(path3, wasb.getWorkingDirectory());
assertEquals(path3, abfs.getWorkingDirectory());
}
+
+ // Scenario wise testing
+
+ //Scenario 1: - Create and write via WASB, read via ABFS
+ @Test
+ public void testScenario1() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+ // Check file status
+ ContractTestUtils.assertIsFile(wasb, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 2: - Create and write via WASB, read via ABFS and then write the
same file via ABFS
+ @Test
+ public void testScenario2() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+ // Check file status
+ ContractTestUtils.assertIsFile(wasb, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Write
+ try (FSDataOutputStream abfsOutputStream = abfs.append(path)) {
+ abfsOutputStream.write(TEST_CONTEXT1.getBytes());
+ abfsOutputStream.flush();
+ abfsOutputStream.hsync();
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 3: - Create and write via ABFS and the read via WASB
+ @Test
+ public void testScenario3() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(wasb.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + wasb,
+ TEST_CONTEXT, line);
+ }
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 4:- Create via WASB, write via ABFS and then write via WASB
+ @Test
+ public void testScenario4() throws Exception {
+ AzureBlobFileSystem abfs = getFileSystem();
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ wasb.create(path, true);
+ try (FSDataOutputStream abfsOutputStream = abfs.append(path)) {
+ abfsOutputStream.write(TEST_CONTEXT.getBytes());
+ abfsOutputStream.flush();
+ abfsOutputStream.hsync();
+ }
+
+ try (FSDataOutputStream nativeFsStream = wasb.append(path)) {
+ nativeFsStream.write(TEST_CONTEXT1.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 5:- Create via ABFS, write via WASB, read via ABFS (Checksum
validation disabled)
+ @Test
+ public void testScenario5() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, false);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ abfs.create(path, true);
+ try (FSDataOutputStream nativeFsStream = wasb.append(path)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ //Scenario 6: - Create via ABFS, write via WASB, read via ABFS (Checksum
validation enabled)
+ @Test
+ public void testScenario6() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ assumeBlobServiceType();
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ abfs.create(path, true);
+ try (FSDataOutputStream nativeFsStream = wasb.append(path)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ // Scenario 7 :- Create via WASB and then create overwrite true using ABFS
+ @Test
+ public void testScenario7() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+ abfs.create(path, true);
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ // Scenario 8 :- Create via WASB and then create overwrite false using ABFS
+ @Test
+ public void testScenario8() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+ try {
+ abfs.create(path, false);
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("AlreadyExists"));
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ // Scenario 9 :- Create via ABFS and then create overwrite true using WASB
+ @Test
+ public void testScenario9() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+ wasb.create(path, true);
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ // Scenario 10 :- Create via ABFS and then create overwrite false using WASB
+ @Test
+ public void testScenario10() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+ try {
+ wasb.create(path, false);
+ } catch (Exception e) {
+ assertTrue(e.getMessage().toLowerCase().contains("exists"));
+ }
+
+ // Remove file
+ assertDeleted(abfs, path, true);
+ }
+
+ // Scenario 11 :- Create via ABFS and then write via WASB and delete via ABFS
+ @Test
+ public void testScenario11() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ abfs.create(path, true);
+ try (FSDataOutputStream nativeFsStream = wasb.append(path)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ }
+
+ // Check file status
+ ContractTestUtils.assertIsFile(abfs, path);
+
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(abfs.open(path)))) {
+ String line = br.readLine();
+ assertEquals("Wrong text from " + abfs,
+ TEST_CONTEXT, line);
+ }
+ abfs.delete(path, true);
+ }
+
+ // Scenario 12 :- Create and write via ABFS and delete via WASB
+ @Test
+ public void testScenario12() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeFalse("Namespace enabled account does not support this test",
+ getIsNamespaceEnabled(abfs));
+ NativeAzureFileSystem wasb = getWasbFileSystem();
+
+ Path testFile = path("/testReadFile");
+ Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID());
+
+ // Write
+ try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) {
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
Review Comment:
The outputstream name is confusing corrected it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
