anmolanmol1234 commented on code in PR #7777: URL: https://github.com/apache/hadoop/pull/7777#discussion_r2212635589
########## hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java: ########## @@ -201,4 +325,1650 @@ public void testSetWorkingDirectory() throws Exception { assertEquals(path3, wasb.getWorkingDirectory()); assertEquals(path3, abfs.getWorkingDirectory()); } + + // Scenario wise testing + + //Scenario 1: - Create and write via WASB, read via ABFS + @Test + public void testScenario1() throws Exception { + AzureBlobFileSystem abfs = getFileSystem(); + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // Check file status + ContractTestUtils.assertIsFile(wasb, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + + // Remove file + assertDeleted(abfs, path, true); + } + + //Scenario 2: - Create and write via WASB, read via ABFS and then write the same file via ABFS + @Test + public void testScenario2() throws Exception { + AzureBlobFileSystem abfs = getFileSystem(); + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // Check file status + ContractTestUtils.assertIsFile(wasb, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + + // Write + try (FSDataOutputStream abfsOutputStream = abfs.append(path)) { + abfsOutputStream.write(TEST_CONTEXT1.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + + // Remove file + assertDeleted(abfs, path, true); + } + + //Scenario 3: - Create and write via ABFS and the read via WASB + @Test + public void testScenario3() throws Exception { + AzureBlobFileSystem abfs = getFileSystem(); + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + // Remove file + assertDeleted(abfs, path, true); + } + + //Scenario 4:- Create via WASB, write via ABFS and then write via WASB + @Test + public void testScenario4() throws Exception { + AzureBlobFileSystem abfs = getFileSystem(); + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + wasb.create(path, true); + try (FSDataOutputStream abfsOutputStream = abfs.append(path)) { + abfsOutputStream.write(TEST_CONTEXT.getBytes()); + abfsOutputStream.flush(); + abfsOutputStream.hsync(); + } + + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT1.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + // Remove file + assertDeleted(abfs, path, true); + } + + //Scenario 5:- Create via ABFS, write via WASB, read via ABFS (Checksum validation disabled) + @Test + public void testScenario5() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, false); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + + // Remove file + assertDeleted(abfs, path, true); + } + + //Scenario 6: - Create via ABFS, write via WASB, read via ABFS (Checksum validation enabled) + @Test + public void testScenario6() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + assumeBlobServiceType(); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + + // Remove file + assertDeleted(abfs, path, true); + } + + // Scenario 7 :- Create via WASB and then create overwrite true using ABFS + @Test + public void testScenario7() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + abfs.create(path, true); + + // Remove file + assertDeleted(abfs, path, true); + } + + // Scenario 8 :- Create via WASB and then create overwrite false using ABFS + @Test + public void testScenario8() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + try { + abfs.create(path, false); + } catch (Exception e) { + assertTrue(e.getMessage().contains("AlreadyExists")); + } + + // Remove file + assertDeleted(abfs, path, true); + } + + // Scenario 9 :- Create via ABFS and then create overwrite true using WASB + @Test + public void testScenario9() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.create(path, true); + + // Remove file + assertDeleted(abfs, path, true); + } + + // Scenario 10 :- Create via ABFS and then create overwrite false using WASB + @Test + public void testScenario10() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + try { + wasb.create(path, false); + } catch (Exception e) { + assertTrue(e.getMessage().toLowerCase().contains("exists")); + } + + // Remove file + assertDeleted(abfs, path, true); + } + + // Scenario 11 :- Create via ABFS and then write via WASB and delete via ABFS + @Test + public void testScenario11() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + abfs.delete(path, true); + } + + // Scenario 12 :- Create and write via ABFS and delete via WASB + @Test + public void testScenario12() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.delete(path, true); + } + + // Scenario 13:- Create via ABFS, write via WASB, and read via wasb + @Test + public void testScenario13() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + abfs.delete(path, true); + } + + // Scenario 14:- Create via ABFS, write via WASB, and delete via wasb + @Test + public void testScenario14() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.create(path, true); + try (FSDataOutputStream nativeFsStream = wasb.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + wasb.delete(path, true); + } + + // Scenario 15 :- Create and write via WASB and delete via ABFS + @Test + public void testScenario15() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(wasb.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + wasb, + TEST_CONTEXT, line); + } + abfs.delete(path, true); + } + + // Scenario 16: Create via WASB, write via ABFS, and delete via WASB + @Test + public void testScenario16() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + wasb.create(path, true); + try (FSDataOutputStream nativeFsStream = abfs.append(path)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, path); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(path)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.delete(path, true); + } + + // Scenario 17: Create, setXAttr and getXAttr via ABFS + @Test + public void testScenario17() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = abfs.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + abfs.delete(path, true); + } + + // Scenario 17: Create, setXAttr and getXAttr via WASB + @Test + public void testScenario18() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.delete(path, true); + } + + // Scenario 19: Create, setXAttr via wasb and getXAttr via ABFS + @Test + public void testScenario19() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.delete(path, true); + } + + // Scenario 20: Create, setXAttr via wasb and getXAttr via ABFS and create overwrite via ABFS + @Test + public void testScenario20() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + abfs.create(path, true); + wasb.delete(path, true); + } + + // Scenario 21: Create, setXAttr ABFS, getXAttr WASB and create overwrite via WASB + @Test + public void testScenario21() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.create(path, true); + wasb.delete(path, true); + } + + // Scenario 22: Create via WASB, setXAttr ABFS, getXAttr wasb and create overwrite via WASB + @Test + public void testScenario22() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.create(path, true); + wasb.delete(path, true); + } + + // Scenario 23: Create via WASB, setXAttr ABFS, then setXAttr via WASB and getXAttr via ABFS + @Test + public void testScenario23() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.delete(path, true); + } + + // Scenario 24: Create via ABFS, then setXAttr via WASB and getXAttr via ABFS + @Test + public void testScenario24() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.delete(path, true); + } + + // Scenario 24: Create via WASB, then setXAttr getXAttr via ABFS and delete via WASB + @Test + public void testScenario25() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + abfs.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = abfs.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.delete(path, true); + } + + // Scenario 26: Create via ABFS, then setXAttr getXAttr via WASB and delete via WASB + @Test + public void testScenario26() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = abfs.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = abfs.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2, CREATE_FLAG); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.delete(path, true); + } + + // Scenario 27: Create and write via ABFS, rename via wasb + @Test + public void testScenario27() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = abfs.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(wasb, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath2, true); + } + + // Scenario 28: Create and write via WASB, rename via ABFS, list via ABFS + @Test + public void testScenario28() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(abfs, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath2, true); + } + + // Scenario 29: Create via WASB and write via ABFS, rename via ABFS, list via ABFS + @Test + public void testScenario29() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + assumeBlobServiceType(); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + wasb.create(testPath1, true); + try (FSDataOutputStream nativeFsStream = abfs.append(testPath1)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(abfs, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath2, true); + } + + //Scenario 30: Create and write via WASB, rename via WASB, rename via ABFS, list via ABFS + @Test + public void testScenario30() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME FILE --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + + // --- RENAME FILE --- + boolean renamed1 = abfs.rename(testPath2, testPath3); + Assertions.assertThat(renamed1) + .as("Rename failed") + .isTrue(); + + // --- LIST FILES IN DIRECTORY --- + Path parentDir = new Path(testFile + "/~12/!008"); + int noOfFiles = listAllFilesAndDirs(abfs, parentDir); + Assertions.assertThat(noOfFiles) + .as("Expected only 1 file or directory under path: %s", parentDir) + .isEqualTo(1); + wasb.delete(testPath3, true); + } + + //Scenario 31: Create and write via WASB, delete via WASB, rename via ABFS -> should fail + @Test + public void testScenario31() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + wasb.delete(testPath1, true); + + // --- RENAME FILE --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename operation should have failed but returned true") + .isFalse(); + } + + //Scenario 32 :Create Dir & File via WASB → Rename Dir via ABFS → List Files via ABFS + @Test + public void testScenario32() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testFile1 = path("/testReadFile1"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath2, true); + wasb.create(testPath3, true); + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = abfs.rename(testFile, testFile1); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(abfs, testFile1); + Assertions.assertThat(listResult) + .as("Expected only 5 entries under path: %s", testFile1) + .isEqualTo(5); + } + + //Scenario 33 :Create Dir & File via ABFS → Rename Dir via WASB → List Files via WASB + @Test + public void testScenario33() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testFile1 = path("/testReadFile1"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = abfs.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + abfs.create(testPath2, true); + abfs.create(testPath3, true); + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = wasb.rename(testFile, testFile1); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(wasb, testFile1); + Assertions.assertThat(listResult) + .as("Expected only 5 entries under path: %s", testFile1) + .isEqualTo(5); + } + + //Scenario 34: Create Dir via ABFS → Rename File inside Dir via WASB → List via ABFS + @Test + public void testScenario34() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = abfs.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + abfs.create(testPath3, true); + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(abfs, testFile); + Assertions.assertThat(listResult) + .as("Expected only 4 entries under path: %s", testFile) + .isEqualTo(4); + } + + //Scenario 35: Create Dir via WASB → Rename File inside Dir via ABFS → List via WASB + @Test + public void testScenario35() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath3, true); + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = abfs.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(wasb, testFile); + Assertions.assertThat(listResult) + .as("Expected only 4 entries under path: %s", testFile) + .isEqualTo(4); + } + + //Scenario 36: Create via WASB → Rename to existing name via ABFS → List via WASB + @Test + public void testScenario36() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath3, true); + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = abfs.rename(testFile, testFile); + Assertions.assertThat(renamed) + .as("Rename operation should have failed but returned true") + .isFalse(); + } + + //Scenario 37: Rename a non-existent file via WASB + @Test + public void testScenario37() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + abfs.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = abfs.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + abfs.create(testPath3, true); + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME NON EXISTENT FILE --- + boolean renamed = wasb.rename(testPath2, testPath3); + Assertions.assertThat(renamed) + .as("Rename operation should have failed but returned true") + .isFalse(); + } + + // Scenario 38: Create via WASB, setXAttr and getXAttr WASB and create overwrite via WASB + @Test + public void testScenario38() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + Path testFile = path("/testReadFile"); + Path path = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + try (FSDataOutputStream nativeFsStream = wasb.create(path, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + // --- VALIDATE FILE --- + FileStatus status = wasb.getFileStatus(path); + assertIsFile(path, status); + + // --- SET XATTR #1 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_1, ATTRIBUTE_VALUE_1); + byte[] readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + // --- SET XATTR #2 --- + wasb.setXAttr(path, ATTRIBUTE_NAME_2, ATTRIBUTE_VALUE_2); + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_2); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_2, "two"); + + // --- VERIFY XATTR #1 AGAIN --- + readValue = wasb.getXAttr(path, ATTRIBUTE_NAME_1); + assertAttributeEqual(readValue, ATTRIBUTE_VALUE_1, "one"); + + wasb.create(path, true); + wasb.delete(path, true); + } + + // Scenario 39: Create and write via WASB, rename via wasb and list via WASB + @Test + public void testScenario39() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true); + FileSystem fileSystem = FileSystem.newInstance(conf); + AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem; + Assume.assumeFalse("Namespace enabled account does not support this test", + getIsNamespaceEnabled(abfs)); + NativeAzureFileSystem wasb = getWasbFileSystem(); + + String testRunId = UUID.randomUUID().toString(); + Path baseDir = path("/testScenario39_" + testRunId); + Path testFile = new Path(baseDir, "testReadFile"); + Path testPath1 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath2 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + Path testPath3 = new Path(testFile + "/~12/!008/testfile_" + UUID.randomUUID()); + + // Write + wasb.mkdirs(testFile); + try (FSDataOutputStream nativeFsStream = wasb.create(testPath1, true)) { + nativeFsStream.write(TEST_CONTEXT.getBytes()); + nativeFsStream.flush(); + nativeFsStream.hsync(); + } + wasb.create(testPath3, true); + + // Check file status + ContractTestUtils.assertIsFile(abfs, testPath1); + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(abfs.open(testPath1)))) { + String line = br.readLine(); + assertEquals("Wrong text from " + abfs, + TEST_CONTEXT, line); + } + // --- RENAME DIR --- + boolean renamed = wasb.rename(testPath1, testPath2); + Assertions.assertThat(renamed) + .as("Rename failed") + .isTrue(); + // --- LIST FILES IN DIRECTORY --- + int listResult = listAllFilesAndDirs(wasb, testFile); + Assertions.assertThat(listResult) + .as("Expected only 4 entries under path: %s", testFile) + .isEqualTo(4); + } + + /** + * Recursively counts all files and directories under the given path. + * + * @param fs The file system to use. + * @param path The starting path. + * @return Total number of files and directories. + * @throws IOException If an error occurs while accessing the file system. + */ + public static int listAllFilesAndDirs(FileSystem fs, Path path) throws IOException { Review Comment: taken -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org