meethngala commented on code in PR #3560:
URL: https://github.com/apache/gobblin/pull/3560#discussion_r972531688
##########
gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetTest.java:
##########
@@ -77,80 +74,112 @@ public void testGetFilePaths() throws IOException {
}
/**
- * Test case to copy all the file paths for a mocked iceberg table. This is
a full copy overwriting everything on the destination
+ * Test case to generate copy entities for all the file paths for a mocked
iceberg table.
+ * The assumption here is that we create copy entities for all the matching
file paths,
+ * without calculating any difference between the source and destination
*/
@Test
public void testGenerateCopyEntitiesForTableFileSet() throws IOException,
URISyntaxException {
FileSystem fs = Mockito.mock(FileSystem.class);
String test_db_name = "test_db_name";
String test_table_name = "test_tbl_name";
- Set<String> setOfFilePaths = new HashSet<>(Arrays.asList(METADATA_PATH,
MANIFEST_PATH, MANIFEST_FILE_PATH1, MANIFEST_FILE_PATH2));
-
+ String test_qualified_path =
"/root/iceberg/test/destination/sub_path_destination";
+ String test_uri_path = "/root/iceberg/test/output";
Properties properties = new Properties();
properties.setProperty("data.publisher.final.dir", "/test");
+ List<String> expected = new ArrayList<>(Arrays.asList(METADATA_PATH,
MANIFEST_PATH, MANIFEST_LIST_PATH, MANIFEST_FILE_PATH1, MANIFEST_FILE_PATH2));
CopyConfiguration copyConfiguration = CopyConfiguration.builder(null,
properties)
.preserve(PreserveAttributes.fromMnemonicString(""))
.copyContext(new CopyContext())
.build();
- TableOperations tableOperations = Mockito.mock(TableOperations.class);
- IcebergTable icebergTable = new MockedIcebergTable(tableOperations);
+ IcebergTable icebergTable = new MockedIcebergTable(METADATA_PATH,
MANIFEST_PATH, MANIFEST_LIST_PATH, new
ArrayList<>(Arrays.asList(MANIFEST_FILE_PATH1, MANIFEST_FILE_PATH2)));
IcebergDataset icebergDataset = new IcebergDataset(test_db_name,
test_table_name, icebergTable, new Properties(), fs);
+ Path path1 = new Path(METADATA_PATH);
+ Path path2 = new Path(MANIFEST_PATH);
+ Path path3 = new Path(MANIFEST_LIST_PATH);
+ Path path4 = new Path(MANIFEST_FILE_PATH1);
+ Path path5 = new Path(MANIFEST_FILE_PATH2);
+
FileStatus fileStatus1 = new FileStatus();
- fileStatus1.setPath(new Path(METADATA_PATH));
+ fileStatus1.setPath(path1);
FileStatus fileStatus2 = new FileStatus();
- fileStatus2.setPath(new Path(MANIFEST_PATH));
+ fileStatus2.setPath(path2);
FileStatus fileStatus3 = new FileStatus();
- fileStatus3.setPath(new Path(MANIFEST_FILE_PATH1));
+ fileStatus3.setPath(path3);
FileStatus fileStatus4 = new FileStatus();
- fileStatus4.setPath(new Path(MANIFEST_FILE_PATH2));
-
- Path path1 = new Path(METADATA_PATH);
- Path path2 = new Path(MANIFEST_PATH);
- Path path3 = new Path(MANIFEST_FILE_PATH1);
- Path path4 = new Path(MANIFEST_FILE_PATH2);
+ fileStatus4.setPath(path4);
+ FileStatus fileStatus5 = new FileStatus();
+ fileStatus5.setPath(path5);
+ Map<Path, FileStatus> mapOfPathAndFileStatus = Maps.newHashMap();
+ mapOfPathAndFileStatus.put(path1, fileStatus1);
+ mapOfPathAndFileStatus.put(path2, fileStatus2);
+ mapOfPathAndFileStatus.put(path3, fileStatus3);
+ mapOfPathAndFileStatus.put(path4, fileStatus4);
+ mapOfPathAndFileStatus.put(path5, fileStatus5);
Review Comment:
that's a good idea... pushed the implementation as part of the latest commit
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
