[
https://issues.apache.org/jira/browse/GOBBLIN-2159?focusedWorklogId=939673&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-939673
]
ASF GitHub Bot logged work on GOBBLIN-2159:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 23/Oct/24 10:39
Start Date: 23/Oct/24 10:39
Worklog Time Spent: 10m
Work Description: Blazer-007 commented on code in PR #4058:
URL: https://github.com/apache/gobblin/pull/4058#discussion_r1812461753
##########
gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/iceberg/IcebergPartitionDatasetTest.java:
##########
@@ -225,39 +199,57 @@ public void
testWithDifferentSrcAndDestTableWriteLocation() throws IOException {
List<CopyEntity> copyEntities =
(List<CopyEntity>)
icebergPartitionDataset.generateCopyEntities(targetFs, copyConfiguration);
- Assert.assertEquals(copyEntities.size(), 2);
- verifyCopyEntities(copyEntities, false);
+ verifyCopyEntities(copyEntities, 2, false);
}
- private List<DataFile> getDataFiles() throws IOException {
+ private static void setupSrcFileSystem() throws IOException {
+ sourceFs = Mockito.mock(FileSystem.class);
+ Mockito.when(sourceFs.getUri()).thenReturn(SRC_FS_URI);
+ Mockito.when(sourceFs.makeQualified(any(Path.class)))
+ .thenAnswer(invocation -> invocation.getArgument(0,
Path.class).makeQualified(SRC_FS_URI, new Path("/")));
+
Mockito.when(sourceFs.getFileStatus(any(Path.class))).thenAnswer(invocation -> {
+ Path path = invocation.getArgument(0, Path.class);
+ Path qualifiedPath = sourceFs.makeQualified(path);
+ return
IcebergDatasetTest.MockFileSystemBuilder.createEmptyFileStatus(qualifiedPath.toString());
+ });
+ }
+
+ private static void setupDestFileSystem() throws IOException {
+ targetFs = Mockito.mock(FileSystem.class);
+ Mockito.when(targetFs.getUri()).thenReturn(DEST_FS_URI);
+ Mockito.when(targetFs.makeQualified(any(Path.class)))
+ .thenAnswer(invocation -> invocation.getArgument(0,
Path.class).makeQualified(DEST_FS_URI, new Path("/")));
+ // Since we are adding UUID to the file name for every file while creating
destination path,
+ // so return file not found exception if trying to find file status on
destination file system
+ Mockito.when(targetFs.getFileStatus(any(Path.class))).thenThrow(new
FileNotFoundException());
+ }
+
+ private static List<DataFile> createDataFileMocks() throws IOException {
List<DataFile> dataFiles = new ArrayList<>();
for (String srcFilePath : srcFilePaths) {
DataFile dataFile = Mockito.mock(DataFile.class);
Path dataFilePath = new Path(srcFilePath);
Path qualifiedPath = sourceFs.makeQualified(dataFilePath);
Mockito.when(dataFile.path()).thenReturn(dataFilePath.toString());
-
Mockito.when(sourceFs.getFileStatus(Mockito.eq(dataFilePath))).thenReturn(getFileStatus(qualifiedPath));
+
Mockito.when(sourceFs.getFileStatus(Mockito.eq(dataFilePath))).thenReturn(
+
IcebergDatasetTest.MockFileSystemBuilder.createEmptyFileStatus(qualifiedPath.toString()));
dataFiles.add(dataFile);
}
return dataFiles;
}
- private static FileStatus getFileStatus(Path path) {
- FileStatus fileStatus = new FileStatus();
- fileStatus.setPath(path);
- return fileStatus;
- }
-
- private static void verifyCopyEntities(Collection<CopyEntity> copyEntities,
boolean sameSrcAndDestWriteLocation) {
+ private static void verifyCopyEntities(Collection<CopyEntity> copyEntities,
int expectedCopyEntitiesSize,
Review Comment:
Done
Issue Time Tracking
-------------------
Worklog Id: (was: 939673)
Time Spent: 11h 10m (was: 11h)
> Support Partition Based Copy in Iceberg Distcp
> ----------------------------------------------
>
> Key: GOBBLIN-2159
> URL: https://issues.apache.org/jira/browse/GOBBLIN-2159
> Project: Apache Gobblin
> Issue Type: Task
> Reporter: Vivek Rai
> Priority: Major
> Time Spent: 11h 10m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
