nsivabalan commented on code in PR #19045:
URL: https://github.com/apache/hudi/pull/19045#discussion_r3611504288
##########
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java:
##########
@@ -1518,27 +1518,52 @@ private static List<FileSlice>
getPartitionFileSlices(HoodieTableMetaClient meta
HoodieTableFileSystemView fsView = null;
try {
fsView = fileSystemView.orElseGet(() ->
getFileSystemViewForMetadataTable(metaClient));
- Stream<FileSlice> fileSliceStream;
- if (mergeFileSlices) {
- if
(metaClient.getActiveTimeline().filterCompletedInstants().lastInstant().isPresent())
{
- fileSliceStream = fsView.getLatestMergedFileSlicesBeforeOrOn(
- // including pending compaction instant as the last instant so
that the finished delta commits
- // that start earlier than the compaction can be queried.
- partition,
metaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant().get().requestedTime());
+ List<String> physicalPartitions = resolvePhysicalPartitions(metaClient,
partition);
+ List<FileSlice> all = new ArrayList<>();
+ boolean any = false;
+ for (String physical : physicalPartitions) {
+ Stream<FileSlice> fileSliceStream;
+ if (mergeFileSlices) {
+ if
(metaClient.getActiveTimeline().filterCompletedInstants().lastInstant().isPresent())
{
+ fileSliceStream = fsView.getLatestMergedFileSlicesBeforeOrOn(
+ // including pending compaction instant as the last instant so
that the finished delta commits
+ // that start earlier than the compaction can be queried.
+ physical,
metaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant().get().requestedTime());
+ } else {
+ return Collections.emptyList();
+ }
} else {
- return Collections.emptyList();
+ fileSliceStream = fsView.getLatestFileSlices(physical);
}
- } else {
- fileSliceStream = fsView.getLatestFileSlices(partition);
+ fileSliceStream.forEach(all::add);
+ any = true;
+ }
+ if (!any) {
+ return Collections.emptyList();
}
- return
fileSliceStream.sorted(Comparator.comparing(FileSlice::getFileId)).collect(Collectors.toList());
+ all.sort(Comparator.comparing(FileSlice::getFileId));
+ return all;
} finally {
- if (!fileSystemView.isPresent()) {
+ if (!fileSystemView.isPresent() && fsView != null) {
fsView.close();
}
}
}
+ /**
+ * Resolve the physical sub-paths to scan for the given MDT partition under
the configured
+ * layout. Layout-aware: for the flat layout this returns {@code
[partition]}; for sub-directory
+ * bucketing it returns one entry per bucket directory. {@code
fileGroupCount} is sourced from
+ * the MDT's persisted layout state — this method does not perform any
filesystem listing.
+ */
+ private static List<String> resolvePhysicalPartitions(HoodieTableMetaClient
metaClient, String partition) {
Review Comment:
Agree — grouping this with the "move heuristic to common place" thread above
(already addressed in `8a7c60f5` via
`HoodieTableMetadataUtil.isMDTBucketSubPath`). If there’s another specific
helper you had in mind here that isn’t `isMDTBucketSubPath`, let me know which
one and I’ll route it through the same class.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
