yihua commented on code in PR #13254:
URL: https://github.com/apache/hudi/pull/13254#discussion_r2076390571
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -665,38 +670,37 @@ private Pair<Integer, HoodieData<HoodieRecord>>
initializeSecondaryIndexPartitio
return Pair.of(fileGroupCount, records);
}
- private List<Pair<String, FileSlice>> getPartitionFileSlicePairs() throws
IOException {
- String latestInstant =
dataMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()
- .map(HoodieInstant::requestedTime).orElse(SOLO_COMMIT_TIMESTAMP);
- try (HoodieTableFileSystemView fsView = getMetadataView()) {
- // Collect the list of latest file slices present in each partition
- List<String> partitions = metadata.getAllPartitionPaths();
- fsView.loadAllPartitions();
- List<Pair<String, FileSlice>> partitionFileSlicePairs = new
ArrayList<>();
- partitions.forEach(partition ->
fsView.getLatestMergedFileSlicesBeforeOrOn(partition, latestInstant)
- .forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition, fs))));
- return partitionFileSlicePairs;
- }
+ private Lazy<List<Pair<String, FileSlice>>>
getLazyLatestMergedPartitionFileSliceList() {
+ return Lazy.lazily(() -> {
+ String latestInstant =
dataMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant()
+ .map(HoodieInstant::requestedTime).orElse(SOLO_COMMIT_TIMESTAMP);
+ try (HoodieTableFileSystemView fsView = getMetadataView()) {
+ // Collect the list of latest file slices present in each partition
+ List<String> partitions = metadata.getAllPartitionPaths();
+ fsView.loadAllPartitions();
+ List<Pair<String, FileSlice>> partitionFileSlicePairs = new
ArrayList<>();
+ partitions.forEach(partition ->
fsView.getLatestMergedFileSlicesBeforeOrOn(partition, latestInstant)
+ .forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition,
fs))));
+ return partitionFileSlicePairs;
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ });
}
- private Pair<Integer, HoodieData<HoodieRecord>>
initializeRecordIndexPartition() throws IOException {
- final HoodieTableFileSystemView fsView = getMetadataView();
+ private Pair<Integer, HoodieData<HoodieRecord>>
initializeRecordIndexPartition(
+ Lazy<List<Pair<String, FileSlice>>>
lazyLatestMergedPartitionFileSliceList) {
Review Comment:
`metadata` thus `lazyLatestMergedPartitionFileSliceList` does not change in
`initializeFromFilesystem`. Conceptually and logically that makes sense; so
that's not needed. Eventually we should get rid of the `getMetadataView` and
make it final, because it causes confusion if it's mutable but not always
changed in different flows in the MDT writer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
