This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new aff607b1c975 fix: fix get empty completion time in corner case (#14379)
aff607b1c975 is described below
commit aff607b1c9756c10715afa98c8fa0c599225924a
Author: TheR1sing3un <[email protected]>
AuthorDate: Thu Nov 27 18:10:51 2025 +0800
fix: fix get empty completion time in corner case (#14379)
the corner case: the load instant range is contained within one of the
archived file instant range.
---------
Signed-off-by: TheR1sing3un <[email protected]>
Co-authored-by: danny0405 <[email protected]>
---
.../timeline/TestCompletionTimeQueryView.java | 47 ++++++++++++++++++++--
.../table/timeline/HoodieArchivedTimeline.java | 13 ++++++
.../hudi/common/table/timeline/LSMTimeline.java | 2 +-
3 files changed, 57 insertions(+), 5 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
index 5e5434644c16..8b77c1459fc2 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
@@ -53,6 +53,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import static
org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_FILE_NAME_GENERATOR;
@@ -105,6 +106,27 @@ public class TestCompletionTimeQueryView {
}
}
+ @Test
+ void testReadCompletionTimeWithCornerCase() throws Exception {
+ String tableName = "testTable";
+ String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR +
tableName;
+ HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+ HoodieTestUtils.getDefaultStorageConf(), tablePath,
HoodieTableType.COPY_ON_WRITE, tableName);
+ prepareTimeline(tablePath, metaClient, (lsmTimelineWriter, activeActions)
-> {
+ // archive [1, 2], [3, 6] specifically to create corner cases
+ lsmTimelineWriter.write(activeActions.subList(0, 2), Option.empty(),
Option.empty());
+ lsmTimelineWriter.write(activeActions.subList(2, 6), Option.empty(),
Option.empty());
+ });
+ try (CompletionTimeQueryView view =
+
metaClient.getTableFormat().getTimelineFactory().createCompletionTimeQueryView(metaClient))
{
+ // 1. first time, we try to load archived instant from 5, so the cursor
will move forward from 7 to 5
+ assertThat(view.getCompletionTime(String.format("%08d", 5)).orElse(""),
is(String.format("%08d", 1005)));
+
+ // 2. then we try to load archived instant from 4, it should get the
completion time correctly
+ assertThat(view.getCompletionTime(String.format("%08d", 4)).orElse(""),
is(String.format("%08d", 1004)));
+ }
+ }
+
@Test
void testReadStartTime() throws Exception {
String tableName = "testTable";
@@ -180,6 +202,16 @@ public class TestCompletionTimeQueryView {
}
private void prepareTimeline(String tablePath, HoodieTableMetaClient
metaClient, List<Pair<String, String>> instantRequestedAndCompletionTime,
String requestedInstantTime) throws Exception {
+ prepareTimeline(tablePath, metaClient, instantRequestedAndCompletionTime,
requestedInstantTime, (writer, activeActions) -> {
+ // archive [1,2], [3,4], [5,6] separately
+ writer.write(activeActions.subList(0, 2), Option.empty(),
Option.empty());
+ writer.write(activeActions.subList(2, 4), Option.empty(),
Option.empty());
+ writer.write(activeActions.subList(4, 6), Option.empty(),
Option.empty());
+ });
+ }
+
+ private void prepareTimeline(String tablePath, HoodieTableMetaClient
metaClient, List<Pair<String, String>> instantRequestedAndCompletionTime,
String requestedInstantTime,
+ BiConsumer<LSMTimelineWriter,
List<ActiveAction>> timelineWriterListBiConsumer) throws Exception {
HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().withPath(tablePath)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
.withMarkersType("DIRECT")
@@ -199,10 +231,7 @@ public class TestCompletionTimeQueryView {
testTable.addRequestedCommit(requestedInstantTime);
List<HoodieInstant> instants =
TIMELINE_FACTORY.createActiveTimeline(metaClient,
false).getInstantsAsStream().sorted().collect(Collectors.toList());
LSMTimelineWriter writer = LSMTimelineWriter.getInstance(writeConfig,
getMockHoodieTable(metaClient));
- // archive [1,2], [3,4], [5,6] separately
- writer.write(activeActions.subList(0, 2), Option.empty(), Option.empty());
- writer.write(activeActions.subList(2, 4), Option.empty(), Option.empty());
- writer.write(activeActions.subList(4, 6), Option.empty(), Option.empty());
+ timelineWriterListBiConsumer.accept(writer, activeActions);
// reconcile the active timeline
instants.subList(0, 3 * 6).forEach(
instant -> TimelineUtils.deleteInstantFile(metaClient.getStorage(),
@@ -222,6 +251,16 @@ public class TestCompletionTimeQueryView {
prepareTimeline(tablePath, metaClient, instantRequestedAndCompletionTime,
String.format("%08d", 11));
}
+ private void prepareTimeline(String tablePath, HoodieTableMetaClient
metaClient, BiConsumer<LSMTimelineWriter, List<ActiveAction>>
timelineWriterListBiConsumer) throws Exception {
+ List<Pair<String, String>> instantRequestedAndCompletionTime = new
ArrayList<>();
+ for (int i = 1; i < 11; i++) {
+ String instantTime = String.format("%08d", i);
+ String completionTime = String.format("%08d", i + 1000);
+ instantRequestedAndCompletionTime.add(Pair.of(instantTime,
completionTime));
+ }
+ prepareTimeline(tablePath, metaClient, instantRequestedAndCompletionTime,
String.format("%08d", 11), timelineWriterListBiConsumer);
+ }
+
@SuppressWarnings("rawtypes")
private HoodieTable getMockHoodieTable(HoodieTableMetaClient metaClient) {
HoodieTable hoodieTable = mock(HoodieTable.class);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
index 5c79864fe5ae..6393f417b22e 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
@@ -82,6 +82,19 @@ public interface HoodieArchivedTimeline extends
HoodieTimeline {
public boolean isInRange(String instantTime) {
return InstantComparison.isInRange(instantTime, this.startTs,
this.endTs);
}
+
+ /**
+ * Returns whether the given instant time range has overlapping with the
current range.
+ */
+ public boolean hasOverlappingInRange(String startInstant, String
endInstant) {
+ return isInRange(startInstant) || isInRange(endInstant) ||
isContained(startInstant, endInstant);
+ }
+
+ private boolean isContained(String startInstant, String endInstant) {
+ // the given range is finite and expected to be covered by the current
range which naturally cannot be infinite.
+ return startTs != null && InstantComparison.compareTimestamps(startTs,
InstantComparison.GREATER_THAN_OR_EQUALS, startInstant)
+ && endTs != null && InstantComparison.compareTimestamps(endTs,
InstantComparison.LESSER_THAN_OR_EQUALS, endInstant);
+ }
}
/**
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java
index 2c06bdcd010d..c99dcb67f0bd 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java
@@ -142,7 +142,7 @@ public class LSMTimeline {
public static boolean isFileInRange(HoodieArchivedTimeline.TimeRangeFilter
filter, String fileName) {
String minInstant = getMinInstantTime(fileName);
String maxInstant = getMaxInstantTime(fileName);
- return filter.isInRange(minInstant) || filter.isInRange(maxInstant);
+ return filter.hasOverlappingInRange(minInstant, maxInstant);
}
/**
