skywalker0618 commented on code in PR #20735:
URL: https://github.com/apache/hudi/pull/20735#discussion_r4647961959


##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/split/DefaultHoodieSplitDiscover.java:
##########
@@ -65,6 +86,141 @@ public HoodieContinuousSplitBatch discoverSplits(String 
lastInstant) {
     return incrementalInputSplits.inputHoodieSourceSplits(metaClient, 
lastInstant, scanContext.isCdcEnabled());
   }
 
+  @Override
+  public List<String> getUndiscoveredCommits(@Nullable String offset, boolean 
reloadTimeline) {
+    if (metaClient == null || offset == null) {
+      return Collections.emptyList();
+    }
+    if (reloadTimeline) {
+      metaClient.reloadActiveTimeline();
+    }
+    HoodieTimeline timeline = 
IncrementalQueryAnalyzer.filterInstantsAsPerUserConfigs(
+        metaClient,
+        metaClient.getCommitsAndCompactionTimeline().filterCompletedInstants(),
+        scanContext.isSkipCompaction(),
+        scanContext.isSkipClustering(),
+        scanContext.isSkipInsertOverwrite());
+    return timeline.getInstantsAsStream()
+        .map(HoodieInstant::getCompletionTime)
+        .filter(completionTime -> completionTime != null && 
completionTime.compareTo(offset) > 0)
+        .sorted()
+        .collect(Collectors.toList());
+  }
+
+  /**
+   * Returns the lag info of the splits from the write stats of the commits in 
their exact-match ranges.
+   * A copy-on-write or append-only split only emits the records of the 
commits that wrote its file
+   * group, so those are its read commits. Its records are their inserts and 
updates, which counts a
+   * record written by several of them more than once, and its bytes are the 
size of the latest file
+   * written. Splits without an exact-match range read their latest commit, 
with unknown estimates.
+   */
+  @Override
+  public Map<String, HoodieSourceSplitLagInfo> 
getLagInfo(Collection<HoodieSourceSplit> splits) {
+    if (metaClient == null || splits.isEmpty()) {
+      return HoodieContinuousSplitDiscover.super.getLagInfo(splits);
+    }
+    Map<String, Pair<HoodieInstant, HoodieTimeline>> commits = 
getCommits(splits);
+    Map<String, Map<String, long[]>> writesByCommit = new HashMap<>();
+    for (String commit : getExactMatchCommits(splits)) {
+      if (commits.containsKey(commit)) {
+        writesByCommit.put(commit, 
getWritesByFileId(commits.get(commit).getLeft(), 
commits.get(commit).getRight()));
+      }
+    }
+    Map<String, HoodieSourceSplitLagInfo> lagInfos = new HashMap<>();
+    for (HoodieSourceSplit split : splits) {
+      lagInfos.put(split.splitId(), getLagInfo(split, commits, 
writesByCommit));
+    }
+    return lagInfos;
+  }
+
+  private static HoodieSourceSplitLagInfo getLagInfo(
+      HoodieSourceSplit split,
+      Map<String, Pair<HoodieInstant, HoodieTimeline>> commits,
+      Map<String, Map<String, long[]>> writesByCommit) {
+    Collection<String> rangeCommits = 
HoodieSourceSplitLagInfo.getRangeCommits(split);
+    List<String> readCommits = new ArrayList<>();
+    long records = 0L;
+    long bytes = 0L;
+    if (isExactMatch(split)) {
+      for (String commit : new TreeSet<>(rangeCommits)) {

Review Comment:
   Sure, done.



##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/enumerator/HoodieContinuousSplitEnumerator.java:
##########
@@ -47,6 +58,12 @@ public class HoodieContinuousSplitEnumerator extends 
AbstractHoodieSplitEnumerat
    */
   private final AtomicReference<HoodieEnumeratorPosition> position;
 
+  // Read lag metrics, null when the context has no metric group
+  private final FlinkStreamReadLagMetrics lagMetrics;
+
+  // Splits restored from the state, whose lag info is resolved once the 
enumerator starts
+  private final List<HoodieSourceSplit> restoredSplits;

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to