danny0405 commented on code in PR #10874: URL: https://github.com/apache/hudi/pull/10874#discussion_r1537190750
########## hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java: ########## @@ -1407,43 +1382,7 @@ protected void cleanIfNecessary(BaseHoodieWriteClient writeClient, String instan writeClient.lazyRollbackFailedIndexing(); } - /** - * Validates the timeline for both main and metadata tables to ensure compaction on MDT can be scheduled. - */ - protected boolean validateTimelineBeforeSchedulingCompaction(Option<String> inFlightInstantTimestamp, String latestDeltaCommitTimeInMetadataTable) { - // we need to find if there are any inflights in data table timeline before or equal to the latest delta commit in metadata table. - // Whenever you want to change this logic, please ensure all below scenarios are considered. - // a. There could be a chance that latest delta commit in MDT is committed in MDT, but failed in DT. And so findInstantsBeforeOrEquals() should be employed - // b. There could be DT inflights after latest delta commit in MDT and we are ok with it. bcoz, the contract is, the latest compaction instant time in MDT represents - // any instants before that is already synced with metadata table. - // c. Do consider out of order commits. For eg, c4 from DT could complete before c3. and we can't trigger compaction in MDT with c4 as base instant time, until every - // instant before c4 is synced with metadata table. - List<HoodieInstant> pendingInstants = dataMetaClient.reloadActiveTimeline().filterInflightsAndRequested() - .findInstantsBeforeOrEquals(latestDeltaCommitTimeInMetadataTable).getInstants(); - - if (!pendingInstants.isEmpty()) { - checkNumDeltaCommits(metadataMetaClient, dataWriteConfig.getMetadataConfig().getMaxNumDeltacommitsWhenPending()); - LOG.info(String.format( - "Cannot compact metadata table as there are %d inflight instants in data table before latest deltacommit in metadata table: %s. Inflight instants in data table: %s", - pendingInstants.size(), latestDeltaCommitTimeInMetadataTable, Arrays.toString(pendingInstants.toArray()))); - return false; - } - - // Check if there are any pending compaction or log compaction instants in the timeline. - // If pending compact/logCompaction operations are found abort scheduling new compaction/logCompaction operations. - Option<HoodieInstant> pendingLogCompactionInstant = Review Comment: Currently log compaction is only used for MDT, we should move it out in the future, cc @suryaprasanna to address this in issue: https://issues.apache.org/jira/browse/HUDI-7533 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org