This is an automated email from the ASF dual-hosted git repository. danny0405 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 4f50fde3da3 [HUDI-8009] Optimize the code of HoodieTable#getPendingCommitTimeline (#11665) 4f50fde3da3 is described below commit 4f50fde3da345bea989b26ae1e7944d94e426dfd Author: usberkeley <150880684+usberke...@users.noreply.github.com> AuthorDate: Wed Jul 24 08:55:56 2024 +0800 [HUDI-8009] Optimize the code of HoodieTable#getPendingCommitTimeline (#11665) --- .../hudi/index/bucket/ConsistentBucketIndexUtils.java | 2 +- .../src/main/java/org/apache/hudi/table/HoodieTable.java | 4 ++-- .../action/rollback/BaseRollbackActionExecutor.java | 4 ++-- .../apache/hudi/table/upgrade/TestUpgradeDowngrade.java | 16 ++++++++-------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java index 7473e0d6632..453f08a486a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java @@ -270,7 +270,7 @@ public class ConsistentBucketIndexUtils { private static boolean recommitMetadataFile(HoodieTable table, FileStatus metaFile, String partition) { Path partitionPath = new Path(FSUtils.constructAbsolutePath(table.getMetaClient().getBasePath(), partition).toUri()); String timestamp = getTimestampFromFile(metaFile.getPath().getName()); - if (table.getPendingCommitTimeline().containsInstant(timestamp)) { + if (table.getPendingCommitsTimeline().containsInstant(timestamp)) { return false; } Option<HoodieConsistentHashingMetadata> hoodieConsistentHashingMetadataOption = loadMetadataFromGivenFile(table, metaFile); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java index ef14396bf9e..9e51ef99ae4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -367,8 +367,8 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable { /** * Get only the inflights (no-completed) commit timeline. */ - public HoodieTimeline getPendingCommitTimeline() { - return metaClient.getCommitsTimeline().filterPendingExcludingMajorAndMinorCompaction(); + public HoodieTimeline getPendingCommitsTimeline() { + return metaClient.getCommitsTimeline().filterInflightsAndRequested(); } /** diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java index 5e02f20c86a..e4094e86864 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java @@ -165,7 +165,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE if (config.getFailedWritesCleanPolicy().isEager() && !HoodieTableMetadata.isMetadataTable(config.getBasePath())) { final String instantTimeToRollback = instantToRollback.getTimestamp(); HoodieTimeline commitTimeline = table.getCompletedCommitsTimeline(); - HoodieTimeline inflightAndRequestedCommitTimeline = table.getPendingCommitTimeline(); + HoodieTimeline pendingCommitsTimeline = table.getPendingCommitsTimeline(); // Check validity of completed commit timeline. // Make sure only the last n commits are being rolled back // If there is a commit in-between or after that is not rolled back, then abort @@ -184,7 +184,7 @@ public abstract class BaseRollbackActionExecutor<T, I, K, O> extends BaseActionE } } - List<String> inflights = inflightAndRequestedCommitTimeline.getInstantsAsStream() + List<String> inflights = pendingCommitsTimeline.getInstantsAsStream() .filter(instant -> !ClusteringUtils.isClusteringInstant(table.getActiveTimeline(), instant)) .map(HoodieInstant::getTimestamp) .collect(Collectors.toList()); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java index c993292a6d4..4e3d98f52bb 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java @@ -199,16 +199,16 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { HoodieTable table = this.getHoodieTable(metaClient, cfg); prepForUpgradeFromZeroToOne(table); - HoodieInstant commitInstant = table.getPendingCommitTimeline().lastInstant().get(); + HoodieInstant commitsInstant = table.getPendingCommitsTimeline().lastInstant().get(); // delete one of the marker files in 2nd commit if need be. WriteMarkers writeMarkers = - WriteMarkersFactory.get(getConfig().getMarkersType(), table, commitInstant.getTimestamp()); + WriteMarkersFactory.get(getConfig().getMarkersType(), table, commitsInstant.getTimestamp()); List<String> markerPaths = new ArrayList<>(writeMarkers.allMarkerFilePaths()); if (deletePartialMarkerFiles) { String toDeleteMarkerFile = markerPaths.get(0); table.getStorage().deleteDirectory(new StoragePath( - table.getMetaClient().getTempFolderPath() + "/" + commitInstant.getTimestamp() + table.getMetaClient().getTempFolderPath() + "/" + commitsInstant.getTimestamp() + "/" + toDeleteMarkerFile)); markerPaths.remove(toDeleteMarkerFile); } @@ -225,7 +225,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { .run(HoodieTableVersion.ONE, null); // assert marker files - assertMarkerFilesForUpgrade(table, commitInstant, firstPartitionCommit2FileSlices, secondPartitionCommit2FileSlices); + assertMarkerFilesForUpgrade(table, commitsInstant, firstPartitionCommit2FileSlices, secondPartitionCommit2FileSlices); // verify hoodie.table.version got upgraded metaClient = HoodieTableMetaClient.builder() @@ -614,15 +614,15 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { twoUpsertCommitDataWithTwoPartitions(firstPartitionCommit2FileSlices, secondPartitionCommit2FileSlices, cfg, client, false); HoodieTable table = this.getHoodieTable(metaClient, cfg); - HoodieInstant commitInstant = table.getPendingCommitTimeline().lastInstant().get(); + HoodieInstant commitsInstant = table.getPendingCommitsTimeline().lastInstant().get(); // delete one of the marker files in 2nd commit if need be. - WriteMarkers writeMarkers = WriteMarkersFactory.get(markerType, table, commitInstant.getTimestamp()); + WriteMarkers writeMarkers = WriteMarkersFactory.get(markerType, table, commitsInstant.getTimestamp()); List<String> markerPaths = new ArrayList<>(writeMarkers.allMarkerFilePaths()); if (deletePartialMarkerFiles) { String toDeleteMarkerFile = markerPaths.get(0); table.getStorage().deleteDirectory(new StoragePath( - table.getMetaClient().getTempFolderPath() + "/" + commitInstant.getTimestamp() + table.getMetaClient().getTempFolderPath() + "/" + commitsInstant.getTimestamp() + "/" + toDeleteMarkerFile)); markerPaths.remove(toDeleteMarkerFile); } @@ -635,7 +635,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { if (fromVersion.versionCode() == HoodieTableVersion.TWO.versionCode()) { // assert marker files - assertMarkerFilesForDowngrade(table, commitInstant, toVersion == HoodieTableVersion.ONE); + assertMarkerFilesForDowngrade(table, commitsInstant, toVersion == HoodieTableVersion.ONE); } // verify hoodie.table.version got downgraded