This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20563-65c9034e4fe192832aa62e0ee552e85192f165d9 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 6037444c385bb36394eff4d134926882eb103a60 Author: vamshipasunuru1 <[email protected]> AuthorDate: Thu Nov 19 01:13:31 2026 +0000 fix(multi-writer): abort compaction commit when its heartbeat has expired (#20563) * fix(client): abort compaction commit when its heartbeat has expired Similar to the write commit path, check the heartbeat of the compaction job before committing. Once the heartbeat expires a concurrent writer may roll the compaction back, so committing afterwards would reference deleted files. * test(multi-writer): cover compaction commit fencing on expired heartbeat - Add a multi-writer functional test for the scenario in #20562: writer1 executes compaction T without committing and its heartbeat expires, writer2 rolls back, re-executes and commits T, and writer1's commitCompaction must then fail with only one completed T on the timeline. - Assert in the unit tests that an aborted compaction is never transitioned to completed and that a committed one is. - Add unit tests for a live heartbeat and for the EAGER failed-writes policy, where an expired heartbeat does not block the commit. * test(multi-writer): widen heartbeat window in live-heartbeat compaction test A single-interval heartbeat window can expire on one late scheduler tick, which would make the live-heartbeat case fail on a slow CI host. --------- Co-authored-by: vamshi <[email protected]> Co-authored-by: sivabalan <[email protected]> --- .../hudi/client/BaseHoodieTableServiceClient.java | 4 + .../client/TestBaseHoodieTableServiceClient.java | 110 +++++++++++++++++++ .../hudi/client/TestHoodieClientMultiWriter.java | 119 +++++++++++++++++++++ 3 files changed, 233 insertions(+) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java index 06020e059199..c715b8ac00b1 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java @@ -404,6 +404,10 @@ public abstract class BaseHoodieTableServiceClient<I, T, O> extends BaseHoodieCl final HoodieInstant compactionInstant = instantGenerator.getCompactionInflightInstant(compactionCommitTime); try { this.txnManager.beginStateChange(Option.of(compactionInstant), Option.empty()); + // a concurrent writer may have rolled back the compaction once its heartbeat expired + if (heartbeatClient.getHeartbeat(compactionCommitTime) != null) { + WriterHeartbeatUtils.abortIfHeartbeatExpired(compactionCommitTime, table, heartbeatClient, config); + } preCommit(metadata, false); finalizeWrite(table, compactionCommitTime, writeStats); // commit to data table after committing to metadata table. diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java index 76aa9828f559..8d6c9f27c4cb 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java @@ -26,6 +26,7 @@ import org.apache.hudi.client.embedded.EmbeddedTimelineService; import org.apache.hudi.common.HoodiePendingRollbackInfo; import org.apache.hudi.common.config.metrics.HoodieMetricsConfig; import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -41,8 +42,10 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieClusteringConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metrics.MetricsReporterType; import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; @@ -64,8 +67,10 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorage import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; @@ -426,6 +431,111 @@ class TestBaseHoodieTableServiceClient extends HoodieCommonTestHarness { "Clustering instants should NOT be rolled back when config is disabled"); } + @Test + void completeCompactionAbortsWhenHeartbeatExpired() throws Exception { + initMetaClient(); + HoodieTable<String, String, String, String> table = mockTableForCompactionCommit(); + TestTableServiceClient client = new TestTableServiceClient(buildLazyConfigWithShortHeartbeat(), Collections.singletonList(table).iterator(), + Option.empty(), Collections.emptyMap(), Collections.emptyIterator()); + String compactionTime = InProcessTimeGenerator.createNewInstantTime(); + + // the job executing the compaction stops heartbeating (e.g. infra issue) until the heartbeat expires, + // at which point a concurrent writer may roll the compaction back + client.getHeartbeatClient().start(compactionTime); + client.getHeartbeatClient().stopHeartbeatTimers(); + Thread.sleep(1500); + assertTrue(client.getHeartbeatClient().isHeartbeatExpired(compactionTime)); + + HoodieException e = assertThrows(HoodieException.class, + () -> client.completeCompaction(mock(HoodieCommitMetadata.class), table, compactionTime, Collections.emptyList())); + assertTrue(e.getMessage().contains("Heartbeat for instant " + compactionTime + " has expired"), e.getMessage()); + // the compaction must not be transitioned to completed + verify(table.getActiveTimeline(), never()).transitionCompactionInflightToComplete(anyBoolean(), any(), any()); + } + + @Test + void completeCompactionWithoutHeartbeatSucceeds() throws Exception { + initMetaClient(); + HoodieTable<String, String, String, String> table = mockTableForCompactionCommit(); + TestTableServiceClient client = new TestTableServiceClient(buildLazyConfigWithShortHeartbeat(), Collections.singletonList(table).iterator(), + Option.empty(), Collections.emptyMap(), Collections.emptyIterator()); + + // no heartbeat was started by this client for the instant, so there is nothing to check + client.completeCompaction(mock(HoodieCommitMetadata.class), table, InProcessTimeGenerator.createNewInstantTime(), Collections.emptyList()); + verify(table.getActiveTimeline()).transitionCompactionInflightToComplete(anyBoolean(), any(), any()); + } + + @Test + void completeCompactionWithLiveHeartbeatSucceeds() throws Exception { + initMetaClient(); + HoodieTable<String, String, String, String> table = mockTableForCompactionCommit(); + // a wide heartbeat window so a late scheduler tick cannot expire the heartbeat mid-test + HoodieWriteConfig liveHeartbeatConfig = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) + .build()) + .withHeartbeatIntervalInMs(60 * 1000) + .withHeartbeatTolerableMisses(2) + .build(); + TestTableServiceClient client = new TestTableServiceClient(liveHeartbeatConfig, Collections.singletonList(table).iterator(), + Option.empty(), Collections.emptyMap(), Collections.emptyIterator()); + String compactionTime = InProcessTimeGenerator.createNewInstantTime(); + + client.getHeartbeatClient().start(compactionTime); + try { + assertFalse(client.getHeartbeatClient().isHeartbeatExpired(compactionTime)); + client.completeCompaction(mock(HoodieCommitMetadata.class), table, compactionTime, Collections.emptyList()); + verify(table.getActiveTimeline()).transitionCompactionInflightToComplete(anyBoolean(), any(), any()); + } finally { + client.getHeartbeatClient().stop(compactionTime); + } + } + + @Test + void completeCompactionWithExpiredHeartbeatSucceedsForEagerPolicy() throws Exception { + initMetaClient(); + HoodieTable<String, String, String, String> table = mockTableForCompactionCommit(); + HoodieWriteConfig eagerConfig = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.EAGER) + .build()) + .withHeartbeatIntervalInMs(1000) + .withHeartbeatTolerableMisses(1) + .build(); + TestTableServiceClient client = new TestTableServiceClient(eagerConfig, Collections.singletonList(table).iterator(), + Option.empty(), Collections.emptyMap(), Collections.emptyIterator()); + String compactionTime = InProcessTimeGenerator.createNewInstantTime(); + + // heartbeat expiry only fences the commit under the LAZY failed-writes policy + client.getHeartbeatClient().start(compactionTime); + client.getHeartbeatClient().stopHeartbeatTimers(); + Thread.sleep(1500); + assertTrue(client.getHeartbeatClient().isHeartbeatExpired(compactionTime)); + + client.completeCompaction(mock(HoodieCommitMetadata.class), table, compactionTime, Collections.emptyList()); + verify(table.getActiveTimeline()).transitionCompactionInflightToComplete(anyBoolean(), any(), any()); + } + + private HoodieTable<String, String, String, String> mockTableForCompactionCommit() { + HoodieTable<String, String, String, String> table = mock(HoodieTable.class, RETURNS_DEEP_STUBS); + when(table.getMetaClient().getBasePath()).thenReturn(new StoragePath(basePath)); + when(table.getMetaClient().getMarkerFolderPath(any())).thenReturn(basePath + "/.hoodie/.temp/marker"); + return table; + } + + private HoodieWriteConfig buildLazyConfigWithShortHeartbeat() { + return HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) + .build()) + .withHeartbeatIntervalInMs(1000) + .withHeartbeatTolerableMisses(1) + .build(); + } + private void createClusteringInstant(String instantTime, String clusteringAction) { HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); HoodieInstant requestedInstant = metaClient.getInstantGenerator() diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java index aa1405e527f8..9cfe32750acf 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java @@ -18,6 +18,7 @@ package org.apache.hudi.client; +import org.apache.hudi.client.heartbeat.HoodieHeartbeatClient; import org.apache.hudi.client.transaction.ConflictResolutionStrategy; import org.apache.hudi.client.transaction.PreferWriterConflictResolutionStrategy; import org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy; @@ -34,6 +35,7 @@ import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.model.WriteOperationType; @@ -131,6 +133,7 @@ import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -1227,6 +1230,122 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase { FileIOUtils.deleteDirectory(new File(basePath)); } + /** + * Test that a compaction writer whose heartbeat expired cannot commit after a concurrent writer has + * rolled back and re-executed the same compaction instant. + * + * <p>Writer1 starts executing the compaction but stops heartbeating (e.g. an infra stall) until its + * heartbeat expires. Writer2 then rolls back the inflight compaction, re-executes and commits it. + * When writer1 resumes and tries to commit, it must abort instead of committing a second time with + * write stats that do not match the committed file slices. + */ + @Test + public void testCompactionCommitAbortsAfterHeartbeatExpiredAndConcurrentRollback() throws Exception { + setUpMORTestTable(); + + int heartbeatIntervalMs = 1000; + int numTolerableHeartbeatMisses = 3; + + Properties properties = new Properties(); + properties.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath + "/.hoodie/.locks"); + properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000"); + properties.setProperty(LockConfiguration.LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "1000"); + properties.setProperty(LockConfiguration.LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY, "3"); + + HoodieWriteConfig cfg = getConfigBuilder() + .withHeartbeatIntervalInMs(heartbeatIntervalMs) + .withHeartbeatTolerableMisses(numTolerableHeartbeatMisses) + .withCleanConfig(HoodieCleanConfig.newBuilder() + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) + .withAutoClean(false).build()) + .withArchivalConfig(HoodieArchivalConfig.newBuilder() + .withAutoArchive(false).build()) + .withCompactionConfig(HoodieCompactionConfig.newBuilder() + .withInlineCompaction(false) + .withMaxNumDeltaCommitsBeforeCompaction(2).build()) + .withEmbeddedTimelineServerEnabled(false) + .withMarkersType(MarkerType.DIRECT.name()) + .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder() + .withStorageType(FileSystemViewStorageType.MEMORY) + .withSecondaryStorageType(FileSystemViewStorageType.MEMORY).build()) + .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL) + .withLockConfig(HoodieLockConfig.newBuilder() + .withLockProvider(InProcessLockProvider.class) + .withConflictResolutionStrategy(new SimpleConcurrentFileWritesConflictResolutionStrategy()) + .build()) + .withProperties(properties) + .build(); + + // Base files from inserts, then two delta commits with log files to compact + SparkRDDWriteClient client = getHoodieWriteClient(cfg); + String firstCommitTime = WriteClientTestUtils.createNewInstantTime(); + createCommitWithInserts(cfg, client, "000", firstCommitTime, 200); + String secondCommitTime = WriteClientTestUtils.createNewInstantTime(); + createCommitWithUpserts(cfg, client, firstCommitTime, Option.empty(), secondCommitTime, 100); + String thirdCommitTime = WriteClientTestUtils.createNewInstantTime(); + createCommitWithUpserts(cfg, client, secondCommitTime, Option.empty(), thirdCommitTime, 100); + + Option<String> compactionInstantOpt = client.scheduleTableService(Option.empty(), Option.empty(), TableServiceType.COMPACT); + assertTrue(compactionInstantOpt.isPresent(), "Compaction should be scheduled"); + String compactionInstantTime = compactionInstantOpt.get(); + client.close(); + + // Writer1 starts executing the compaction without committing: this starts its heartbeat and + // transitions the compaction to inflight + SparkRDDWriteClient client1 = getHoodieWriteClient(cfg); + HoodieWriteMetadata<JavaRDD<WriteStatus>> writer1Metadata = client1.compact(compactionInstantTime); + assertTrue(metaClient.reloadActiveTimeline().filterPendingCompactionTimeline() + .containsInstant(INSTANT_GENERATOR.getCompactionInflightInstant(compactionInstantTime)), + "Compaction instant should be inflight after writer1 starts executing it"); + + // Writer1 stalls and stops heartbeating until its heartbeat expires. The compaction heartbeat is + // owned by the table service client, not by the write client's own heartbeat client. + HoodieHeartbeatClient writer1HeartbeatClient = client1.tableServiceClient.getHeartbeatClient(); + assertNotNull(writer1HeartbeatClient.getHeartbeat(compactionInstantTime), + "Writer1 should have started a heartbeat for the compaction instant"); + writer1HeartbeatClient.stopHeartbeatTimers(); + Thread.sleep((long) heartbeatIntervalMs * numTolerableHeartbeatMisses + 1000); + assertTrue(writer1HeartbeatClient.isHeartbeatExpired(compactionInstantTime)); + + // Writer2 sees the expired heartbeat, rolls back the inflight compaction, re-executes and commits it + SparkRDDWriteClient client2 = getHoodieWriteClient(cfg); + assertDoesNotThrow(() -> client2.compact(compactionInstantTime, true), + "Writer2 should roll back and re-execute the compaction once writer1's heartbeat expired"); + client2.close(); + + HoodieTimeline timelineAfterWriter2 = metaClient.reloadActiveTimeline(); + HoodieInstant completedCompaction = timelineAfterWriter2.filterCompletedInstants().getCommitTimeline() + .getInstantsAsStream() + .filter(instant -> instant.requestedTime().equals(compactionInstantTime)) + .findFirst() + .orElseThrow(() -> new AssertionError("Compaction should be completed by writer2")); + assertEquals(1, timelineAfterWriter2.getRollbackTimeline().filterCompletedInstants().countInstants(), + "Writer2 should have rolled back writer1's inflight compaction"); + HoodieCommitMetadata committedMetadata = timelineAfterWriter2.readCommitMetadata(completedCompaction); + + // Writer1 resumes and tries to commit: it must abort because its heartbeat expired + HoodieException e = assertThrows(HoodieException.class, + () -> client1.commitCompaction(compactionInstantTime, writer1Metadata, Option.empty())); + assertTrue(e.getMessage().contains("Heartbeat for instant " + compactionInstantTime + " has expired"), e.getMessage()); + client1.close(); + + // The timeline still has exactly one completed compaction, committed by writer2 + HoodieTimeline finalTimeline = metaClient.reloadActiveTimeline(); + assertEquals(1, finalTimeline.filterCompletedInstants().getCommitTimeline().getInstantsAsStream() + .filter(instant -> instant.requestedTime().equals(compactionInstantTime)).count()); + assertFalse(finalTimeline.filterPendingCompactionTimeline().containsInstant(compactionInstantTime)); + assertEquals(committedMetadata.getWriteStats().size(), + finalTimeline.readCommitMetadata(completedCompaction).getWriteStats().size()); + + // Every base file referenced by the committed compaction exists + for (HoodieWriteStat stat : committedMetadata.getWriteStats()) { + StoragePath filePath = new StoragePath(basePath, stat.getPath()); + assertTrue(metaClient.getStorage().exists(filePath), "Committed compaction file should exist: " + filePath); + } + + FileIOUtils.deleteDirectory(new File(basePath)); + } + /** * Test that when a writer starts compaction but the compaction instant is no longer in the * active timeline (either already completed or removed), it should throw an appropriate error.
