This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20563-65c9034e4fe192832aa62e0ee552e85192f165d9
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 6037444c385bb36394eff4d134926882eb103a60
Author: vamshipasunuru1 <[email protected]>
AuthorDate: Thu Nov 19 01:13:31 2026 +0000

    fix(multi-writer): abort compaction commit when its heartbeat has expired 
(#20563)
    
    * fix(client): abort compaction commit when its heartbeat has expired
    
    Similar to the write commit path, check the heartbeat of the compaction job 
before committing. Once the
    heartbeat expires a concurrent writer may roll the compaction back, so 
committing afterwards would
    reference deleted files.
    
    * test(multi-writer): cover compaction commit fencing on expired heartbeat
    
    - Add a multi-writer functional test for the scenario in #20562: writer1
      executes compaction T without committing and its heartbeat expires,
      writer2 rolls back, re-executes and commits T, and writer1's
      commitCompaction must then fail with only one completed T on the timeline.
    - Assert in the unit tests that an aborted compaction is never transitioned
      to completed and that a committed one is.
    - Add unit tests for a live heartbeat and for the EAGER failed-writes 
policy,
      where an expired heartbeat does not block the commit.
    
    * test(multi-writer): widen heartbeat window in live-heartbeat compaction 
test
    
    A single-interval heartbeat window can expire on one late scheduler tick,
    which would make the live-heartbeat case fail on a slow CI host.
    
    ---------
    
    Co-authored-by: vamshi <[email protected]>
    Co-authored-by: sivabalan <[email protected]>
---
 .../hudi/client/BaseHoodieTableServiceClient.java  |   4 +
 .../client/TestBaseHoodieTableServiceClient.java   | 110 +++++++++++++++++++
 .../hudi/client/TestHoodieClientMultiWriter.java   | 119 +++++++++++++++++++++
 3 files changed, 233 insertions(+)

diff --git 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
index 06020e059199..c715b8ac00b1 100644
--- 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
+++ 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
@@ -404,6 +404,10 @@ public abstract class BaseHoodieTableServiceClient<I, T, 
O> extends BaseHoodieCl
       final HoodieInstant compactionInstant = 
instantGenerator.getCompactionInflightInstant(compactionCommitTime);
       try {
         this.txnManager.beginStateChange(Option.of(compactionInstant), 
Option.empty());
+        // a concurrent writer may have rolled back the compaction once its 
heartbeat expired
+        if (heartbeatClient.getHeartbeat(compactionCommitTime) != null) {
+          WriterHeartbeatUtils.abortIfHeartbeatExpired(compactionCommitTime, 
table, heartbeatClient, config);
+        }
         preCommit(metadata, false);
         finalizeWrite(table, compactionCommitTime, writeStats);
         // commit to data table after committing to metadata table.
diff --git 
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java
 
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java
index 76aa9828f559..8d6c9f27c4cb 100644
--- 
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java
+++ 
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieTableServiceClient.java
@@ -26,6 +26,7 @@ import 
org.apache.hudi.client.embedded.EmbeddedTimelineService;
 import org.apache.hudi.common.HoodiePendingRollbackInfo;
 import org.apache.hudi.common.config.metrics.HoodieMetricsConfig;
 import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
 import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
 import org.apache.hudi.common.model.WriteOperationType;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -41,8 +42,10 @@ import org.apache.hudi.common.util.Option;
 import org.apache.hudi.config.HoodieCleanConfig;
 import org.apache.hudi.config.HoodieClusteringConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
 import org.apache.hudi.metrics.MetricsReporterType;
 import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
 import org.apache.hudi.table.HoodieTable;
 import org.apache.hudi.table.action.HoodieWriteMetadata;
 
@@ -64,8 +67,10 @@ import static 
org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorage
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
 import static org.mockito.Mockito.mock;
@@ -426,6 +431,111 @@ class TestBaseHoodieTableServiceClient extends 
HoodieCommonTestHarness {
         "Clustering instants should NOT be rolled back when config is 
disabled");
   }
 
+  @Test
+  void completeCompactionAbortsWhenHeartbeatExpired() throws Exception {
+    initMetaClient();
+    HoodieTable<String, String, String, String> table = 
mockTableForCompactionCommit();
+    TestTableServiceClient client = new 
TestTableServiceClient(buildLazyConfigWithShortHeartbeat(), 
Collections.singletonList(table).iterator(),
+        Option.empty(), Collections.emptyMap(), Collections.emptyIterator());
+    String compactionTime = InProcessTimeGenerator.createNewInstantTime();
+
+    // the job executing the compaction stops heartbeating (e.g. infra issue) 
until the heartbeat expires,
+    // at which point a concurrent writer may roll the compaction back
+    client.getHeartbeatClient().start(compactionTime);
+    client.getHeartbeatClient().stopHeartbeatTimers();
+    Thread.sleep(1500);
+    assertTrue(client.getHeartbeatClient().isHeartbeatExpired(compactionTime));
+
+    HoodieException e = assertThrows(HoodieException.class,
+        () -> client.completeCompaction(mock(HoodieCommitMetadata.class), 
table, compactionTime, Collections.emptyList()));
+    assertTrue(e.getMessage().contains("Heartbeat for instant " + 
compactionTime + " has expired"), e.getMessage());
+    // the compaction must not be transitioned to completed
+    verify(table.getActiveTimeline(), 
never()).transitionCompactionInflightToComplete(anyBoolean(), any(), any());
+  }
+
+  @Test
+  void completeCompactionWithoutHeartbeatSucceeds() throws Exception {
+    initMetaClient();
+    HoodieTable<String, String, String, String> table = 
mockTableForCompactionCommit();
+    TestTableServiceClient client = new 
TestTableServiceClient(buildLazyConfigWithShortHeartbeat(), 
Collections.singletonList(table).iterator(),
+        Option.empty(), Collections.emptyMap(), Collections.emptyIterator());
+
+    // no heartbeat was started by this client for the instant, so there is 
nothing to check
+    client.completeCompaction(mock(HoodieCommitMetadata.class), table, 
InProcessTimeGenerator.createNewInstantTime(), Collections.emptyList());
+    
verify(table.getActiveTimeline()).transitionCompactionInflightToComplete(anyBoolean(),
 any(), any());
+  }
+
+  @Test
+  void completeCompactionWithLiveHeartbeatSucceeds() throws Exception {
+    initMetaClient();
+    HoodieTable<String, String, String, String> table = 
mockTableForCompactionCommit();
+    // a wide heartbeat window so a late scheduler tick cannot expire the 
heartbeat mid-test
+    HoodieWriteConfig liveHeartbeatConfig = HoodieWriteConfig.newBuilder()
+        .withPath(basePath)
+        .withCleanConfig(HoodieCleanConfig.newBuilder()
+            
.withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
+            .build())
+        .withHeartbeatIntervalInMs(60 * 1000)
+        .withHeartbeatTolerableMisses(2)
+        .build();
+    TestTableServiceClient client = new 
TestTableServiceClient(liveHeartbeatConfig, 
Collections.singletonList(table).iterator(),
+        Option.empty(), Collections.emptyMap(), Collections.emptyIterator());
+    String compactionTime = InProcessTimeGenerator.createNewInstantTime();
+
+    client.getHeartbeatClient().start(compactionTime);
+    try {
+      
assertFalse(client.getHeartbeatClient().isHeartbeatExpired(compactionTime));
+      client.completeCompaction(mock(HoodieCommitMetadata.class), table, 
compactionTime, Collections.emptyList());
+      
verify(table.getActiveTimeline()).transitionCompactionInflightToComplete(anyBoolean(),
 any(), any());
+    } finally {
+      client.getHeartbeatClient().stop(compactionTime);
+    }
+  }
+
+  @Test
+  void completeCompactionWithExpiredHeartbeatSucceedsForEagerPolicy() throws 
Exception {
+    initMetaClient();
+    HoodieTable<String, String, String, String> table = 
mockTableForCompactionCommit();
+    HoodieWriteConfig eagerConfig = HoodieWriteConfig.newBuilder()
+        .withPath(basePath)
+        .withCleanConfig(HoodieCleanConfig.newBuilder()
+            
.withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.EAGER)
+            .build())
+        .withHeartbeatIntervalInMs(1000)
+        .withHeartbeatTolerableMisses(1)
+        .build();
+    TestTableServiceClient client = new TestTableServiceClient(eagerConfig, 
Collections.singletonList(table).iterator(),
+        Option.empty(), Collections.emptyMap(), Collections.emptyIterator());
+    String compactionTime = InProcessTimeGenerator.createNewInstantTime();
+
+    // heartbeat expiry only fences the commit under the LAZY failed-writes 
policy
+    client.getHeartbeatClient().start(compactionTime);
+    client.getHeartbeatClient().stopHeartbeatTimers();
+    Thread.sleep(1500);
+    assertTrue(client.getHeartbeatClient().isHeartbeatExpired(compactionTime));
+
+    client.completeCompaction(mock(HoodieCommitMetadata.class), table, 
compactionTime, Collections.emptyList());
+    
verify(table.getActiveTimeline()).transitionCompactionInflightToComplete(anyBoolean(),
 any(), any());
+  }
+
+  private HoodieTable<String, String, String, String> 
mockTableForCompactionCommit() {
+    HoodieTable<String, String, String, String> table = 
mock(HoodieTable.class, RETURNS_DEEP_STUBS);
+    when(table.getMetaClient().getBasePath()).thenReturn(new 
StoragePath(basePath));
+    when(table.getMetaClient().getMarkerFolderPath(any())).thenReturn(basePath 
+ "/.hoodie/.temp/marker");
+    return table;
+  }
+
+  private HoodieWriteConfig buildLazyConfigWithShortHeartbeat() {
+    return HoodieWriteConfig.newBuilder()
+        .withPath(basePath)
+        .withCleanConfig(HoodieCleanConfig.newBuilder()
+            
.withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
+            .build())
+        .withHeartbeatIntervalInMs(1000)
+        .withHeartbeatTolerableMisses(1)
+        .build();
+  }
+
   private void createClusteringInstant(String instantTime, String 
clusteringAction) {
     HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
     HoodieInstant requestedInstant = metaClient.getInstantGenerator()
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
 
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
index aa1405e527f8..9cfe32750acf 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
@@ -18,6 +18,7 @@
 
 package org.apache.hudi.client;
 
+import org.apache.hudi.client.heartbeat.HoodieHeartbeatClient;
 import org.apache.hudi.client.transaction.ConflictResolutionStrategy;
 import 
org.apache.hudi.client.transaction.PreferWriterConflictResolutionStrategy;
 import 
org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy;
@@ -34,6 +35,7 @@ import 
org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
 import org.apache.hudi.common.model.HoodieFileFormat;
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.HoodieWriteStat;
 import org.apache.hudi.common.model.TableServiceType;
 import org.apache.hudi.common.model.WriteConcurrencyMode;
 import org.apache.hudi.common.model.WriteOperationType;
@@ -131,6 +133,7 @@ import static 
org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -1227,6 +1230,122 @@ public class TestHoodieClientMultiWriter extends 
HoodieClientTestBase {
     FileIOUtils.deleteDirectory(new File(basePath));
   }
 
+  /**
+   * Test that a compaction writer whose heartbeat expired cannot commit after 
a concurrent writer has
+   * rolled back and re-executed the same compaction instant.
+   *
+   * <p>Writer1 starts executing the compaction but stops heartbeating (e.g. 
an infra stall) until its
+   * heartbeat expires. Writer2 then rolls back the inflight compaction, 
re-executes and commits it.
+   * When writer1 resumes and tries to commit, it must abort instead of 
committing a second time with
+   * write stats that do not match the committed file slices.
+   */
+  @Test
+  public void 
testCompactionCommitAbortsAfterHeartbeatExpiredAndConcurrentRollback() throws 
Exception {
+    setUpMORTestTable();
+
+    int heartbeatIntervalMs = 1000;
+    int numTolerableHeartbeatMisses = 3;
+
+    Properties properties = new Properties();
+    properties.setProperty(FILESYSTEM_LOCK_PATH_PROP_KEY, basePath + 
"/.hoodie/.locks");
+    
properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, 
"3000");
+    
properties.setProperty(LockConfiguration.LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY,
 "1000");
+    
properties.setProperty(LockConfiguration.LOCK_ACQUIRE_NUM_RETRIES_PROP_KEY, 
"3");
+
+    HoodieWriteConfig cfg = getConfigBuilder()
+        .withHeartbeatIntervalInMs(heartbeatIntervalMs)
+        .withHeartbeatTolerableMisses(numTolerableHeartbeatMisses)
+        .withCleanConfig(HoodieCleanConfig.newBuilder()
+            
.withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
+            .withAutoClean(false).build())
+        .withArchivalConfig(HoodieArchivalConfig.newBuilder()
+            .withAutoArchive(false).build())
+        .withCompactionConfig(HoodieCompactionConfig.newBuilder()
+            .withInlineCompaction(false)
+            .withMaxNumDeltaCommitsBeforeCompaction(2).build())
+        .withEmbeddedTimelineServerEnabled(false)
+        .withMarkersType(MarkerType.DIRECT.name())
+        .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
+            .withStorageType(FileSystemViewStorageType.MEMORY)
+            
.withSecondaryStorageType(FileSystemViewStorageType.MEMORY).build())
+        
.withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
+        .withLockConfig(HoodieLockConfig.newBuilder()
+            .withLockProvider(InProcessLockProvider.class)
+            .withConflictResolutionStrategy(new 
SimpleConcurrentFileWritesConflictResolutionStrategy())
+            .build())
+        .withProperties(properties)
+        .build();
+
+    // Base files from inserts, then two delta commits with log files to 
compact
+    SparkRDDWriteClient client = getHoodieWriteClient(cfg);
+    String firstCommitTime = WriteClientTestUtils.createNewInstantTime();
+    createCommitWithInserts(cfg, client, "000", firstCommitTime, 200);
+    String secondCommitTime = WriteClientTestUtils.createNewInstantTime();
+    createCommitWithUpserts(cfg, client, firstCommitTime, Option.empty(), 
secondCommitTime, 100);
+    String thirdCommitTime = WriteClientTestUtils.createNewInstantTime();
+    createCommitWithUpserts(cfg, client, secondCommitTime, Option.empty(), 
thirdCommitTime, 100);
+
+    Option<String> compactionInstantOpt = 
client.scheduleTableService(Option.empty(), Option.empty(), 
TableServiceType.COMPACT);
+    assertTrue(compactionInstantOpt.isPresent(), "Compaction should be 
scheduled");
+    String compactionInstantTime = compactionInstantOpt.get();
+    client.close();
+
+    // Writer1 starts executing the compaction without committing: this starts 
its heartbeat and
+    // transitions the compaction to inflight
+    SparkRDDWriteClient client1 = getHoodieWriteClient(cfg);
+    HoodieWriteMetadata<JavaRDD<WriteStatus>> writer1Metadata = 
client1.compact(compactionInstantTime);
+    
assertTrue(metaClient.reloadActiveTimeline().filterPendingCompactionTimeline()
+            
.containsInstant(INSTANT_GENERATOR.getCompactionInflightInstant(compactionInstantTime)),
+        "Compaction instant should be inflight after writer1 starts executing 
it");
+
+    // Writer1 stalls and stops heartbeating until its heartbeat expires. The 
compaction heartbeat is
+    // owned by the table service client, not by the write client's own 
heartbeat client.
+    HoodieHeartbeatClient writer1HeartbeatClient = 
client1.tableServiceClient.getHeartbeatClient();
+    assertNotNull(writer1HeartbeatClient.getHeartbeat(compactionInstantTime),
+        "Writer1 should have started a heartbeat for the compaction instant");
+    writer1HeartbeatClient.stopHeartbeatTimers();
+    Thread.sleep((long) heartbeatIntervalMs * numTolerableHeartbeatMisses + 
1000);
+    
assertTrue(writer1HeartbeatClient.isHeartbeatExpired(compactionInstantTime));
+
+    // Writer2 sees the expired heartbeat, rolls back the inflight compaction, 
re-executes and commits it
+    SparkRDDWriteClient client2 = getHoodieWriteClient(cfg);
+    assertDoesNotThrow(() -> client2.compact(compactionInstantTime, true),
+        "Writer2 should roll back and re-execute the compaction once writer1's 
heartbeat expired");
+    client2.close();
+
+    HoodieTimeline timelineAfterWriter2 = metaClient.reloadActiveTimeline();
+    HoodieInstant completedCompaction = 
timelineAfterWriter2.filterCompletedInstants().getCommitTimeline()
+        .getInstantsAsStream()
+        .filter(instant -> 
instant.requestedTime().equals(compactionInstantTime))
+        .findFirst()
+        .orElseThrow(() -> new AssertionError("Compaction should be completed 
by writer2"));
+    assertEquals(1, 
timelineAfterWriter2.getRollbackTimeline().filterCompletedInstants().countInstants(),
+        "Writer2 should have rolled back writer1's inflight compaction");
+    HoodieCommitMetadata committedMetadata = 
timelineAfterWriter2.readCommitMetadata(completedCompaction);
+
+    // Writer1 resumes and tries to commit: it must abort because its 
heartbeat expired
+    HoodieException e = assertThrows(HoodieException.class,
+        () -> client1.commitCompaction(compactionInstantTime, writer1Metadata, 
Option.empty()));
+    assertTrue(e.getMessage().contains("Heartbeat for instant " + 
compactionInstantTime + " has expired"), e.getMessage());
+    client1.close();
+
+    // The timeline still has exactly one completed compaction, committed by 
writer2
+    HoodieTimeline finalTimeline = metaClient.reloadActiveTimeline();
+    assertEquals(1, 
finalTimeline.filterCompletedInstants().getCommitTimeline().getInstantsAsStream()
+        .filter(instant -> 
instant.requestedTime().equals(compactionInstantTime)).count());
+    
assertFalse(finalTimeline.filterPendingCompactionTimeline().containsInstant(compactionInstantTime));
+    assertEquals(committedMetadata.getWriteStats().size(),
+        
finalTimeline.readCommitMetadata(completedCompaction).getWriteStats().size());
+
+    // Every base file referenced by the committed compaction exists
+    for (HoodieWriteStat stat : committedMetadata.getWriteStats()) {
+      StoragePath filePath = new StoragePath(basePath, stat.getPath());
+      assertTrue(metaClient.getStorage().exists(filePath), "Committed 
compaction file should exist: " + filePath);
+    }
+
+    FileIOUtils.deleteDirectory(new File(basePath));
+  }
+
   /**
    * Test that when a writer starts compaction but the compaction instant is 
no longer in the
    * active timeline (either already completed or removed), it should throw an 
appropriate error.

Reply via email to