This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20677-b9426d7963ff4a4bda29548067687015ee7b819a
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit f757583aa3e07d2a6e3394e9765d7398a31b8fa2
Author: Danny Chan <[email protected]>
AuthorDate: Thu Nov 19 03:29:56 2026 +0000

    fix(spark): prevent streaming batch rewrite after post-commit failures 
(#20677)
---
 hudi-integ-test/spark-pipelines/SINK_RECOVERY.md   | 42 ++++++++++++
 .../org/apache/hudi/HoodieStreamingSink.scala      | 49 +++++++++++---
 .../functional/TestStreamingSinkRecovery.scala     | 79 ++++++++++++++++++++++
 3 files changed, 162 insertions(+), 8 deletions(-)

diff --git a/hudi-integ-test/spark-pipelines/SINK_RECOVERY.md 
b/hudi-integ-test/spark-pipelines/SINK_RECOVERY.md
new file mode 100644
index 000000000000..f241793a1356
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/SINK_RECOVERY.md
@@ -0,0 +1,42 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+# Streaming sink failures after a data commit
+
+A Hudi write can publish its data commit and then fail during cleaning, 
archival,
+or metastore synchronization. Retrying the entire write would create another
+commit for the same Spark batch and can duplicate rows in insert/append tables.
+
+The streaming sink checks durable batch metadata after a failed attempt. If the
+batch is already committed, it fails the query without retrying the data write.
+The original failure remains in the exception chain. If commit status cannot be
+read, it also stops rather than risking another write. Failures before a commit
+retain the configured retry behavior.
+
+Resolve the reported post-commit problem, including separately rerunning a 
failed
+metastore synchronization when needed, and restart with the same checkpoint and
+streaming checkpoint identifier. The committed batch is skipped; this does not
+rerun the failed post-commit operation. Do not delete the checkpoint to 
recover.
+
+The existing explicit 
`hoodie.datasource.write.streaming.ignore.failed.batch=true`
+opt-out retains its behavior. For reliable failure reporting keep it false.
+This change does not add multi-writer/fan-in support or checkpoint-generation 
resets.
+
+The regression suite injects a failure through a real metadata-sync tool after 
the
+writer commits, for new and pre-existing COW/MOR tables. It checks one write 
attempt,
+visible query failure, exact rows, an absent Spark commit log before restart, 
and
+unchanged Hudi commits after restart from that checkpoint.
diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
index 0084ea10a416..ec8e13f4e452 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
@@ -133,7 +133,7 @@ class HoodieStreamingSink(sqlContext: SQLContext,
     var updatedOptions = 
options.updated(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key, "true")
     updatedOptions = 
updatedOptions.updated(HoodieSparkSqlWriter.SPARK_STREAMING_BATCH_ID, 
batchId.toString)
 
-    retry(retryCnt, retryIntervalMs)(
+    retry(batchId, retryCnt, retryIntervalMs)(
       Try(
         HoodieSparkSqlWriter.write(
           sqlContext, mode, updatedOptions, data, 
Some(StreamingWriteParams(hoodieTableConfig,
@@ -226,16 +226,49 @@ class HoodieStreamingSink(sqlContext: SQLContext,
   override def toString: String = s"HoodieStreamingSink[${options("path")}]"
 
   @annotation.tailrec
-  private def retry[T](n: Int, waitInMillis: Long)(fn: => Try[T]): Try[T] = {
+  private def retry[T](batchId: Long, n: Int, waitInMillis: Long)(fn: => 
Try[T]): Try[T] = {
     fn match {
       case x: Success[T] =>
         x
-      case _ if n > 1 =>
-        Thread.sleep(waitInMillis)
-        retry(n - 1, waitInMillis * 2)(fn)
-      case f =>
-        reset(false)
-        f
+      case failure: Failure[T] =>
+        // A writer can throw during clean/archive/meta sync after publishing 
the commit. Never
+        // retry the write in that case. Fail the query so the post-commit 
failure remains visible;
+        // restarting from the same Spark checkpoint deduplicates the already 
committed batch.
+        val committed = Try(isBatchCommitted(batchId))
+        committed match {
+          case Success(false) if n > 1 =>
+            Thread.sleep(waitInMillis)
+            retry(batchId, n - 1, waitInMillis * 2)(fn)
+          case _ =>
+            reset(false)
+            committed match {
+              case Success(true) => Failure(new HoodieException(
+                s"Micro batch id=$batchId is already committed; refusing to 
retry the write after a post-commit failure. "
+                  + "Resolve the failed post-commit operation and restart from 
the same checkpoint.", failure.exception))
+              case Failure(checkError) =>
+                // An unknown commit outcome is not permission to write the 
batch a second time.
+                failure.exception.addSuppressed(checkError)
+                failure
+              case _ => failure
+            }
+        }
+    }
+  }
+
+  private def isBatchCommitted(batchId: Long): Boolean = {
+    if (metaClient.isEmpty) {
+      // The first batch may have created and committed the table before the 
writer threw.
+      metaClient = try {
+        Some(HoodieTableMetaClient.builder()
+          
.setConf(HadoopFSUtils.getStorageConfWithCopy(sqlContext.sparkContext.hadoopConfiguration))
+          .setBasePath(tablePath.get).build())
+      } catch {
+        case _: TableNotFoundException => None
+      }
+    }
+    metaClient.exists { client =>
+      client.reloadActiveTimeline()
+      canSkipBatch(batchId, options.getOrElse(OPERATION.key, 
UPSERT_OPERATION_OPT_VAL))
     }
   }
 
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala
index 9f43a314efaa..9d65aa621184 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala
@@ -19,6 +19,8 @@ package org.apache.hudi.functional
 
 import org.apache.hudi.{DataSourceWriteOptions, HoodieStreamingSink}
 import org.apache.hudi.common.testutils.HoodieTestUtils
+import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.sync.common.HoodieSyncTool
 import org.apache.hudi.testutils.HoodieSparkClientTestBase
 
 import org.apache.spark.sql.{DataFrame, SQLContext}
@@ -30,7 +32,9 @@ import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.CsvSource
 
 import java.net.{InetSocketAddress, Socket}
+import java.util.Properties
 import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit}
+import java.util.concurrent.atomic.AtomicInteger
 
 import scala.collection.JavaConverters._
 
@@ -71,7 +75,82 @@ class RecoveryTestSink(context: SQLContext, options: 
Map[String, String], partit
   }
 }
 
+/** The real writer invokes this only after publishing its data commit. */
+class FailingStreamingSyncTool(properties: Properties) extends 
HoodieSyncTool(properties) {
+  override def syncHoodieTable(): Unit = {
+    FailingStreamingSyncTool.attempts.incrementAndGet()
+    throw new HoodieException("injected post-commit meta sync failure")
+  }
+}
+
+object FailingStreamingSyncTool {
+  val attempts = new AtomicInteger()
+}
+
 class TestStreamingSinkRecovery extends HoodieSparkClientTestBase {
+  @ParameterizedTest
+  @CsvSource(Array("COPY_ON_WRITE,false", "MERGE_ON_READ,false", 
"COPY_ON_WRITE,true", "MERGE_ON_READ,true"))
+  def testPostCommitFailureDoesNotRetryWrite(tableType: String, existingTable: 
Boolean): Unit = {
+    val spark = sqlContext.sparkSession
+    val source = s"$basePath/source"
+    val target = s"$basePath/target"
+    val checkpoint = s"$basePath/checkpoint"
+    val input = spark.sql("select 1 as id, 10L as ts, 'first' as value")
+    val options = Map(
+      "hoodie.table.name" -> "post_commit_recovery",
+      "hoodie.datasource.write.table.type" -> tableType,
+      "hoodie.datasource.write.recordkey.field" -> "id",
+      "hoodie.datasource.write.precombine.field" -> "ts",
+      "hoodie.datasource.write.partitionpath.field" -> "",
+      "hoodie.datasource.write.operation" -> "insert",
+      "hoodie.metadata.enable" -> "false",
+      "hoodie.insert.shuffle.parallelism" -> "1",
+      DataSourceWriteOptions.STREAMING_DISABLE_COMPACTION.key -> "true",
+      DataSourceWriteOptions.STREAMING_RETRY_CNT.key -> "3",
+      DataSourceWriteOptions.STREAMING_RETRY_INTERVAL_MS.key -> "1",
+      "checkpointLocation" -> checkpoint)
+    val seed = spark.sql("select 0 as id, 0L as ts, 'seed' as value")
+    if (existingTable) {
+      seed.write.format("hudi").options(options).save(target)
+    }
+    input.coalesce(1).write.json(source)
+    FailingStreamingSyncTool.attempts.set(0)
+    val query = 
spark.readStream.schema(input.schema).json(source).writeStream.format("hudi")
+      .options(options ++ Map("hoodie.datasource.meta.sync.enable" -> "true",
+        DataSourceWriteOptions.META_SYNC_CLIENT_TOOL_CLASS_NAME.key -> 
classOf[FailingStreamingSyncTool].getName))
+      .trigger(Trigger.AvailableNow()).start(target)
+    try {
+      val error = assertThrows(classOf[StreamingQueryException], () => 
query.awaitTermination(60000))
+      val causes = Iterator.iterate[Throwable](error)(_.getCause).takeWhile(_ 
!= null)
+        .map(_.getMessage).mkString("\n")
+      assertTrue(causes.contains("already committed"), causes)
+      assertTrue(causes.contains("injected post-commit meta sync failure"), 
causes)
+      assertEquals(1, FailingStreamingSyncTool.attempts.get(), "Post-commit 
failure retried the data write")
+      assertFalse(new java.io.File(s"$checkpoint/commits/0").exists())
+    } finally {
+      query.stop()
+    }
+    def commits: Seq[String] = HoodieTestUtils.createMetaClient(storage, 
target).getActiveTimeline
+      
.getCommitsTimeline.filterCompletedInstants().getInstants.asScala.map(_.requestedTime()).toSeq
+    val before = commits
+    assertEquals(if (existingTable) 2 else 1, before.size)
+    val expected = if (existingTable) seed.union(input) else input
+    def checkRows(): Unit = 
assertEquals(expected.orderBy("id").collect().toSeq,
+      spark.read.format("hudi").load(target).select("id", "ts", 
"value").orderBy("id").collect().toSeq)
+    checkRows()
+    // The failed sync is external to the data transaction; disable the 
injected tool after repair.
+    val replay = 
spark.readStream.schema(input.schema).json(source).writeStream.format("hudi")
+      .options(options).trigger(Trigger.AvailableNow()).start(target)
+    try {
+      assertTrue(replay.awaitTermination(60000))
+    } finally {
+      replay.stop()
+    }
+    assertEquals(before, commits, "Restart recommitted a completed Hudi batch")
+    assertTrue(new java.io.File(s"$checkpoint/commits/0").exists())
+    checkRows()
+  }
+
   // This covers Spark checkpoint replay, not failures inside the sink retry 
loop (tracked by #20406).
   @ParameterizedTest
   @CsvSource(Array("available-now,COPY_ON_WRITE", "stop,COPY_ON_WRITE", 
"failure,COPY_ON_WRITE",

Reply via email to