This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20677-b9426d7963ff4a4bda29548067687015ee7b819a in repository https://gitbox.apache.org/repos/asf/hudi.git
commit f757583aa3e07d2a6e3394e9765d7398a31b8fa2 Author: Danny Chan <[email protected]> AuthorDate: Thu Nov 19 03:29:56 2026 +0000 fix(spark): prevent streaming batch rewrite after post-commit failures (#20677) --- hudi-integ-test/spark-pipelines/SINK_RECOVERY.md | 42 ++++++++++++ .../org/apache/hudi/HoodieStreamingSink.scala | 49 +++++++++++--- .../functional/TestStreamingSinkRecovery.scala | 79 ++++++++++++++++++++++ 3 files changed, 162 insertions(+), 8 deletions(-) diff --git a/hudi-integ-test/spark-pipelines/SINK_RECOVERY.md b/hudi-integ-test/spark-pipelines/SINK_RECOVERY.md new file mode 100644 index 000000000000..f241793a1356 --- /dev/null +++ b/hudi-integ-test/spark-pipelines/SINK_RECOVERY.md @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Streaming sink failures after a data commit + +A Hudi write can publish its data commit and then fail during cleaning, archival, +or metastore synchronization. Retrying the entire write would create another +commit for the same Spark batch and can duplicate rows in insert/append tables. + +The streaming sink checks durable batch metadata after a failed attempt. If the +batch is already committed, it fails the query without retrying the data write. +The original failure remains in the exception chain. If commit status cannot be +read, it also stops rather than risking another write. Failures before a commit +retain the configured retry behavior. + +Resolve the reported post-commit problem, including separately rerunning a failed +metastore synchronization when needed, and restart with the same checkpoint and +streaming checkpoint identifier. The committed batch is skipped; this does not +rerun the failed post-commit operation. Do not delete the checkpoint to recover. + +The existing explicit `hoodie.datasource.write.streaming.ignore.failed.batch=true` +opt-out retains its behavior. For reliable failure reporting keep it false. +This change does not add multi-writer/fan-in support or checkpoint-generation resets. + +The regression suite injects a failure through a real metadata-sync tool after the +writer commits, for new and pre-existing COW/MOR tables. It checks one write attempt, +visible query failure, exact rows, an absent Spark commit log before restart, and +unchanged Hudi commits after restart from that checkpoint. diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala index 0084ea10a416..ec8e13f4e452 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala @@ -133,7 +133,7 @@ class HoodieStreamingSink(sqlContext: SQLContext, var updatedOptions = options.updated(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key, "true") updatedOptions = updatedOptions.updated(HoodieSparkSqlWriter.SPARK_STREAMING_BATCH_ID, batchId.toString) - retry(retryCnt, retryIntervalMs)( + retry(batchId, retryCnt, retryIntervalMs)( Try( HoodieSparkSqlWriter.write( sqlContext, mode, updatedOptions, data, Some(StreamingWriteParams(hoodieTableConfig, @@ -226,16 +226,49 @@ class HoodieStreamingSink(sqlContext: SQLContext, override def toString: String = s"HoodieStreamingSink[${options("path")}]" @annotation.tailrec - private def retry[T](n: Int, waitInMillis: Long)(fn: => Try[T]): Try[T] = { + private def retry[T](batchId: Long, n: Int, waitInMillis: Long)(fn: => Try[T]): Try[T] = { fn match { case x: Success[T] => x - case _ if n > 1 => - Thread.sleep(waitInMillis) - retry(n - 1, waitInMillis * 2)(fn) - case f => - reset(false) - f + case failure: Failure[T] => + // A writer can throw during clean/archive/meta sync after publishing the commit. Never + // retry the write in that case. Fail the query so the post-commit failure remains visible; + // restarting from the same Spark checkpoint deduplicates the already committed batch. + val committed = Try(isBatchCommitted(batchId)) + committed match { + case Success(false) if n > 1 => + Thread.sleep(waitInMillis) + retry(batchId, n - 1, waitInMillis * 2)(fn) + case _ => + reset(false) + committed match { + case Success(true) => Failure(new HoodieException( + s"Micro batch id=$batchId is already committed; refusing to retry the write after a post-commit failure. " + + "Resolve the failed post-commit operation and restart from the same checkpoint.", failure.exception)) + case Failure(checkError) => + // An unknown commit outcome is not permission to write the batch a second time. + failure.exception.addSuppressed(checkError) + failure + case _ => failure + } + } + } + } + + private def isBatchCommitted(batchId: Long): Boolean = { + if (metaClient.isEmpty) { + // The first batch may have created and committed the table before the writer threw. + metaClient = try { + Some(HoodieTableMetaClient.builder() + .setConf(HadoopFSUtils.getStorageConfWithCopy(sqlContext.sparkContext.hadoopConfiguration)) + .setBasePath(tablePath.get).build()) + } catch { + case _: TableNotFoundException => None + } + } + metaClient.exists { client => + client.reloadActiveTimeline() + canSkipBatch(batchId, options.getOrElse(OPERATION.key, UPSERT_OPERATION_OPT_VAL)) } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala index 9f43a314efaa..9d65aa621184 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala @@ -19,6 +19,8 @@ package org.apache.hudi.functional import org.apache.hudi.{DataSourceWriteOptions, HoodieStreamingSink} import org.apache.hudi.common.testutils.HoodieTestUtils +import org.apache.hudi.exception.HoodieException +import org.apache.hudi.sync.common.HoodieSyncTool import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.spark.sql.{DataFrame, SQLContext} @@ -30,7 +32,9 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.CsvSource import java.net.{InetSocketAddress, Socket} +import java.util.Properties import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit} +import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConverters._ @@ -71,7 +75,82 @@ class RecoveryTestSink(context: SQLContext, options: Map[String, String], partit } } +/** The real writer invokes this only after publishing its data commit. */ +class FailingStreamingSyncTool(properties: Properties) extends HoodieSyncTool(properties) { + override def syncHoodieTable(): Unit = { + FailingStreamingSyncTool.attempts.incrementAndGet() + throw new HoodieException("injected post-commit meta sync failure") + } +} + +object FailingStreamingSyncTool { + val attempts = new AtomicInteger() +} + class TestStreamingSinkRecovery extends HoodieSparkClientTestBase { + @ParameterizedTest + @CsvSource(Array("COPY_ON_WRITE,false", "MERGE_ON_READ,false", "COPY_ON_WRITE,true", "MERGE_ON_READ,true")) + def testPostCommitFailureDoesNotRetryWrite(tableType: String, existingTable: Boolean): Unit = { + val spark = sqlContext.sparkSession + val source = s"$basePath/source" + val target = s"$basePath/target" + val checkpoint = s"$basePath/checkpoint" + val input = spark.sql("select 1 as id, 10L as ts, 'first' as value") + val options = Map( + "hoodie.table.name" -> "post_commit_recovery", + "hoodie.datasource.write.table.type" -> tableType, + "hoodie.datasource.write.recordkey.field" -> "id", + "hoodie.datasource.write.precombine.field" -> "ts", + "hoodie.datasource.write.partitionpath.field" -> "", + "hoodie.datasource.write.operation" -> "insert", + "hoodie.metadata.enable" -> "false", + "hoodie.insert.shuffle.parallelism" -> "1", + DataSourceWriteOptions.STREAMING_DISABLE_COMPACTION.key -> "true", + DataSourceWriteOptions.STREAMING_RETRY_CNT.key -> "3", + DataSourceWriteOptions.STREAMING_RETRY_INTERVAL_MS.key -> "1", + "checkpointLocation" -> checkpoint) + val seed = spark.sql("select 0 as id, 0L as ts, 'seed' as value") + if (existingTable) { + seed.write.format("hudi").options(options).save(target) + } + input.coalesce(1).write.json(source) + FailingStreamingSyncTool.attempts.set(0) + val query = spark.readStream.schema(input.schema).json(source).writeStream.format("hudi") + .options(options ++ Map("hoodie.datasource.meta.sync.enable" -> "true", + DataSourceWriteOptions.META_SYNC_CLIENT_TOOL_CLASS_NAME.key -> classOf[FailingStreamingSyncTool].getName)) + .trigger(Trigger.AvailableNow()).start(target) + try { + val error = assertThrows(classOf[StreamingQueryException], () => query.awaitTermination(60000)) + val causes = Iterator.iterate[Throwable](error)(_.getCause).takeWhile(_ != null) + .map(_.getMessage).mkString("\n") + assertTrue(causes.contains("already committed"), causes) + assertTrue(causes.contains("injected post-commit meta sync failure"), causes) + assertEquals(1, FailingStreamingSyncTool.attempts.get(), "Post-commit failure retried the data write") + assertFalse(new java.io.File(s"$checkpoint/commits/0").exists()) + } finally { + query.stop() + } + def commits: Seq[String] = HoodieTestUtils.createMetaClient(storage, target).getActiveTimeline + .getCommitsTimeline.filterCompletedInstants().getInstants.asScala.map(_.requestedTime()).toSeq + val before = commits + assertEquals(if (existingTable) 2 else 1, before.size) + val expected = if (existingTable) seed.union(input) else input + def checkRows(): Unit = assertEquals(expected.orderBy("id").collect().toSeq, + spark.read.format("hudi").load(target).select("id", "ts", "value").orderBy("id").collect().toSeq) + checkRows() + // The failed sync is external to the data transaction; disable the injected tool after repair. + val replay = spark.readStream.schema(input.schema).json(source).writeStream.format("hudi") + .options(options).trigger(Trigger.AvailableNow()).start(target) + try { + assertTrue(replay.awaitTermination(60000)) + } finally { + replay.stop() + } + assertEquals(before, commits, "Restart recommitted a completed Hudi batch") + assertTrue(new java.io.File(s"$checkpoint/commits/0").exists()) + checkRows() + } + // This covers Spark checkpoint replay, not failures inside the sink retry loop (tracked by #20406). @ParameterizedTest @CsvSource(Array("available-now,COPY_ON_WRITE", "stop,COPY_ON_WRITE", "failure,COPY_ON_WRITE",
