This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 7d54e1410043 test(spark): verify sink cleanup and commit checkpoint
recovery (#20495)
7d54e1410043 is described below
commit 7d54e14100435714b97b68a56191c6e27225d6f2
Author: Danny Chan <[email protected]>
AuthorDate: Wed Nov 4 13:55:15 2026 +0800
test(spark): verify sink cleanup and commit checkpoint recovery (#20495)
* test(spark): verify sink cleanup and commit checkpoint recovery
* test(spark): verify isolated Hudi sink cleanup for COW and MOR
---
.../org/apache/hudi/HoodieStreamingSink.scala | 4 +
.../functional/TestStreamingSinkRecovery.scala | 171 +++++++++++++++++++++
2 files changed, 175 insertions(+)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
index 4f846ef48bb8..0084ea10a416 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
@@ -219,6 +219,10 @@ class HoodieStreamingSink(sqlContext: SQLContext,
}
}
+ private[hudi] def currentWriteClient: Option[SparkRDDWriteClient[_]] =
writeClient
+
+ private[hudi] def currentTimelineService: Option[EmbeddedTimelineService] =
embeddedTimelineService
+
override def toString: String = s"HoodieStreamingSink[${options("path")}]"
@annotation.tailrec
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala
new file mode 100644
index 000000000000..9f43a314efaa
--- /dev/null
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, HoodieStreamingSink}
+import org.apache.hudi.common.testutils.HoodieTestUtils
+import org.apache.hudi.testutils.HoodieSparkClientTestBase
+
+import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.execution.streaming.Sink
+import org.apache.spark.sql.sources.StreamSinkProvider
+import org.apache.spark.sql.streaming.{OutputMode, StreamingQueryException,
Trigger}
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse,
assertThrows, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.CsvSource
+
+import java.net.{InetSocketAddress, Socket}
+import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit}
+
+import scala.collection.JavaConverters._
+
+/** Injects failure after Hudi commits but before addBatch returns to Spark's
checkpoint writer. */
+class RecoveryTestSource extends StreamSinkProvider {
+ override def createSink(context: SQLContext, options: Map[String, String],
partitions: Seq[String],
+ mode: OutputMode): Sink = {
+ val sink = new RecoveryTestSink(context, options, partitions, mode)
+ RecoveryTestSource.sinks.put(options("path"), sink)
+ sink
+ }
+}
+
+object RecoveryTestSource {
+ val sinks = new ConcurrentHashMap[String, RecoveryTestSink]()
+}
+
+class RecoveryTestSink(context: SQLContext, options: Map[String, String],
partitions: Seq[String], mode: OutputMode)
+ extends HoodieStreamingSink(context, options, partitions, mode) {
+ val released = new CountDownLatch(1)
+ @volatile var serverPort = -1
+ @volatile var resourcesCleared = false
+
+ override def addBatch(batchId: Long, data: DataFrame): Unit = {
+ super.addBatch(batchId, data)
+ val client = currentWriteClient.get
+ val service = currentTimelineService.get
+ serverPort =
service.getRemoteFileSystemViewConfig(client.getConfig).getRemoteViewServerPort
+ if (options.getOrElse("test.fail.after.commit", "false").toBoolean) {
+ throw new IllegalStateException("injected failure after Hudi commit
before Spark checkpoint")
+ }
+ }
+
+ override protected def reset(force: Boolean): Unit = {
+ super.reset(force)
+ resourcesCleared = currentWriteClient.isEmpty &&
currentTimelineService.isEmpty
+ released.countDown()
+ }
+}
+
+class TestStreamingSinkRecovery extends HoodieSparkClientTestBase {
+ // This covers Spark checkpoint replay, not failures inside the sink retry
loop (tracked by #20406).
+ @ParameterizedTest
+ @CsvSource(Array("available-now,COPY_ON_WRITE", "stop,COPY_ON_WRITE",
"failure,COPY_ON_WRITE",
+ "available-now,MERGE_ON_READ", "stop,MERGE_ON_READ",
"failure,MERGE_ON_READ"))
+ def testTerminalCleanupAndCommittedBatchReplay(termination: String,
tableType: String): Unit = {
+ val spark = sqlContext.sparkSession
+ val source = s"$basePath/source"
+ val target = s"$basePath/target"
+ val checkpoint = s"$basePath/checkpoint"
+ val input = spark.sql("select 1 as id, 10L as ts, 'first' as value")
+ input.coalesce(1).write.json(source)
+ val options = Map(
+ "hoodie.table.name" -> "recovery",
+ "hoodie.datasource.write.table.type" -> tableType,
+ "hoodie.datasource.write.recordkey.field" -> "id",
+ "hoodie.datasource.write.precombine.field" -> "ts",
+ "hoodie.datasource.write.partitionpath.field" -> "",
+ "hoodie.datasource.write.operation" -> "insert",
+ "hoodie.metadata.enable" -> "false",
+ "hoodie.embed.timeline.server" -> "true",
+ "hoodie.embed.timeline.server.reuse.enabled" -> "false",
+ "hoodie.insert.shuffle.parallelism" -> "1",
+ DataSourceWriteOptions.STREAMING_RETRY_CNT.key -> "1",
+ "checkpointLocation" -> checkpoint,
+ "test.fail.after.commit" -> (termination == "failure").toString)
+ val unrelatedTarget = s"$basePath/unrelated"
+ val unrelatedSource = s"$basePath/unrelated-source"
+ input.coalesce(1).write.json(unrelatedSource)
+ val unrelated =
spark.readStream.schema(input.schema).json(unrelatedSource).writeStream
+ .format(classOf[RecoveryTestSource].getName)
+ .options(options ++ Map("hoodie.table.name" -> "unrelated",
"checkpointLocation" -> s"$basePath/unrelated-checkpoint",
+ "test.fail.after.commit" -> "false"))
+ .start(unrelatedTarget)
+ unrelated.processAllAvailable()
+ val unrelatedSink = RecoveryTestSource.sinks.get(unrelatedTarget)
+ val query = spark.readStream.schema(input.schema).json(source).writeStream
+ .format(classOf[RecoveryTestSource].getName).options(options)
+ .trigger(if (termination == "stop") Trigger.ProcessingTime(100) else
Trigger.AvailableNow())
+ .start(target)
+ try {
+ if (termination == "failure") {
+ val error = assertThrows(classOf[StreamingQueryException], () =>
query.awaitTermination(60000))
+ assertTrue(error.getMessage.contains("injected failure after Hudi
commit"))
+ assertFalse(new java.io.File(s"$checkpoint/commits/0").exists())
+ } else if (termination == "stop") {
+ query.processAllAvailable()
+ query.stop()
+ } else {
+ assertTrue(query.awaitTermination(60000))
+ }
+ val sink = RecoveryTestSource.sinks.get(target)
+ assertTrue(sink.released.await(30, TimeUnit.SECONDS), "Sink cleanup did
not complete")
+ assertTrue(sink.resourcesCleared, "Sink still retains its client or
timeline service")
+ assertTrue(sink.serverPort > 0)
+ val socket = new Socket()
+ try {
+ assertThrows(classOf[java.io.IOException], () => socket.connect(new
InetSocketAddress("127.0.0.1", sink.serverPort), 1000))
+ } finally {
+ socket.close()
+ }
+ val nextInput = spark.sql("select 2 as id, 20L as ts, 'second' as value")
+ nextInput.coalesce(1).write.mode("append").json(unrelatedSource)
+ unrelated.processAllAvailable()
+ assertTrue(unrelated.isActive)
+ assertEquals(1L, unrelatedSink.released.getCount, "Another query
released this sink")
+ assertFalse(unrelatedSink.resourcesCleared)
+ val unrelatedSocket = new Socket()
+ try {
+ unrelatedSocket.connect(new InetSocketAddress("127.0.0.1",
unrelatedSink.serverPort), 1000)
+ } finally {
+ unrelatedSocket.close()
+ }
+ assertEquals(input.union(nextInput).orderBy("id").collect().toSeq,
+ spark.read.format("hudi").load(unrelatedTarget).select("id", "ts",
"value").orderBy("id").collect().toSeq)
+
+ def commits: Seq[String] = HoodieTestUtils.createMetaClient(storage,
target).getActiveTimeline
+
.getCommitsTimeline.filterCompletedInstants().getInstants.asScala.map(_.requestedTime()).toSeq
+ val before = commits
+ assertEquals(1, before.size)
+ val replay =
spark.readStream.schema(input.schema).json(source).writeStream.format("hudi")
+ .options(options -
"test.fail.after.commit").trigger(Trigger.AvailableNow()).start(target)
+ try {
+ assertTrue(replay.awaitTermination(60000))
+ } finally {
+ replay.stop()
+ }
+ assertEquals(before, commits, "Restart recommitted a completed Hudi
batch")
+ assertEquals(input.collect().toSeq,
spark.read.format("hudi").load(target).select("id", "ts",
"value").collect().toSeq)
+ assertTrue(new java.io.File(s"$checkpoint/commits/0").exists())
+ } finally {
+ query.stop()
+ unrelated.stop()
+ RecoveryTestSource.sinks.remove(target)
+ RecoveryTestSource.sinks.remove(unrelatedTarget)
+ }
+ }
+}