nsivabalan commented on code in PR #20677:
URL: https://github.com/apache/hudi/pull/20677#discussion_r4623433906
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala:
##########
@@ -226,16 +226,49 @@ class HoodieStreamingSink(sqlContext: SQLContext,
override def toString: String = s"HoodieStreamingSink[${options("path")}]"
@annotation.tailrec
- private def retry[T](n: Int, waitInMillis: Long)(fn: => Try[T]): Try[T] = {
+ private def retry[T](batchId: Long, n: Int, waitInMillis: Long)(fn: =>
Try[T]): Try[T] = {
fn match {
case x: Success[T] =>
x
- case _ if n > 1 =>
- Thread.sleep(waitInMillis)
- retry(n - 1, waitInMillis * 2)(fn)
- case f =>
- reset(false)
- f
+ case failure: Failure[T] =>
+ // A writer can throw during clean/archive/meta sync after publishing
the commit. Never
+ // retry the write in that case. Fail the query so the post-commit
failure remains visible;
+ // restarting from the same Spark checkpoint deduplicates the already
committed batch.
+ val committed = Try(isBatchCommitted(batchId))
+ committed match {
+ case Success(false) if n > 1 =>
+ Thread.sleep(waitInMillis)
+ retry(batchId, n - 1, waitInMillis * 2)(fn)
+ case _ =>
+ reset(false)
+ committed match {
+ case Success(true) => Failure(new HoodieException(
+ s"Micro batch id=$batchId is already committed; refusing to
retry the write after a post-commit failure. "
+ + "Resolve the failed post-commit operation and restart from
the same checkpoint.", failure.exception))
+ case Failure(checkError) =>
Review Comment:
Agree that an unknown outcome should stop. This branch has no test, though.
Could we add one where the timeline reload throws, asserting the write `fn` ran
once and the check error is attached as suppressed?
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSinkRecovery.scala:
##########
@@ -71,7 +75,82 @@ class RecoveryTestSink(context: SQLContext, options:
Map[String, String], partit
}
}
+/** The real writer invokes this only after publishing its data commit. */
+class FailingStreamingSyncTool(properties: Properties) extends
HoodieSyncTool(properties) {
+ override def syncHoodieTable(): Unit = {
+ FailingStreamingSyncTool.attempts.incrementAndGet()
+ throw new HoodieException("injected post-commit meta sync failure")
+ }
+}
+
+object FailingStreamingSyncTool {
+ val attempts = new AtomicInteger()
+}
+
class TestStreamingSinkRecovery extends HoodieSparkClientTestBase {
+ @ParameterizedTest
+ @CsvSource(Array("COPY_ON_WRITE,false", "MERGE_ON_READ,false",
"COPY_ON_WRITE,true", "MERGE_ON_READ,true"))
Review Comment:
Nice test: a real sync tool, an attempt count, exact rows, and the Spark
commit-log check. Two gaps:
1. Meta sync runs outside `client.commit`. A failure inside it after
completion (inline clean/archive) is the more interesting boundary.
2. `STREAMING_DISABLE_COMPACTION=true` hides the MOR async-compaction
interaction: a requested compaction is left un-enqueued when sync fails.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]