danny0405 commented on code in PR #20516:
URL: https://github.com/apache/hudi/pull/20516#discussion_r4533509810


##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStreamingSource.scala:
##########
@@ -651,6 +651,130 @@ class TestStreamingSource extends StreamTest {
     )
   }
 
+  Seq(COPY_ON_WRITE, MERGE_ON_READ).foreach { tableType =>
+    Seq(false, true).foreach { completionTime =>
+      test(s"guarded source rejects resets and recreation: $tableType 
completionTime=$completionTime") {
+        withTempDir { dir =>
+          val tablePath = s"${dir.getCanonicalPath}/guarded"
+          val conf = 
HadoopFSUtils.getStorageConf(spark.sessionState.newHadoopConf())
+          def initialize(): HoodieTableMetaClient = 
HoodieTableMetaClient.newTableBuilder()
+            
.setTableType(tableType).setTableName("guarded").setRecordKeyFields("id")
+            .setOrderingFields("ts").initTable(conf, tablePath)
+          val metaClient = initialize()
+          addData(tablePath, Seq(("1", "old", "10", "001")))
+          def source(enabled: Option[Boolean] = Some(true)): 
org.apache.spark.sql.execution.streaming.Source = {
+            val options = commonOptions + ("path" -> tablePath) ++ enabled.map 
{ value =>
+              DataSourceReadOptions.STREAMING_READ_FAIL_ON_DATA_LOSS.key() -> 
value.toString
+            }
+            val checkpoint = s"${dir.getCanonicalPath}/checkpoint"
+            if (completionTime) {
+              new 
org.apache.spark.sql.hudi.streaming.HoodieStreamSourceV2(spark.sqlContext,
+                metaClient, checkpoint, None, options,
+                
org.apache.spark.sql.hudi.streaming.HoodieEarliestOffsetRangeLimit,
+                HoodieTableVersion.current())
+            } else {
+              new 
org.apache.spark.sql.hudi.streaming.HoodieStreamSourceV1(spark.sqlContext,
+                metaClient, checkpoint, None, options,
+                
org.apache.spark.sql.hudi.streaming.HoodieEarliestOffsetRangeLimit,
+                HoodieTableVersion.current())
+            }
+          }
+          val reader = source()
+          val saved = reader.getOffset.get
+          addData(tablePath, Seq(("2", "new", "20", "002")), 
enableInlineCluster = true)
+          assert(reader.getBatch(Some(saved), 
reader.getOffset.get).isStreaming)
+          
assert(metaClient.reloadActiveTimeline().getCompletedReplaceTimeline.countInstants()
 > 0)
+          val beforeResetEnd = reader.getOffset.get
+          Seq(("3", "replacement", "30", "003")).toDF(columns: 
_*).write.format("hudi")
+            .options(commonOptions).option(TBL_NAME.key, "guarded")
+            .option("hoodie.datasource.write.operation", 
"insert_overwrite_table")
+            .mode(SaveMode.Append).save(tablePath)
+          val resetEnd = reader.getOffset.get
+          val resetError = 
intercept[org.apache.hudi.exception.HoodieException] {
+            reader.getBatch(Some(saved), resetEnd)
+          }
+          assert(resetError.getMessage.contains("rebuild required"))
+          assert(resetError.getMessage.contains("logical reset"))
+          val concurrentResetError = 
intercept[org.apache.hudi.exception.HoodieException] {
+            reader.getBatch(Some(saved), beforeResetEnd)
+          }
+          assert(concurrentResetError.getMessage.contains("logical reset"))
+          val sessionReader = withSQLConf(

Review Comment:
   The earlier fix assigns the source to a var inside withSQLConf, avoiding 
Spark 3.5's Unit return type. Revalidated with the November 11 changes in 
57883c3ccfe: Spark 3.5 / Scala 2.12 / JDK 17 reactor build and the full source 
suite passed (24 tests, 2 expected variant skips).



##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSourceV2.scala:
##########
@@ -51,6 +51,10 @@ class HoodieStreamSourceV2(sqlContext: SQLContext,
                            writeTableVersion: HoodieTableVersion)
   extends Source with Logging with Serializable with SparkAdapterSupport {
 
+  // Capture flow-scoped SQLConf once; explicit reader options take precedence.
+  private val recoveryStrategy = HoodieStreamingSourceSafety.strategy(
+    DataSourceOptionsHelper.collectSparkHoodieConfs(sqlContext, parameters))

Review Comment:
   Fixed in 57883c3ccfe: both sources now use collectHoodieAndSparkHoodieConfs. 
Tests exercise bare hoodie.* and spark.hoodie.* SQLConf, construct the source 
inside the scope, execute outside it, and verify explicit reader options still 
win.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to