This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e466daf6f4b [HUDI-3636] Disabling embedded timeline server with spark
streaming sink (#9266)
e466daf6f4b is described below
commit e466daf6f4b57d5e2069a534b195434cca0e852f
Author: Sivabalan Narayanan <n.siv...@gmail.com>
AuthorDate: Tue Jul 25 00:19:26 2023 -0400
[HUDI-3636] Disabling embedded timeline server with spark streaming sink
(#9266)
---
.../src/main/scala/org/apache/hudi/HoodieStreamingSink.scala | 5 +++++
1 file changed, 5 insertions(+)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
index 895e8fa5ab7..5667c8870d3 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
@@ -118,6 +118,11 @@ class HoodieStreamingSink(sqlContext: SQLContext,
// we need auto adjustment enabled for streaming sink since async table
services are feasible within the same JVM.
updatedOptions =
updatedOptions.updated(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key, "true")
updatedOptions =
updatedOptions.updated(HoodieSparkSqlWriter.SPARK_STREAMING_BATCH_ID,
batchId.toString)
+ if
(!options.containsKey(HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_ENABLE.key()))
{
+ // if user does not explicitly override, we are disabling timeline
server for streaming sink.
+ // refer to HUDI-3636 for more details
+ updatedOptions =
updatedOptions.updated(HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_ENABLE.key(),
" false")
+ }
retry(retryCnt, retryIntervalMs)(
Try(
