Github user dongjoon-hyun commented on a diff in the pull request:
https://github.com/apache/spark/pull/22529#discussion_r219714244
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
---
@@ -173,12 +173,16 @@ final class DataStreamReader
private[sql](sparkSession: SparkSession) extends Lo
}
ds match {
case s: MicroBatchReadSupport =>
+ val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
+ ds = s, conf = sparkSession.sessionState.conf)
+ val options = sessionOptions ++ extraOptions
+ val dataSourceOptions = new DataSourceOptions(options.asJava)
var tempReader: MicroBatchReader = null
val schema = try {
tempReader = s.createMicroBatchReader(
Optional.ofNullable(userSpecifiedSchema.orNull),
Utils.createTempDir(namePrefix =
s"temporaryReader").getCanonicalPath,
- options)
+ dataSourceOptions)
--- End diff --
So, this part is the difference, isn't it?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
