liujiayi771 commented on code in PR #47996:
URL: https://github.com/apache/spark/pull/47996#discussion_r1745631535
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVTable.scala:
##########
@@ -37,8 +37,13 @@ case class CSVTable(
userSpecifiedSchema: Option[StructType],
fallbackFileFormat: Class[_ <: FileFormat])
extends FileTable(sparkSession, options, paths, userSpecifiedSchema) {
- override def newScanBuilder(options: CaseInsensitiveStringMap):
CSVScanBuilder =
- CSVScanBuilder(sparkSession, fileIndex, schema, dataSchema, options)
+ override def newScanBuilder(options: CaseInsensitiveStringMap):
CSVScanBuilder = {
+ val finalOptions = this.options.asCaseSensitiveMap().asScala.filterNot {
+ case (k, _) => options.containsKey(k)
+ }.toMap ++ options.asScala.toMap
+ CSVScanBuilder(sparkSession, fileIndex, schema, dataSchema,
+ new CaseInsensitiveStringMap(finalOptions.asJava))
Review Comment:
@cloud-fan orc/parquet/json/textTable do not combine the options of table
and relation, while jdbcTable does merge them. I think we can add merging logic
for all of them. How do you think?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
