Github user MaxGekk commented on a diff in the pull request:
https://github.com/apache/spark/pull/22676#discussion_r223730392
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVFileFormat.scala
---
@@ -139,14 +138,15 @@ class CSVFileFormat extends TextBasedFileFormat with
DataSourceRegister {
StructType(dataSchema.filterNot(_.name ==
parsedOptions.columnNameOfCorruptRecord)),
StructType(requiredSchema.filterNot(_.name ==
parsedOptions.columnNameOfCorruptRecord)),
parsedOptions)
+ val schema = if (columnPruning) requiredSchema else dataSchema
+ val headerChecker = new CSVHeaderChecker(
+ schema, parsedOptions, source = s"CSV file: ${file.filePath}",
file.start == 0)
--- End diff --
`isStartOfFile = file.start == 0`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
