Github user MaxGekk commented on a diff in the pull request:
https://github.com/apache/spark/pull/21657#discussion_r198865492
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
---
@@ -1579,4 +1579,16 @@ class CSVSuite extends QueryTest with
SharedSQLContext with SQLTestUtils with Te
}
}
}
+
+ test("SPARK-24676 project required data from parsed data when
columnPruning disabled") {
+ withSQLConf(SQLConf.CSV_PARSER_COLUMN_PRUNING.key -> "false") {
+ withTempPath { path =>
+ val dir = path.getAbsolutePath
+ spark.range(10).selectExpr("id % 2 AS p", "id AS c0", "id AS
c1").write.partitionBy("p")
+ .option("header", "true").csv(dir)
--- End diff --
"true" -> true ?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
