This is an automated email from the ASF dual-hosted git repository. viirya pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new 8f9aa50f730 [SPARK-34863][SQL][FOLLOWUP] Disable `spark.sql.parquet.enableNestedColumnVectorizedReader` by default 8f9aa50f730 is described below commit 8f9aa50f730a48c1697cbb4f4c9ba707963a37a1 Author: Chao Sun <sunc...@apple.com> AuthorDate: Sun Apr 3 18:35:10 2022 -0700 [SPARK-34863][SQL][FOLLOWUP] Disable `spark.sql.parquet.enableNestedColumnVectorizedReader` by default ### What changes were proposed in this pull request? This PR disables `spark.sql.parquet.enableNestedColumnVectorizedReader` by default. ### Why are the changes needed? In #34659 the config was turned mainly for testing reason. As the feature is new, we should turn it off by default. ### Does this PR introduce _any_ user-facing change? The config `spark.sql.parquet.enableNestedColumnVectorizedReader` is turned off by default now. ### How was this patch tested? Existing tests. Closes #36055 from sunchao/disable. Authored-by: Chao Sun <sunc...@apple.com> Signed-off-by: Liang-Chi Hsieh <vii...@gmail.com> (cherry picked from commit 1b08673a6d92e3e0fceb4a686a0ba77a87f1ebbc) Signed-off-by: Liang-Chi Hsieh <vii...@gmail.com> --- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out | 3 +-- sql/core/src/test/resources/sql-tests/results/explain.sql.out | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 5bf59923787..56849b8718e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1015,7 +1015,7 @@ object SQLConf { s"Requires ${PARQUET_VECTORIZED_READER_ENABLED.key} to be enabled.") .version("3.3.0") .booleanConf - .createWithDefault(true) + .createWithDefault(false) val PARQUET_RECORD_FILTER_ENABLED = buildConf("spark.sql.parquet.recordLevelFilter.enabled") .doc("If true, enables Parquet's native record-level filtering using the pushed down " + diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out index f98fb1eb2a5..f5e5b46d29c 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out @@ -1125,8 +1125,7 @@ struct<plan:string> -- !query output == Physical Plan == *Filter v#x IN ([a],null) -+- *ColumnarToRow - +- FileScan parquet default.t[v#x] Batched: true, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct<v:array<string>> ++- FileScan parquet default.t[v#x] Batched: false, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct<v:array<string>> -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out index a563eda1e7b..4e552d51a39 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out @@ -1067,8 +1067,7 @@ struct<plan:string> -- !query output == Physical Plan == *Filter v#x IN ([a],null) -+- *ColumnarToRow - +- FileScan parquet default.t[v#x] Batched: true, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct<v:array<string>> ++- FileScan parquet default.t[v#x] Batched: false, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct<v:array<string>> -- !query --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org