alamb commented on code in PR #3885:
URL: https://github.com/apache/arrow-datafusion/pull/3885#discussion_r1004760255
##########
datafusion/core/src/execution/options.rs:
##########
@@ -168,56 +170,31 @@ pub struct ParquetReadOptions<'a> {
pub file_extension: &'a str,
/// Partition Columns
pub table_partition_cols: Vec<String>,
- /// Should DataFusion parquet reader use the predicate to prune data,
- /// overridden by value on execution::context::SessionConfig
- // TODO move this into ConfigOptions
- pub parquet_pruning: bool,
- /// Tell the parquet reader to skip any metadata that may be in
- /// the file Schema. This can help avoid schema conflicts due to
- /// metadata. Defaults to true.
- // TODO move this into ConfigOptions
- pub skip_metadata: bool,
}
impl<'a> Default for ParquetReadOptions<'a> {
fn default() -> Self {
- let format_default = ParquetFormat::default();
-
Self {
file_extension: DEFAULT_PARQUET_EXTENSION,
table_partition_cols: vec![],
- parquet_pruning: format_default.enable_pruning(),
- skip_metadata: format_default.skip_metadata(),
}
}
}
impl<'a> ParquetReadOptions<'a> {
- /// Specify parquet_pruning
- pub fn parquet_pruning(mut self, parquet_pruning: bool) -> Self {
Review Comment:
I would say "kind of" -- previously you could configure the options on a
per-datasource basis but depending on exactly what codepath you used and what
other options were set your settings might or might not get overridden.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
