jychen7 commented on a change in pull request #2099:
URL: https://github.com/apache/arrow-datafusion/pull/2099#discussion_r835851028
##########
File path: datafusion/src/execution/options.rs
##########
@@ -115,32 +128,81 @@ impl<'a> CsvReadOptions<'a> {
collect_stat: false,
file_extension: self.file_extension.to_owned(),
target_partitions,
+ table_partition_cols: self.table_partition_cols.clone(),
+ }
+ }
+}
+
+/// Parquet read options
+#[derive(Clone)]
+pub struct ParquetReadOptions<'a> {
+ /// File extension; only files with this extension are selected for data
input.
+ /// Defaults to ".parquet".
+ pub file_extension: &'a str,
+ // Partition Columns
+ pub table_partition_cols: Vec<String>,
+}
+
+impl<'a> Default for ParquetReadOptions<'a> {
+ fn default() -> Self {
+ Self {
+ file_extension: DEFAULT_PARQUET_EXTENSION,
table_partition_cols: vec![],
}
}
}
+impl<'a> ParquetReadOptions<'a> {
+ /// Specify table_partition_cols for partition pruning
+ pub fn table_partition_cols(mut self, table_partition_cols: Vec<String>)
-> Self {
+ self.table_partition_cols = table_partition_cols;
+ self
+ }
+
+ /// Helper to convert these user facing options to `ListingTable` options
+ pub fn to_listing_options(&self, target_partitions: usize) ->
ListingOptions {
+ let file_format = ParquetFormat::default();
+
+ ListingOptions {
+ format: Arc::new(file_format),
+ collect_stat: true,
Review comment:
this `true` align with the old code in
`datafusion/src/logical_plan/builder.rs`
https://github.com/apache/arrow-datafusion/blob/81592947e8814327ebdbd1fbc3d4a090796e37a3/datafusion/src/logical_plan/builder.rs#L284-L290
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
