zhuqi-lucas commented on code in PR #19064:
URL: https://github.com/apache/datafusion/pull/19064#discussion_r2616810095
##########
datafusion/datasource/src/file_scan_config.rs:
##########
@@ -844,6 +845,114 @@ impl DataSource for FileScanConfig {
}
}
}
+
+ fn try_pushdown_sort(
+ &self,
+ order: &[PhysicalSortExpr],
+ ) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {
+ let current_ordering = match self.output_ordering.first() {
+ Some(ordering) => ordering.as_ref(),
+ None => return Ok(SortOrderPushdownResult::Unsupported),
+ };
+
+ // Only support reverse ordering pushdown until now
+ if !is_reverse_ordering(order, current_ordering) {
+ return Ok(SortOrderPushdownResult::Unsupported);
+ }
+
+ // Ask the file source if it can handle the sort pushdown
+ let pushdown_result = self.file_source.try_pushdown_sort(order)?;
+
+ // Extract the new file source and determine result type
+ let (new_file_source, is_exact) = match pushdown_result {
+ SortOrderPushdownResult::Exact { inner } => (inner, true),
+ SortOrderPushdownResult::Inexact { inner } => (inner, false),
+ SortOrderPushdownResult::Unsupported => {
+ return Ok(SortOrderPushdownResult::Unsupported);
+ }
+ };
+
+ let mut new_config = self.clone();
+
+ // Reverse file groups: when scanning in reverse, we need to read files
+ // in reverse order to maintain the correct global ordering
+ new_config.file_groups = new_config
+ .file_groups
+ .into_iter()
+ .map(|group| {
+ let mut files = group.into_inner();
+ files.reverse();
+ files.into()
+ })
+ .collect();
+
+ // Phase 1: Make output_ordering unknown since we're only reversing
row groups,
Review Comment:
Good point, i will add a helper function to build the reverse case when it's
exact and is_exact, and also when is_exact, we should remove output order.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
