2010YOUY01 commented on code in PR #23657:
URL: https://github.com/apache/datafusion/pull/23657#discussion_r3637266690
##########
datafusion/physical-plan/src/aggregates/ordered_final_stream.rs:
##########
@@ -47,13 +69,46 @@ pub(crate) struct OrderedFinalAggregateStream {
state: Option<OrderedFinalAggregateState>,
}
+/// Spill configuration and accumulated runs for partially ordered final
+/// aggregation. Each file is one fully group-key-sorted intermediate-state
run;
+/// all runs are replayed together after the original input is exhausted.
+struct OrderedFinalSpillContext {
+ /// Aggregate configuration
+ agg: AggregateExec,
+ /// Task context
+ context: Arc<TaskContext>,
+ /// Original partition index
+ partition: usize,
+ /// Target batch size from configuration
+ batch_size: usize,
+ /// Full group-key ordering, such ordering with be kept in: a) individual
spill
+ /// files, b) order after final merging and streaming aggregate
+ spill_expr: LexOrdering,
+ /// Spill I/O and metrics manager.
+ spill_manager: SpillManager,
+ /// Fully sorted spill runs waiting to be merged.
+ spills: Vec<SortedSpillFile>,
+}
+
/// See comments at `poll_next()` for details.
enum OrderedFinalAggregateState {
ReadingInput {
table: OrderedAggregateTable<FinalMarker>,
+ spill_context: Option<Box<OrderedFinalSpillContext>>,
},
- DrainingFinal {
+ Spilling {
table: OrderedAggregateTable<FinalMarker>,
+ spill_context: Box<OrderedFinalSpillContext>,
+ },
+ ProducingOutput {
+ table: OrderedAggregateTable<FinalMarker>,
+ },
+ PreparingMergeInput {
Review Comment:
[1aeb794](https://github.com/apache/datafusion/pull/23657/commits/1aeb79459568098f7845452dbdbcf0107b63d9ef)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
