2010YOUY01 commented on code in PR #23657:
URL: https://github.com/apache/datafusion/pull/23657#discussion_r3637354976
##########
datafusion/physical-plan/src/aggregates/ordered_final_stream.rs:
##########
@@ -157,21 +409,86 @@ impl OrderedFinalAggregateStream {
if let Err(e) = result {
return ControlFlow::Break((
Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::ReadingInput { table },
+ OrderedFinalAggregateState::ReadingInput {
+ table,
+ spill_context,
+ },
));
}
+ // Check memory reservation, and potentially spill.
let timer = elapsed_compute.timer();
- let result = table.next_output_batch();
+ let resize_result =
+ self.reservation
+ .try_resize(Self::reservation_size_for_table(
+ &table,
+ spill_context.as_deref(),
+ ));
timer.done();
+ match resize_result {
+ Ok(()) => {}
+ Err(e @ DataFusionError::ResourcesExhausted(_)) => {
+ let Some(spill_context) = spill_context else {
+ return ControlFlow::Break((
+ Poll::Ready(Some(Err(e))),
+ OrderedFinalAggregateState::Done,
+ ));
+ };
+ if table.is_empty() {
+ return ControlFlow::Break((
+ Poll::Ready(Some(Err(e))),
+ OrderedFinalAggregateState::Done,
+ ));
+ }
+ return ControlFlow::Continue(
+ OrderedFinalAggregateState::Spilling {
+ table,
+ spill_context,
+ },
+ );
+ }
+ Err(e) => {
+ return ControlFlow::Break((
+ Poll::Ready(Some(Err(e))),
+ OrderedFinalAggregateState::Done,
+ ));
+ }
+ }
+
+ let result = if spill_context
+ .as_ref()
+ .is_some_and(|spill_context| spill_context.has_spills())
+ {
+ // Once one incomplete run is spilled, every remaining
state
+ // must participate in replay so no group is finalized
twice.
+ Ok(None)
+ } else {
+ let timer = elapsed_compute.timer();
+ let result = table.next_output_batch();
+ timer.done();
+ result
+ };
match result {
// Some finalized groups can be emitted. Yield them, then
// continue aggregating input in the current state.
Ok(Some(batch)) => {
- let next_state =
- OrderedFinalAggregateState::ReadingInput { table };
- self.resize_reservation_for_state(&next_state);
+ if let Err(e) =
Review Comment:
Tracked in the clean-up items in
https://github.com/apache/datafusion/issues/22710
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
