mbutrovich commented on code in PR #27399:
URL: https://github.com/apache/datafusion/pull/27399#discussion_r4591122564
##########
datafusion/physical-plan/src/joins/hash_join/exec.rs:
##########
@@ -1749,72 +1805,53 @@ impl ExecutionPlan for HashJoinExec {
.flatten()
.flatten();
- let left_fut = match (&self.prepared_build, self.mode) {
- (Some(prepared), _) => {
- // Public join fields can change after builder validation.
- prepared.validate(self)?;
- let prepared = Arc::clone(prepared);
- self.left_fut.try_once(|| {
- Ok(async move { Ok(prepared.probe_data(right_partitions))
})
- })?
- }
- (None, PartitionMode::CollectLeft) => self.left_fut.try_once(|| {
- let left_stream = self.left.execute(0, Arc::clone(&context))?;
-
- let reservation =
-
MemoryConsumer::new("HashJoinInput").register(context.memory_pool());
-
- Ok(collect_left_input(
- self.random_state.random_state().clone(),
- left_stream,
- on_left.clone(),
- join_metrics.clone(),
- reservation,
- need_produce_result_in_final(self.join_type),
- right_partitions,
- enable_dynamic_filter_pushdown,
- Arc::clone(context.session_config().options()),
- self.null_equality,
- not_in,
- array_map_created_count,
- BuildMode::Ordinary,
- ))
- })?,
- (None, PartitionMode::Partitioned) => {
- let left_stream = self.left.execute(partition,
Arc::clone(&context))?;
-
- let reservation =
- MemoryConsumer::new(format!("HashJoinInput[{partition}]"))
- .register(context.memory_pool());
- OnceFut::new(collect_left_input(
- self.random_state.random_state().clone(),
- left_stream,
- on_left.clone(),
- join_metrics.clone(),
- reservation,
- need_produce_result_in_final(self.join_type),
- 1,
- enable_dynamic_filter_pushdown,
- Arc::clone(context.session_config().options()),
- self.null_equality,
- not_in,
- array_map_created_count,
- BuildMode::Ordinary,
- ))
- }
- (None, PartitionMode::Auto) => {
- return plan_err!(
- "Invalid HashJoinExec, unsupported PartitionMode {:?} in
execute()",
- PartitionMode::Auto
- );
- }
+ // See the comments on `enable_migration_hash_join` for details.
+ let use_regular = if let Some(prepared) = &self.prepared_build {
+ // Public join fields can change after builder validation.
+ prepared.validate(self)?;
+ prepared.is_regular()
+ } else {
+ context
+ .session_config()
+ .options()
+ .execution
+ .enable_migration_hash_join
+ };
Review Comment:
> Today a long-lived prepared build keeps whichever path was set at
preparation.
This choice is also written into public docs. The
[`PreparedHashJoinBuild`](https://github.com/apache/datafusion/blob/b93c3dda9ea46c4aa9c06f81a16be29ac765ae5a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs#L22-L23)
docs and the
[`prepare_build`](https://github.com/apache/datafusion/blob/b93c3dda9ea46c4aa9c06f81a16be29ac765ae5a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs#L121-L124)
rustdoc now say that preparation picks the executor for every consumer,
whatever the consumer's `enable_migration_hash_join` is. That makes a temporary
option part of the contract of a public type and method, and both paragraphs
have to change again when the option is removed. Reading the flag in `execute`
keeps the option out of the prepared-build API, so I'd make this change in this
PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]