This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 1b87239a9 improve logs (#1482)
1b87239a9 is described below
commit 1b87239a9e5d1d92093127839839181720667893
Author: Marko Milenković <[email protected]>
AuthorDate: Sun Mar 1 14:11:33 2026 +0000
improve logs (#1482)
---
.../core/src/execution_plans/sort_shuffle/writer.rs | 19 ++++++++++++++++++-
ballista/executor/src/execution_engine.rs | 2 +-
ballista/executor/src/metrics/mod.rs | 2 +-
ballista/scheduler/src/display.rs | 2 +-
ballista/scheduler/src/state/task_manager.rs | 2 +-
5 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs
b/ballista/core/src/execution_plans/sort_shuffle/writer.rs
index 84c7f61ec..91fb34742 100644
--- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs
+++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs
@@ -53,7 +53,7 @@ use datafusion::physical_plan::repartition::BatchPartitioner;
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
- SendableRecordBatchStream, Statistics,
+ SendableRecordBatchStream, Statistics, displayable,
};
use futures::{StreamExt, TryFutureExt, TryStreamExt};
use log::{debug, info};
@@ -634,6 +634,23 @@ fn result_schema() -> SchemaRef {
]))
}
+impl std::fmt::Display for SortShuffleWriterExec {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ let printable_plan = displayable(self.plan.as_ref())
+ .set_show_statistics(true)
+ .indent(false);
+ write!(
+ f,
+ "SortShuffleWriterExec: job={} stage={} work_dir={}
partitioning={:?} plan: \n {}",
+ self.job_id,
+ self.stage_id,
+ self.work_dir,
+ self.shuffle_output_partitioning,
+ printable_plan
+ )
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/ballista/executor/src/execution_engine.rs
b/ballista/executor/src/execution_engine.rs
index 2417e1d8a..1f3b91a7e 100644
--- a/ballista/executor/src/execution_engine.rs
+++ b/ballista/executor/src/execution_engine.rs
@@ -179,7 +179,7 @@ impl Display for DefaultQueryStageExec {
.collect();
write!(
f,
- "DefaultQueryStageExec(Sort): ({})\n{:?}",
+ "DefaultQueryStageExec(Sort): ({})\n{}",
stage_metrics.join(", "),
writer
)
diff --git a/ballista/executor/src/metrics/mod.rs
b/ballista/executor/src/metrics/mod.rs
index acc2cceed..506d8f8d1 100644
--- a/ballista/executor/src/metrics/mod.rs
+++ b/ballista/executor/src/metrics/mod.rs
@@ -49,7 +49,7 @@ impl ExecutorMetricsCollector for LoggingMetricsCollector {
plan: Arc<dyn QueryStageExecutor>,
) {
info!(
- "=== [{job_id}/{stage_id}/{partition}] Physical plan with metrics
===\n{plan}\n"
+ "\n=== [{job_id}/{stage_id}/{partition}] Physical plan with
metrics ===\n{plan}\n"
);
}
}
diff --git a/ballista/scheduler/src/display.rs
b/ballista/scheduler/src/display.rs
index 692aa9a28..59dc2f475 100644
--- a/ballista/scheduler/src/display.rs
+++ b/ballista/scheduler/src/display.rs
@@ -48,7 +48,7 @@ pub fn print_stage_metrics(
);
info!(
- "=== [{}/{}] Stage finished, physical plan with metrics ===\n{}\n",
+ "\n=== [{}/{}] Stage finished, physical plan with metrics ===\n{}",
job_id,
stage_id,
DisplayableBallistaExecutionPlan::new(plan, &plan_metrics).indent()
diff --git a/ballista/scheduler/src/state/task_manager.rs
b/ballista/scheduler/src/state/task_manager.rs
index d89215b97..ae3686901 100644
--- a/ballista/scheduler/src/state/task_manager.rs
+++ b/ballista/scheduler/src/state/task_manager.rs
@@ -305,7 +305,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static +
AsExecutionPlan> TaskManager<T, U>
)?) as ExecutionGraphBox
};
- info!("Submitting execution graph: {graph:?}");
+ info!("Submitting execution graph:\n\n{graph:?}");
self.state.submit_job(job_id.to_string(), &graph).await?;
graph.revive();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
