andygrove commented on code in PR #2467:
URL: https://github.com/apache/datafusion-comet/pull/2467#discussion_r2395526166
##########
native/core/src/execution/memory_pools/unified_pool.rs:
##########
@@ -96,13 +104,20 @@ impl MemoryPool for CometUnifiedMemoryPool {
}
fn shrink(&self, _: &MemoryReservation, size: usize) {
- self.release_to_spark(size)
- .unwrap_or_else(|_| panic!("Failed to release {size} bytes"));
+ if let Err(e) = self.release_to_spark(size) {
+ panic!(
+ "Task {} failed to return {size} bytes to Spark: {e:?}",
Review Comment:
I was following Spark's convention for logging so that we can use grep to
search for `Task 1234` and view the combined logging from both Spark and Comet.
```
~/git/apache/apache-spark-3.5.6$ find . -name "*.scala" -exec grep
taskAttemptId {} \; | grep log
logDebug(s"Starting pushing blocks for the task
${context.taskAttemptId()}")
logWarning(s"Task ${taskAttemptId.get} already completed, not
releasing lock for $blockId")
logTrace(s"Task $taskAttemptId trying to acquire read lock for $blockId")
logTrace(s"Task $taskAttemptId acquired read lock for $blockId")
logTrace(s"Task $taskAttemptId trying to acquire write lock for
$blockId")
logTrace(s"Task $taskAttemptId acquired write lock for $blockId")
logTrace(s"Task $taskAttemptId downgrading write lock for $blockId")
logTrace(s"Task $taskAttemptId releasing lock for $blockId")
logTrace(s"Task $taskAttemptId trying to remove block $blockId")
logInfo(s"Task ${TaskContext.get().taskAttemptId} force spilling
in-memory map to disk " +
logInfo(s"Task ${context.taskAttemptId} force spilling in-memory map
to disk and " +
logInfo(s"TID $taskAttemptId waiting for at least 1/2N of $poolName
pool to be free")
logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt
$stageAttemptNumber) has entered " +
logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt
$stageAttemptNumber) waiting " +
logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt
$stageAttemptNumber) finished " +
logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt
$stageAttemptNumber) failed " +
``
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
