This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 4884764f5 [GLUTEN-6930][VL] Print memory statistics during task ending
when leak is found
4884764f5 is described below
commit 4884764f5c024e3d756b40681a57f80e362acae7
Author: Hongze Zhang <[email protected]>
AuthorDate: Wed Aug 21 15:10:09 2024 +0800
[GLUTEN-6930][VL] Print memory statistics during task ending when leak is
found
---
.../scala/org/apache/gluten/runtime/Runtime.scala | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
index 8edceea44..7ba72b379 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala
@@ -97,16 +97,17 @@ object Runtime {
throw new GlutenException(
s"Runtime instance already released: $handle, ${resourceName()},
${priority()}")
}
+
+ def dump(): KnownNameAndStats = {
+ new KnownNameAndStats() {
+ override def name: String = resourceName()
+ override def stats: MemoryUsageStats = collectMemoryUsage()
+ }
+ }
+
if (LOGGER.isDebugEnabled) {
LOGGER.debug(
- SparkMemoryUtil.prettyPrintStats(
- "About to release memory manager, usage dump:",
- new KnownNameAndStats() {
- override def name: String = resourceName()
-
- override def stats: MemoryUsageStats = collectMemoryUsage()
- }
- ))
+ SparkMemoryUtil.prettyPrintStats("About to release memory manager,
usage dump:", dump()))
}
RuntimeJniWrapper.releaseRuntime(handle)
@@ -115,10 +116,11 @@ object Runtime {
LOGGER.warn(
String.format(
"%s Reservation listener %s still reserved non-zero bytes, which
may cause memory" +
- " leak, size: %s. ",
+ " leak, size: %s, dump: %s ",
name,
rl.toString,
- SparkMemoryUtil.bytesToString(rl.getUsedBytes)
+ SparkMemoryUtil.bytesToString(rl.getUsedBytes),
+ dump()
))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
