This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d6457b77c9 [VL] Fix wrong warning of "Memory overhead is set to ..."
under default Spark config settings (#8448)
d6457b77c9 is described below
commit d6457b77c9741a52560c4960afffd403dca900bf
Author: Hongze Zhang <[email protected]>
AuthorDate: Wed Jan 8 08:34:00 2025 +0800
[VL] Fix wrong warning of "Memory overhead is set to ..." under default
Spark config settings (#8448)
---
.../org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 45a30f8cf3..5d75521b84 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -67,10 +67,11 @@ class VeloxListenerApi extends ListenerApi with Logging {
ByteUnit.BYTE.toMiB(desiredOverheadSize).toString)
}
val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf)
- if (overheadSize < desiredOverheadSize) {
+ if (ByteUnit.BYTE.toMiB(overheadSize) <
ByteUnit.BYTE.toMiB(desiredOverheadSize)) {
logWarning(
- s"Memory overhead is set to $overheadSize which is smaller than the
recommended size" +
- s" $desiredOverheadSize. This may cause OOM.")
+ s"Memory overhead is set to ${ByteUnit.BYTE.toMiB(overheadSize)}MiB
which is smaller than" +
+ s" the recommended size
${ByteUnit.BYTE.toMiB(desiredOverheadSize)}MiB." +
+ s" This may cause OOM.")
}
conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY,
overheadSize.toString)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
