This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2127adcc5 [GLUTEN-5611] [VL] Avoid trigger Spark memory listener when
native memory request can be handled internally (#5631)
2127adcc5 is described below
commit 2127adcc51c67d4f499434b8dc71ca4c744021e4
Author: Yang Zhang <[email protected]>
AuthorDate: Tue May 7 13:45:17 2024 +0800
[GLUTEN-5611] [VL] Avoid trigger Spark memory listener when native memory
request can be handled internally (#5631)
---
cpp/velox/memory/VeloxMemoryManager.cc | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/cpp/velox/memory/VeloxMemoryManager.cc
b/cpp/velox/memory/VeloxMemoryManager.cc
index c8d13b871..93eb93f6b 100644
--- a/cpp/velox/memory/VeloxMemoryManager.cc
+++ b/cpp/velox/memory/VeloxMemoryManager.cc
@@ -40,6 +40,9 @@ class ListenableArbitrator : public
velox::memory::MemoryArbitrator {
}
uint64_t growCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes)
override {
+ if (targetBytes == 0) {
+ return 0;
+ }
std::lock_guard<std::recursive_mutex> l(mutex_);
return growPoolLocked(pool, targetBytes);
}
@@ -66,8 +69,8 @@ class ListenableArbitrator : public
velox::memory::MemoryArbitrator {
uint64_t shrinkCapacity(
const std::vector<std::shared_ptr<velox::memory::MemoryPool>>& pools,
uint64_t targetBytes,
- bool allowSpill = true,
- bool allowAbort = false) override {
+ bool allowSpill,
+ bool allowAbort) override {
facebook::velox::exec::MemoryReclaimer::Stats status;
GLUTEN_CHECK(pools.size() == 1, "Should shrink a single pool at a time");
std::lock_guard<std::recursive_mutex> l(mutex_); // FIXME: Do we have
recursive locking for this mutex?
@@ -102,13 +105,14 @@ class ListenableArbitrator : public
velox::memory::MemoryArbitrator {
reserved,
"Unexpected: Failed to reserve " + std::to_string(bytes) +
" bytes although there is enough space, free bytes: " +
std::to_string(freeBytes));
+ return 0;
}
listener_->allocationChanged(bytes);
return pool->grow(bytes, bytes);
}
uint64_t releaseMemoryLocked(velox::memory::MemoryPool* pool, uint64_t
bytes) {
- uint64_t freeBytes = pool->shrink(0);
+ uint64_t freeBytes = pool->shrink(bytes);
listener_->allocationChanged(-freeBytes);
return freeBytes;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
