This is an automated email from the ASF dual-hosted git repository.
justinchen pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 16f7ee38630 [To dev/1.3] Load: Fixed the memory allocation bug that
may cause more free query memory & Optimized logger for memory not enough
problem (#16206)(#16222)
16f7ee38630 is described below
commit 16f7ee3863035f3ba100b02dca793bfa64671a15
Author: Caideyipi <[email protected]>
AuthorDate: Thu Aug 21 16:21:03 2025 +0800
[To dev/1.3] Load: Fixed the memory allocation bug that may cause more free
query memory & Optimized logger for memory not enough problem (#16206)(#16222)
---
.../db/queryengine/plan/planner/LocalExecutionPlanner.java | 7 +++++--
.../load/memory/LoadTsFileDataCacheMemoryBlock.java | 8 +++-----
.../storageengine/load/memory/LoadTsFileMemoryManager.java | 14 +++++++-------
3 files changed, 15 insertions(+), 14 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
index b643047dcfe..d91e68e7413 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
@@ -225,9 +225,12 @@ public class LocalExecutionPlanner {
}
}
- public synchronized long tryAllocateFreeMemoryForOperators(long
memoryInBytes) {
+ public synchronized long tryAllocateFreeMemory4Load(final long
memoryInBytes) {
if (freeMemoryForOperators - memoryInBytes <=
MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD) {
- long result = freeMemoryForOperators -
MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;
+ final long result = freeMemoryForOperators -
MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;
+ if (result <= 0) {
+ return 0;
+ }
freeMemoryForOperators = MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;
return result;
} else {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
index 9932b37df02..cbe3b5de26b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
@@ -65,9 +65,7 @@ public class LoadTsFileDataCacheMemoryBlock extends
LoadTsFileAbstractMemoryBloc
@Override
public synchronized void reduceMemoryUsage(long memoryInBytes) {
- if (memoryUsageInBytes.addAndGet(-memoryInBytes) < 0) {
- LOGGER.warn("{} has reduce memory usage to negative", this);
- }
+ memoryUsageInBytes.addAndGet(-memoryInBytes);
}
@Override
@@ -90,11 +88,11 @@ public class LoadTsFileDataCacheMemoryBlock extends
LoadTsFileAbstractMemoryBloc
return true;
}
- if (limitedMemorySizeInBytes.get() - shrinkMemoryInBytes <=
MINIMUM_MEMORY_SIZE_IN_BYTES) {
+ if (limitedMemorySizeInBytes.get() - shrinkMemoryInBytes
+ <= Math.max(MINIMUM_MEMORY_SIZE_IN_BYTES, memoryUsageInBytes.get())) {
return false;
}
- MEMORY_MANAGER.releaseToQuery(shrinkMemoryInBytes);
limitedMemorySizeInBytes.addAndGet(-shrinkMemoryInBytes);
return true;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
index 417d09c1103..a3e349678a5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
@@ -69,19 +69,19 @@ public class LoadTsFileMemoryManager {
+ "current load used memory size %s bytes, load requested
memory size %s bytes",
MEMORY_ALLOCATE_MAX_RETRIES,
QUERY_ENGINE_MEMORY_MANAGER.getAllocateMemoryForOperators(),
- QUERY_ENGINE_MEMORY_MANAGER.getFreeMemoryForLoadTsFile(),
+ Math.max(0L,
QUERY_ENGINE_MEMORY_MANAGER.getFreeMemoryForLoadTsFile()),
usedMemorySizeInBytes.get(),
sizeInBytes));
}
- public synchronized long tryAllocateFromQuery(long sizeInBytes) {
- long actuallyAllocateMemoryInBytes =
- Math.max(0L,
QUERY_ENGINE_MEMORY_MANAGER.tryAllocateFreeMemoryForOperators(sizeInBytes));
+ public synchronized long tryAllocateFromQuery(final long sizeInBytes) {
+ final long actuallyAllocateMemoryInBytes =
+ QUERY_ENGINE_MEMORY_MANAGER.tryAllocateFreeMemory4Load(sizeInBytes);
usedMemorySizeInBytes.addAndGet(actuallyAllocateMemoryInBytes);
return actuallyAllocateMemoryInBytes;
}
- public synchronized void releaseToQuery(long sizeInBytes) {
+ public synchronized void releaseToQuery(final long sizeInBytes) {
if (usedMemorySizeInBytes.get() < sizeInBytes) {
LOGGER.error(
"Load: Attempting to release more memory ({}) than allocated ({})",
@@ -106,7 +106,6 @@ public class LoadTsFileMemoryManager {
LOGGER.info(
"Load: Query engine's memory is not sufficient, allocated
MemoryBlock from DataCacheMemoryBlock, size: {}",
sizeInBytes);
- usedMemorySizeInBytes.addAndGet(sizeInBytes);
return new LoadTsFileMemoryBlock(sizeInBytes);
}
throw e;
@@ -117,9 +116,10 @@ public class LoadTsFileMemoryManager {
public synchronized LoadTsFileDataCacheMemoryBlock
allocateDataCacheMemoryBlock()
throws LoadRuntimeOutOfMemoryException {
if (dataCacheMemoryBlock == null) {
- long actuallyAllocateMemoryInBytes =
+ final long actuallyAllocateMemoryInBytes =
tryAllocateFromQuery(MEMORY_TOTAL_SIZE_FROM_QUERY_IN_BYTES >> 2);
dataCacheMemoryBlock = new
LoadTsFileDataCacheMemoryBlock(actuallyAllocateMemoryInBytes);
+ usedMemorySizeInBytes.addAndGet(actuallyAllocateMemoryInBytes);
LOGGER.info(
"Create Data Cache Memory Block {}, allocate memory {}",
dataCacheMemoryBlock,
