This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 06621d0f045 mon: add diagnose information for MemoryNotEnoughException
(#17578)
06621d0f045 is described below
commit 06621d0f045a15a66e2f16434fd0724caef226db
Author: shizy <[email protected]>
AuthorDate: Thu Apr 30 08:52:34 2026 +0800
mon: add diagnose information for MemoryNotEnoughException (#17578)
---
.../fragment/FragmentInstanceContext.java | 2 +-
.../schemaregion/utils/ResourceByPathUtils.java | 15 ++++++--
.../memtable/AbstractWritableMemChunk.java | 2 +-
.../memtable/AlignedReadOnlyMemChunk.java | 4 +--
.../dataregion/memtable/ReadOnlyMemChunk.java | 4 +--
.../db/utils/datastructure/AlignedTVList.java | 5 +--
.../iotdb/db/utils/datastructure/TVList.java | 42 ++++++++++++++++++++--
.../fragment/FragmentInstanceExecutionTest.java | 4 +--
.../dataregion/memtable/PrimitiveMemTableTest.java | 2 +-
9 files changed, 64 insertions(+), 16 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index 5102319aeee..3adc5373a75 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -934,7 +934,7 @@ public class FragmentInstanceContext extends QueryContext {
*/
private void releaseTVListOwnedByQuery() {
for (TVList tvList : tvListSet) {
- long tvListRamSize = tvList.calculateRamSize();
+ long tvListRamSize = tvList.calculateRamSize().getRamSize();
tvList.lockQueryList();
Set<QueryContext> queryContextSet = tvList.getQueryContextSet();
try {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
index 1f177864dd1..e0590a1410e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.schemaengine.schemaregion.utils;
+import org.apache.iotdb.calc.exception.MemoryNotEnoughException;
import org.apache.iotdb.calc.exception.QueryProcessException;
import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager;
import org.apache.iotdb.commons.path.AlignedFullPath;
@@ -155,7 +156,7 @@ public abstract class ResourceByPathUtils {
// mutable tvlist
TVList list = memChunk.getWorkingTVList();
TVList cloneList = null;
- long tvListRamSize = list.calculateRamSize();
+ TVList.RamInfo listRamInfo = list.calculateRamSize();
list.lockQueryList();
try {
if (copyTimeFilter != null
@@ -196,8 +197,8 @@ public abstract class ResourceByPathUtils {
if (firstQuery instanceof FragmentInstanceContext) {
MemoryReservationManager memoryReservationManager =
((FragmentInstanceContext)
firstQuery).getMemoryReservationContext();
- memoryReservationManager.reserveMemoryCumulatively(tvListRamSize);
- list.setReservedMemoryBytes(tvListRamSize);
+
memoryReservationManager.reserveMemoryCumulatively(listRamInfo.getRamSize());
+ list.setReservedMemoryBytes(listRamInfo.getRamSize());
}
list.setOwnerQuery(firstQuery);
@@ -207,6 +208,14 @@ public abstract class ResourceByPathUtils {
tvListQueryMap.put(cloneList, cloneList.rowCount());
}
}
+ } catch (MemoryNotEnoughException ex) {
+ LOGGER.warn(
+ "Failed to reserve memory for TVList: ramSize {}, timestampsSize {},
arrayMemCost {}, rowCount {}, dataTypes {}",
+ listRamInfo.getRamSize(),
+ listRamInfo.getTimestampsSize(),
+ listRamInfo.getArrayMemCost(),
+ listRamInfo.getRowCount(),
+ listRamInfo.getDataTypes());
} finally {
list.unlockQueryList();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
index 0f78e0717c3..6ae9333b2de 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
@@ -103,7 +103,7 @@ public abstract class AbstractWritableMemChunk implements
IWritableMemChunk {
}
private void tryReleaseTvList(TVList tvList) {
- long tvListRamSize = tvList.calculateRamSize();
+ long tvListRamSize = tvList.calculateRamSize().getRamSize();
tvList.lockQueryList();
try {
if (tvList.getQueryContextSet().isEmpty()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
index 517566ed78c..05c1602bf2a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
@@ -127,7 +127,7 @@ public class AlignedReadOnlyMemChunk extends
ReadOnlyMemChunk {
// We must update queryRowCount here, otherwise, it may be used later
to build
// BitMaps, causing bitmap array size mismatch and possible out of
bound.
entry.setValue(alignedTvList.sort());
- long alignedTvListRamSize = alignedTvList.calculateRamSize();
+ long alignedTvListRamSize =
alignedTvList.calculateRamSize().getRamSize();
alignedTvList.lockQueryList();
try {
FragmentInstanceContext ownerQuery =
@@ -384,7 +384,7 @@ public class AlignedReadOnlyMemChunk extends
ReadOnlyMemChunk {
int queryLength = entry.getValue();
if (!alignedTvList.isSorted() && queryLength >
alignedTvList.seqRowCount()) {
entry.setValue(alignedTvList.sort());
- long alignedTvListRamSize = alignedTvList.calculateRamSize();
+ long alignedTvListRamSize =
alignedTvList.calculateRamSize().getRamSize();
alignedTvList.lockQueryList();
try {
FragmentInstanceContext ownerQuery =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java
index 811a080c22f..496035e91ad 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java
@@ -137,7 +137,7 @@ public class ReadOnlyMemChunk {
int queryRowCount = entry.getValue();
if (!tvList.isSorted() && queryRowCount > tvList.seqRowCount()) {
entry.setValue(tvList.sort());
- long tvListRamSize = tvList.calculateRamSize();
+ long tvListRamSize = tvList.calculateRamSize().getRamSize();
tvList.lockQueryList();
try {
FragmentInstanceContext ownerQuery = (FragmentInstanceContext)
tvList.getOwnerQuery();
@@ -294,7 +294,7 @@ public class ReadOnlyMemChunk {
int queryLength = entry.getValue();
if (!tvList.isSorted() && queryLength > tvList.seqRowCount()) {
entry.setValue(tvList.sort());
- long tvListRamSize = tvList.calculateRamSize();
+ long tvListRamSize = tvList.calculateRamSize().getRamSize();
tvList.lockQueryList();
try {
FragmentInstanceContext ownerQuery = (FragmentInstanceContext)
tvList.getOwnerQuery();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 6293a0fc4b9..cb1379dc19d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -1064,8 +1064,9 @@ public abstract class AlignedTVList extends TVList {
}
@Override
- public synchronized long calculateRamSize() {
- return timestamps.size() * alignedTvListArrayMemCost();
+ public synchronized RamInfo calculateRamSize() {
+ return new RamInfo(
+ timestamps.size(), alignedTvListArrayMemCost(), rowCount, new
ArrayList<>(dataTypes));
}
/**
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 8b571fed01f..fb0cc005813 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -28,6 +28,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager;
import org.apache.iotdb.db.utils.MathUtils;
+import com.google.common.collect.ImmutableList;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TimeValuePair;
@@ -61,6 +62,42 @@ import static
org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
public abstract class TVList implements WALEntryValue {
protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not
consistent";
+
+ public static class RamInfo {
+ private final int timestampsSize;
+ private final long arrayMemCost;
+ private final int rowCount;
+ private final List<TSDataType> dataTypes;
+
+ public RamInfo(
+ int timestampCount, long arrayMemCost, int rowCount, List<TSDataType>
dataTypes) {
+ this.timestampsSize = timestampCount;
+ this.rowCount = rowCount;
+ this.arrayMemCost = arrayMemCost;
+ this.dataTypes = dataTypes;
+ }
+
+ public long getRamSize() {
+ return timestampsSize * arrayMemCost;
+ }
+
+ public int getTimestampsSize() {
+ return timestampsSize;
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ public long getArrayMemCost() {
+ return arrayMemCost;
+ }
+
+ public List<TSDataType> getDataTypes() {
+ return dataTypes;
+ }
+ }
+
// list of timestamp array, add 1 when expanded -> data point timestamp array
// index relation: arrayIndex -> elementIndex
protected List<long[]> timestamps;
@@ -165,8 +202,9 @@ public abstract class TVList implements WALEntryValue {
return size;
}
- public synchronized long calculateRamSize() {
- return timestamps.size() * tvListArrayMemCost();
+ public synchronized RamInfo calculateRamSize() {
+ return new RamInfo(
+ timestamps.size(), tvListArrayMemCost(), rowCount,
ImmutableList.of(getDataType()));
}
public synchronized boolean isSorted() {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java
index 393884a519b..99a94eb02a3 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java
@@ -146,7 +146,7 @@ public class FragmentInstanceExecutionTest {
// mock flush's behavior
fragmentInstanceContext1
.getMemoryReservationContext()
- .reserveMemoryCumulatively(tvList.calculateRamSize());
+ .reserveMemoryCumulatively(tvList.calculateRamSize().getRamSize());
tvList.setOwnerQuery(fragmentInstanceContext1);
fragmentInstanceContext1.decrementNumOfUnClosedDriver();
@@ -226,7 +226,7 @@ public class FragmentInstanceExecutionTest {
pointReader.nextTimeValuePair();
}
assertTrue(tvList.isSorted());
- assertEquals(tvList.calculateRamSize(), tvList.getReservedMemoryBytes());
+ assertEquals(tvList.calculateRamSize().getRamSize(),
tvList.getReservedMemoryBytes());
} catch (QueryProcessException
| IOException
| MetadataException
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java
index 70ea839382c..c47297fbeb4 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java
@@ -748,7 +748,7 @@ public class PrimitiveMemTableTest {
Mockito.mock(MemoryReservationManager.class);
Mockito.doThrow(new MemoryNotEnoughException(""))
.when(memoryReservationManager)
- .reserveMemoryCumulatively(list.calculateRamSize());
+ .reserveMemoryCumulatively(list.calculateRamSize().getRamSize());
// create FragmentInstanceId
QueryId queryId = new QueryId("stub_query");
