This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch opt_aligned_tvlist in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ead01cd0a4c4751402a88094b5393e4d7c2de438 Author: HTHou <[email protected]> AuthorDate: Wed Aug 27 17:16:26 2025 +0800 dev --- .../memtable/AlignedWritableMemChunk.java | 61 ++++++++++++++++++++++ .../dataregion/memtable/TsFileProcessor.java | 29 +++++----- .../db/utils/datastructure/AlignedTVList.java | 18 +++++-- 3 files changed, 93 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index d41ef855d36..d7b7b4b027c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo; import org.apache.iotdb.db.utils.datastructure.MemPointIterator; @@ -55,6 +56,8 @@ import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted; +import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; +import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; public class AlignedWritableMemChunk extends AbstractWritableMemChunk { @@ -860,4 +863,62 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { } return avgPointSizeOfLargestColumn; } + + public long getTvListArrayMemCostIncrement1( + List<String> insertingMeasurements, List<TSDataType> insertingTypes) { + long size = 0; + List<BitMap> bitMaps = list.getBitMap(); + // value & bitmap array mem size + for (int column = 0; column < dataTypes.size(); column++) { + TSDataType type = dataTypes.get(column); + if (type != null) { + if (bitMaps != null && bitMaps.get(column) != null) { + size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1; + } + } + } + int newMeasurementCount = 0; + for (int i = 0; i < insertingMeasurements.size(); i++) { + String measurementName = insertingMeasurements.get(i); + TSDataType type = insertingTypes.get(i); + size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); + if (!measurementIndexMap.containsKey(measurementName)) { + newMeasurementCount++; + } + } + // size is 0 when all types are null + if (size == 0) { + return size; + } + // time array mem size + size += PrimitiveArrayManager.ARRAY_SIZE * 8L; + // index array mem size + size += (list.getIndices() != null) ? PrimitiveArrayManager.ARRAY_SIZE * 4L : 0; + // array headers mem size + size += (long) NUM_BYTES_ARRAY_HEADER * (2 + insertingTypes.size()); + // Object references size in ArrayList + size += (long) NUM_BYTES_OBJECT_REF * (2 + dataTypes.size() + newMeasurementCount); + return size; + } + + public long getTvListArrayMemCostIncrement( + List<String> insertingMeasurements, List<TSDataType> insertingTypes) { + long memCostIncrement = 0; + for (int i = 0; i < insertingMeasurements.size(); i++) { + String measurementName = insertingMeasurements.get(i); + TSDataType dataType = insertingTypes.get(i); + Integer columIndex = measurementIndexMap.get(measurementName); + if (columIndex == null) { + memCostIncrement += + (long) PrimitiveArrayManager.ARRAY_SIZE * (long) dataType.getDataTypeSize(); + } else { + List<Object> columnArries = list.getValues().get(columIndex); + if (columnArries.get(columnArries.size() - 1) == null) { + memCostIncrement += + (long) PrimitiveArrayManager.ARRAY_SIZE * (long) dataType.getDataTypeSize(); + } + } + } + return memCostIncrement; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 6602a3b07bb..d28880fd4cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -792,9 +792,9 @@ public class TsFileProcessor { } } // this insertion will result in a new array - if ((alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE) == 0) { - memTableIncrement += alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost(); - } + // if ((alignedMemChunk.alignedListSize() % PrimitiveArrayManager.ARRAY_SIZE) == 0) { + // memTableIncrement += alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost(); + // } } for (int i = 0; i < dataTypes.length; i++) { @@ -874,8 +874,7 @@ public class TsFileProcessor { > 0 ? 1 : 0); - memTableIncrement += - currentArrayNum * AlignedTVList.emptyValueListArrayMemCost(); + memTableIncrement += currentArrayNum * AlignedTVList.emptyValueListArrayMemCost(); } } int addingPointNum = addingPointNumInfo.right; @@ -885,11 +884,11 @@ public class TsFileProcessor { dataTypesInTVList.addAll(alignedMemChunk.getWorkingTVList().getTsDataTypes()); } dataTypesInTVList.addAll(addingPointNumInfo.left.values()); - memTableIncrement += - alignedMemChunk != null - ? alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost() - : AlignedTVList.alignedTvListArrayMemCost( - dataTypesInTVList.toArray(new TSDataType[0]), null); + // memTableIncrement += + // alignedMemChunk != null + // ? alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost() + // : AlignedTVList.alignedTvListArrayMemCost( + // dataTypesInTVList.toArray(new TSDataType[0]), null); } addingPointNumInfo.setRight(addingPointNum + 1); } @@ -1066,6 +1065,7 @@ public class TsFileProcessor { AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; int currentPointNum = alignedMemChunk.alignedListSize(); int newPointNum = currentPointNum + incomingPointNum; + List<String> insertingMeasurements = new ArrayList<>(); List<TSDataType> insertingTypes = new ArrayList<>(); for (int i = 0; i < dataTypes.length; i++) { TSDataType dataType = dataTypes[i]; @@ -1077,6 +1077,7 @@ public class TsFileProcessor { || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { continue; } + insertingMeasurements.add(measurement); insertingTypes.add(dataType); if (!alignedMemChunk.containsMeasurement(measurementIds[i])) { @@ -1098,9 +1099,13 @@ public class TsFileProcessor { if (acquireArray != 0) { // memory of extending the TVList - memIncrements[0] += acquireArray * alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost(insertingTypes); + memIncrements[0] += + acquireArray + * alignedMemChunk.getTvListArrayMemCostIncrement1( + insertingMeasurements, insertingTypes); } else { - + memIncrements[0] += + alignedMemChunk.getTvListArrayMemCostIncrement(insertingMeasurements, insertingTypes); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 996eb1282a8..2f346de64a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -528,10 +528,18 @@ public abstract class AlignedTVList extends TVList { return columnBitMaps.get(arrayIndex).isMarked(elementIndex); } + public List<List<Object>> getValues() { + return values; + } + public List<TSDataType> getTsDataTypes() { return dataTypes; } + public List<int[]> getIndices() { + return indices; + } + @Override public int delete(long lowerBound, long upperBound) { int deletedNumber = 0; @@ -983,7 +991,9 @@ public abstract class AlignedTVList extends TVList { @Override public long calculateRamSize() { - return timestamps.size() * alignedTvListArrayMemCost(); + // return timestamps.size() * alignedTvListArrayMemCost(); + // FIXME:(Haonan) + return 0; } /** @@ -1030,12 +1040,14 @@ public abstract class AlignedTVList extends TVList { for (int column = 0; column < dataTypes.size(); column++) { TSDataType type = dataTypes.get(column); if (type != null) { - size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); if (bitMaps != null && bitMaps.get(column) != null) { size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1; } } } + for (TSDataType type : insertingTypes) { + size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); + } // size is 0 when all types are null if (size == 0) { return size; @@ -1045,7 +1057,7 @@ public abstract class AlignedTVList extends TVList { // index array mem size size += (indices != null) ? PrimitiveArrayManager.ARRAY_SIZE * 4L : 0; // array headers mem size - size += (long) NUM_BYTES_ARRAY_HEADER * (2 + dataTypes.size()); + size += (long) NUM_BYTES_ARRAY_HEADER * (2 + insertingTypes.size()); // Object references size in ArrayList size += (long) NUM_BYTES_OBJECT_REF * (2 + dataTypes.size()); return size;
