This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch memory_allo_12 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d0140d3819f799ba56781c38594b9409ac8f0335 Author: Haonan <[email protected]> AuthorDate: Thu Jun 8 19:35:32 2023 +0800 Fix storage engine memory config initialization (#10077) --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 83 ++++++++++++++-------- 2 files changed, 55 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index b640f9198a2..2d147943b87 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -154,7 +154,7 @@ public class IoTDBConfig { private double rejectProportion = 0.8; /** The proportion of write memory for memtable */ - private double writeProportionForMemtable = 0.8; + private double writeProportionForMemtable = 0.76; /** The proportion of write memory for compaction */ private double compactionProportion = 0.2; @@ -554,7 +554,7 @@ public class IoTDBConfig { private long allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001; /** Memory allocated proportion for time partition info */ - private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 50 / 1001; + private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 8 / 10 / 20; /** Memory allocated proportion for wal pipe cache */ private long allocateMemoryForWALPipeCache = allocateMemoryForConsensus / 10; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 386920dbf4c..2f1c9fdee4c 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1767,36 +1767,59 @@ public class IoTDBDescriptor { } private void initStorageEngineAllocate(Properties properties) { - String allocationRatio = properties.getProperty("storage_engine_memory_proportion", "8:2"); - String[] proportions = allocationRatio.split(":"); - int proportionForWrite = Integer.parseInt(proportions[0].trim()); - int proportionForCompaction = Integer.parseInt(proportions[1].trim()); - - double writeProportion = - ((double) (proportionForWrite) / (double) (proportionForCompaction + proportionForWrite)); - - String allocationRatioForWrite = properties.getProperty("write_memory_proportion", "19:1"); - proportions = allocationRatioForWrite.split(":"); - int proportionForMemTable = Integer.parseInt(proportions[0].trim()); - int proportionForTimePartitionInfo = Integer.parseInt(proportions[1].trim()); - - double memtableProportionForWrite = - ((double) (proportionForMemTable) - / (double) (proportionForMemTable + proportionForTimePartitionInfo)); - - double timePartitionInfoForWrite = - ((double) (proportionForTimePartitionInfo) - / (double) (proportionForMemTable + proportionForTimePartitionInfo)); - conf.setWriteProportionForMemtable(writeProportion * memtableProportionForWrite); - - conf.setAllocateMemoryForTimePartitionInfo( - (long) - ((writeProportion * timePartitionInfoForWrite) - * conf.getAllocateMemoryForStorageEngine())); - - conf.setCompactionProportion( - ((double) (proportionForCompaction) - / (double) (proportionForCompaction + proportionForWrite))); + long storageMemoryTotal = conf.getAllocateMemoryForStorageEngine(); + + int proportionSum = 10; + int writeProportion = 8; + int compactionProportion = 2; + int writeProportionSum = 20; + int memTableProportion = 19; + int timePartitionInfo = 1; + + String storageMemoryAllocatePortion = + properties.getProperty("storage_engine_memory_proportion"); + if (storageMemoryAllocatePortion != null) { + String[] proportions = storageMemoryAllocatePortion.split(":"); + int loadedProportionSum = 0; + for (String proportion : proportions) { + loadedProportionSum += Integer.parseInt(proportion.trim()); + } + + if (loadedProportionSum != 0) { + proportionSum = loadedProportionSum; + writeProportion = Integer.parseInt(proportions[0].trim()); + compactionProportion = Integer.parseInt(proportions[1].trim()); + } + conf.setCompactionProportion((double) compactionProportion / (double) proportionSum); + } + + String allocationRatioForWrite = properties.getProperty("write_memory_proportion"); + if (allocationRatioForWrite != null) { + String[] proportions = allocationRatioForWrite.split(":"); + int loadedProportionSum = 0; + for (String proportion : proportions) { + loadedProportionSum += Integer.parseInt(proportion.trim()); + } + + if (loadedProportionSum != 0) { + writeProportionSum = loadedProportionSum; + memTableProportion = Integer.parseInt(proportions[0].trim()); + timePartitionInfo = Integer.parseInt(proportions[1].trim()); + } + // memtableProportionForWrite = 19/20 default + double memtableProportionForWrite = + ((double) memTableProportion / (double) writeProportionSum); + + // timePartitionInfoForWrite = 1/20 default + double timePartitionInfoForWrite = ((double) timePartitionInfo / (double) writeProportionSum); + // proportionForWrite = 8/10 default + double proportionForWrite = ((double) (writeProportion) / (double) proportionSum); + // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default + conf.setWriteProportionForMemtable(proportionForWrite * memtableProportionForWrite); + // allocateMemoryForTimePartitionInfo = storageMemoryTotal * 8/10 * 1/20 default + conf.setAllocateMemoryForTimePartitionInfo( + (long) ((proportionForWrite * timePartitionInfoForWrite) * storageMemoryTotal)); + } } private void initSchemaMemoryAllocate(Properties properties) {
