This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ccdd24f34a3 Fix schema memory config initialization (#10079)
ccdd24f34a3 is described below
commit ccdd24f34a32bf7bb7a85ef5c90f883b80953d81
Author: Marcos_Zyk <[email protected]>
AuthorDate: Fri Jun 9 14:19:25 2023 +0800
Fix schema memory config initialization (#10079)
---
.../iotdb/it/env/cluster/MppCommonConfig.java | 2 +-
.../org/apache/iotdb/util/AbstractSchemaIT.java | 3 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 33 +++++++------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 54 +++++++++++-----------
4 files changed, 45 insertions(+), 47 deletions(-)
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
index 0f3ff163403..63729a81c8a 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
@@ -331,7 +331,7 @@ public class MppCommonConfig extends MppBaseConfig
implements CommonConfig {
@Override
public CommonConfig setSchemaMemoryAllocate(String schemaMemoryAllocate) {
- setProperty("schema_memory_allocate_proportion",
String.valueOf(schemaMemoryAllocate));
+ setProperty("schema_memory_proportion",
String.valueOf(schemaMemoryAllocate));
return this;
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java
b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java
index 6a9c3bdeb26..618cdc1dc14 100644
--- a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java
@@ -113,8 +113,7 @@ public abstract class AbstractSchemaIT {
protected static void allocateMemoryForSchemaRegion(int
allocateMemoryForSchemaRegion) {
int schemaAllMemory = 25742540;
int sumProportion = schemaAllMemory / allocateMemoryForSchemaRegion;
- int[] proportion =
- new int[] {1, (sumProportion - 1) / 2, (sumProportion - 1) / 4,
(sumProportion - 1) / 4};
+ int[] proportion = new int[] {1, (sumProportion - 1) * 3 / 4,
(sumProportion - 1) / 4};
EnvFactory.getEnv()
.getConfig()
.getCommonConfig()
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 2871122afe0..2d147943b87 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1019,17 +1019,16 @@ public class IoTDBConfig {
/** ThreadPool size for write operation in coordinator */
private int coordinatorWriteExecutorSize = 50;
+ private int[] schemaMemoryProportion = new int[] {5, 4, 1};
+
/** Memory allocated for schemaRegion */
- private long allocateMemoryForSchemaRegion = allocateMemoryForSchema * 8 /
10;
+ private long allocateMemoryForSchemaRegion = allocateMemoryForSchema * 5 /
10;
/** Memory allocated for SchemaCache */
- private long allocateMemoryForSchemaCache = allocateMemoryForSchema / 10;
+ private long allocateMemoryForSchemaCache = allocateMemoryForSchema * 4 / 10;
/** Memory allocated for PartitionCache */
- private long allocateMemoryForPartitionCache = 0;
-
- /** Memory allocated for LastCache */
- private long allocateMemoryForLastCache = allocateMemoryForSchema / 10;
+ private long allocateMemoryForPartitionCache = allocateMemoryForSchema / 10;
/** Policy of DataNodeSchemaCache eviction */
private String dataNodeSchemaCacheEvictionPolicy = "FIFO";
@@ -1945,9 +1944,9 @@ public class IoTDBConfig {
public void setAllocateMemoryForSchema(long allocateMemoryForSchema) {
this.allocateMemoryForSchema = allocateMemoryForSchema;
- this.allocateMemoryForSchemaRegion = allocateMemoryForSchema * 8 / 10;
- this.allocateMemoryForSchemaCache = allocateMemoryForSchema / 10;
- this.allocateMemoryForLastCache = allocateMemoryForSchema / 10;
+ this.allocateMemoryForSchemaRegion = allocateMemoryForSchema * 5 / 10;
+ this.allocateMemoryForSchemaCache = allocateMemoryForSchema * 4 / 10;
+ this.allocateMemoryForPartitionCache = allocateMemoryForSchema / 10;
}
public long getAllocateMemoryForConsensus() {
@@ -3377,6 +3376,14 @@ public class IoTDBConfig {
return new TEndPoint(rpcAddress, rpcPort);
}
+ public int[] getSchemaMemoryProportion() {
+ return schemaMemoryProportion;
+ }
+
+ public void setSchemaMemoryProportion(int[] schemaMemoryProportion) {
+ this.schemaMemoryProportion = schemaMemoryProportion;
+ }
+
public long getAllocateMemoryForSchemaRegion() {
return allocateMemoryForSchemaRegion;
}
@@ -3401,14 +3408,6 @@ public class IoTDBConfig {
this.allocateMemoryForPartitionCache = allocateMemoryForPartitionCache;
}
- public long getAllocateMemoryForLastCache() {
- return allocateMemoryForLastCache;
- }
-
- public void setAllocateMemoryForLastCache(long allocateMemoryForLastCache) {
- this.allocateMemoryForLastCache = allocateMemoryForLastCache;
- }
-
public String getDataNodeSchemaCacheEvictionPolicy() {
return dataNodeSchemaCacheEvictionPolicy;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index ddb685cf94a..2f1c9fdee4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1825,59 +1825,59 @@ public class IoTDBDescriptor {
private void initSchemaMemoryAllocate(Properties properties) {
long schemaMemoryTotal = conf.getAllocateMemoryForSchema();
- int proportionSum = 10;
- int schemaRegionProportion = 5;
- int schemaCacheProportion = 4;
- int partitionCacheProportion = 1;
- int lastCacheProportion = 0;
-
- String schemaMemoryAllocatePortion =
properties.getProperty("schema_memory_proportion");
- if (schemaMemoryAllocatePortion != null) {
- String[] proportions = schemaMemoryAllocatePortion.split(":");
+ String schemaMemoryPortionInput =
properties.getProperty("schema_memory_proportion");
+ if (schemaMemoryPortionInput != null) {
+ String[] proportions = schemaMemoryPortionInput.split(":");
int loadedProportionSum = 0;
for (String proportion : proportions) {
loadedProportionSum += Integer.parseInt(proportion.trim());
}
if (loadedProportionSum != 0) {
- proportionSum = loadedProportionSum;
- schemaRegionProportion = Integer.parseInt(proportions[0].trim());
- schemaCacheProportion = Integer.parseInt(proportions[1].trim());
- partitionCacheProportion = Integer.parseInt(proportions[2].trim());
+ conf.setSchemaMemoryProportion(
+ new int[] {
+ Integer.parseInt(proportions[0].trim()),
+ Integer.parseInt(proportions[1].trim()),
+ Integer.parseInt(proportions[2].trim())
+ });
}
} else {
- schemaMemoryAllocatePortion =
properties.getProperty("schema_memory_allocate_proportion");
- if (schemaMemoryAllocatePortion != null) {
- String[] proportions = schemaMemoryAllocatePortion.split(":");
+ schemaMemoryPortionInput =
properties.getProperty("schema_memory_allocate_proportion");
+ if (schemaMemoryPortionInput != null) {
+ String[] proportions = schemaMemoryPortionInput.split(":");
int loadedProportionSum = 0;
for (String proportion : proportions) {
loadedProportionSum += Integer.parseInt(proportion.trim());
}
if (loadedProportionSum != 0) {
- proportionSum = loadedProportionSum;
- schemaRegionProportion = Integer.parseInt(proportions[0].trim());
- schemaCacheProportion = Integer.parseInt(proportions[1].trim());
- partitionCacheProportion = Integer.parseInt(proportions[2].trim());
- lastCacheProportion = Integer.parseInt(proportions[3].trim());
+ conf.setSchemaMemoryProportion(
+ new int[] {
+ Integer.parseInt(proportions[0].trim()),
+ Integer.parseInt(proportions[1].trim()) +
Integer.parseInt(proportions[3].trim()),
+ Integer.parseInt(proportions[2].trim())
+ });
}
}
}
+ int proportionSum = 0;
+ for (int proportion : conf.getSchemaMemoryProportion()) {
+ proportionSum += proportion;
+ }
+
conf.setAllocateMemoryForSchemaRegion(
- schemaMemoryTotal * schemaRegionProportion / proportionSum);
+ schemaMemoryTotal * conf.getSchemaMemoryProportion()[0] /
proportionSum);
logger.info("allocateMemoryForSchemaRegion = {}",
conf.getAllocateMemoryForSchemaRegion());
- conf.setAllocateMemoryForSchemaCache(schemaMemoryTotal *
schemaCacheProportion / proportionSum);
+ conf.setAllocateMemoryForSchemaCache(
+ schemaMemoryTotal * conf.getSchemaMemoryProportion()[1] /
proportionSum);
logger.info("allocateMemoryForSchemaCache = {}",
conf.getAllocateMemoryForSchemaCache());
conf.setAllocateMemoryForPartitionCache(
- schemaMemoryTotal * partitionCacheProportion / proportionSum);
+ schemaMemoryTotal * conf.getSchemaMemoryProportion()[2] /
proportionSum);
logger.info("allocateMemoryForPartitionCache = {}",
conf.getAllocateMemoryForPartitionCache());
-
- conf.setAllocateMemoryForLastCache(schemaMemoryTotal * lastCacheProportion
/ proportionSum);
- logger.info("allocateMemoryForLastCache = {}",
conf.getAllocateMemoryForLastCache());
}
@SuppressWarnings("squid:S3518") // "proportionSum" can't be zero
