This is an automated email from the ASF dual-hosted git repository. yongzao pushed a commit to branch data-region-per-datanode in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fe1993165780cfe2a8a83a7355f7c41f68c299fa Author: YongzaoDan <[email protected]> AuthorDate: Mon Mar 20 11:30:18 2023 +0800 Finish --- .../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +++++----- .../iotdb/confignode/conf/ConfigNodeDescriptor.java | 4 ++-- .../confignode/manager/ClusterSchemaManager.java | 19 +++++++++---------- .../iotdb/confignode/manager/ConfigManager.java | 6 +++--- docs/UserGuide/Cluster/Cluster-Maintenance.md | 2 +- docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 2 +- .../it/cluster/IoTDBClusterNodeGetterIT.java | 4 ++-- .../confignode/it/utils/ConfigNodeTestUtils.java | 2 +- .../assembly/resources/conf/iotdb-common.properties | 6 +++--- .../db/mpp/common/header/ColumnHeaderConstant.java | 2 +- .../execution/config/metadata/ShowVariablesTask.java | 4 ++-- thrift-confignode/src/main/thrift/confignode.thrift | 2 +- 12 files changed, 31 insertions(+), 32 deletions(-) diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 0a725b42b5..8e963ee0c3 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -101,7 +101,7 @@ public class ConfigNodeConfig { private int defaultDataRegionGroupNumPerDatabase = 2; /** The maximum number of DataRegions expected to be managed by each DataNode. */ - private double dataRegionPerProcessor = 1.0; + private double dataRegionPerDataNode = 5.0; /** RegionGroup allocate policy. */ private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy = @@ -546,12 +546,12 @@ public class ConfigNodeConfig { this.dataRegionConsensusProtocolClass = dataRegionConsensusProtocolClass; } - public double getDataRegionPerProcessor() { - return dataRegionPerProcessor; + public double getDataRegionPerDataNode() { + return dataRegionPerDataNode; } - public void setDataRegionPerProcessor(double dataRegionPerProcessor) { - this.dataRegionPerProcessor = dataRegionPerProcessor; + public void setDataRegionPerDataNode(double dataRegionPerDataNode) { + this.dataRegionPerDataNode = dataRegionPerDataNode; } public RegionBalancer.RegionGroupAllocatePolicy getRegionGroupAllocatePolicy() { diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index db7b8c0a11..ef1536776d 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -250,11 +250,11 @@ public class ConfigNodeDescriptor { "default_data_region_group_num_per_database", String.valueOf(conf.getDefaultDataRegionGroupNumPerDatabase()).trim()))); - conf.setDataRegionPerProcessor( + conf.setDataRegionPerDataNode( Double.parseDouble( properties .getProperty( - "data_region_per_processor", String.valueOf(conf.getDataRegionPerProcessor())) + "data_region_per_data_node", String.valueOf(conf.getDataRegionPerDataNode())) .trim())); try { diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java index 7c6295722f..89d93a6fa0 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java @@ -103,7 +103,7 @@ public class ClusterSchemaManager { private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); private static final double SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode(); - private static final double DATA_REGION_PER_PROCESSOR = CONF.getDataRegionPerProcessor(); + private static final double DATA_REGION_PER_DATA_NODE = CONF.getDataRegionPerDataNode(); private final IManager configManager; private final ClusterSchemaInfo clusterSchemaInfo; @@ -376,7 +376,6 @@ public class ClusterSchemaManager { } int dataNodeNum = getNodeManager().getRegisteredDataNodeCount(); - int totalCpuCoreNum = getNodeManager().getTotalCpuCoreCount(); int databaseNum = databaseSchemaMap.size(); for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) { @@ -389,8 +388,8 @@ public class ClusterSchemaManager { AdjustMaxRegionGroupNumPlan adjustMaxRegionGroupNumPlan = new AdjustMaxRegionGroupNumPlan(); for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) { try { - // Adjust maxSchemaRegionGroupNum for each StorageGroup. - // All StorageGroups share the DataNodes equally. + // Adjust maxSchemaRegionGroupNum for each Database. + // All Databases share the DataNodes equally. // The allocated SchemaRegionGroups will not be shrunk. int allocatedSchemaRegionGroupCount; try { @@ -415,8 +414,8 @@ public class ClusterSchemaManager { databaseSchema.getName(), maxSchemaRegionGroupNum); - // Adjust maxDataRegionGroupNum for each StorageGroup. - // All StorageGroups divide the total cpu cores equally. + // Adjust maxDataRegionGroupNum for each Database. + // All Databases share the DataNodes equally. // The allocated DataRegionGroups will not be shrunk. int allocatedDataRegionGroupCount = getPartitionManager() @@ -424,8 +423,8 @@ public class ClusterSchemaManager { int maxDataRegionGroupNum = calcMaxRegionGroupNum( databaseSchema.getMinDataRegionGroupNum(), - DATA_REGION_PER_PROCESSOR, - totalCpuCoreNum, + DATA_REGION_PER_DATA_NODE, + dataNodeNum, databaseNum, databaseSchema.getDataReplicationFactor(), allocatedDataRegionGroupCount); @@ -447,7 +446,7 @@ public class ClusterSchemaManager { int minRegionGroupNum, double resourceWeight, int resource, - int storageGroupNum, + int databaseNum, int replicationFactor, int allocatedRegionGroupCount) { return Math.max( @@ -460,7 +459,7 @@ public class ClusterSchemaManager { Math.ceil( // The maxRegionGroupNum of the current StorageGroup is expected to be: // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor) - resourceWeight * resource / (double) (storageGroupNum * replicationFactor)), + resourceWeight * resource / (double) (databaseNum * replicationFactor)), // The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount allocatedRegionGroupCount)); } diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index d92883c14f..924cf9ef1e 100644 --- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -468,7 +468,7 @@ public class ConfigManager implements IManager { clusterParameters.setTimePartitionInterval(CONF.getTimePartitionInterval()); clusterParameters.setDataReplicationFactor(CONF.getDataReplicationFactor()); clusterParameters.setSchemaReplicationFactor(CONF.getSchemaReplicationFactor()); - clusterParameters.setDataRegionPerProcessor(CONF.getDataRegionPerProcessor()); + clusterParameters.setDataRegionPerDataNode(CONF.getDataRegionPerDataNode()); clusterParameters.setSchemaRegionPerDataNode(CONF.getSchemaRegionPerDataNode()); clusterParameters.setDiskSpaceWarningThreshold(COMMON_CONF.getDiskSpaceWarningThreshold()); clusterParameters.setReadConsistencyLevel(CONF.getReadConsistencyLevel()); @@ -1077,8 +1077,8 @@ public class ConfigManager implements IManager { if (clusterParameters.getSchemaRegionPerDataNode() != CONF.getSchemaRegionPerDataNode()) { return errorStatus.setMessage(errorPrefix + "schema_region_per_data_node" + errorSuffix); } - if (clusterParameters.getDataRegionPerProcessor() != CONF.getDataRegionPerProcessor()) { - return errorStatus.setMessage(errorPrefix + "data_region_per_processor" + errorSuffix); + if (clusterParameters.getDataRegionPerDataNode() != CONF.getDataRegionPerDataNode()) { + return errorStatus.setMessage(errorPrefix + "data_region_per_data_node" + errorSuffix); } if (!clusterParameters.getReadConsistencyLevel().equals(CONF.getReadConsistencyLevel())) { diff --git a/docs/UserGuide/Cluster/Cluster-Maintenance.md b/docs/UserGuide/Cluster/Cluster-Maintenance.md index f10a4ed906..8c3ab7c809 100644 --- a/docs/UserGuide/Cluster/Cluster-Maintenance.md +++ b/docs/UserGuide/Cluster/Cluster-Maintenance.md @@ -44,7 +44,7 @@ IoTDB> show variables | DefaultTTL(ms)| 9223372036854775807| | ReadConsistencyLevel| strong| | SchemaRegionPerDataNode| 1.0| -| DataRegionPerProcessor| 1.0| +| DataRegionPerDataNode| 5.0| | LeastDataRegionGroupNum| 5| | SeriesSlotNum| 10000| | SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor| diff --git a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md index ea7dd151d0..84bc486703 100644 --- a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md +++ b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md @@ -44,7 +44,7 @@ IoTDB> show variables | DefaultTTL(ms)| 9223372036854775807| | ReadConsistencyLevel| strong| | SchemaRegionPerDataNode| 1.0| -| DataRegionPerProcessor| 1.0| +| DataRegionPerDataNode| 5.0| | LeastDataRegionGroupNum| 5| | SeriesSlotNum| 10000| | SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor| diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java index 351d69a303..5694dbd25e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java @@ -162,8 +162,8 @@ public class IoTDBClusterNodeGetterIT { expectedParameters.getSchemaReplicationFactor(), clusterParameters.getSchemaReplicationFactor()); Assert.assertEquals( - expectedParameters.getDataRegionPerProcessor(), - clusterParameters.getDataRegionPerProcessor(), + expectedParameters.getDataRegionPerDataNode(), + clusterParameters.getDataRegionPerDataNode(), 0.01); Assert.assertEquals( expectedParameters.getSchemaRegionPerDataNode(), diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java index 735082c5d7..2bca1e15cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java @@ -208,7 +208,7 @@ public class ConfigNodeTestUtils { clusterParameters.setTimePartitionInterval(604800000); clusterParameters.setDataReplicationFactor(1); clusterParameters.setSchemaReplicationFactor(1); - clusterParameters.setDataRegionPerProcessor(1.0); + clusterParameters.setDataRegionPerDataNode(5.0); clusterParameters.setSchemaRegionPerDataNode(1.0); clusterParameters.setDiskSpaceWarningThreshold(0.05); clusterParameters.setReadConsistencyLevel("strong"); diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties index 0ac0c6f4ce..02bae820b8 100644 --- a/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -103,7 +103,7 @@ cluster_name=defaultCluster # This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode. # Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema, # this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases. -# Default is equal to the schema_replication_factor. +# Default is equal to the schema_replication_factor to ensure each DataNode will have a SchemaRegionGroupLeader. # Datatype: Double # schema_region_per_data_node=1.0 @@ -122,11 +122,11 @@ cluster_name=defaultCluster # default_data_region_group_num_per_database=2 # Only take effect when set data_region_group_extension_policy=AUTO. -# This parameter is the maximum number of DataRegions expected to be managed by each processor. +# This parameter is the maximum number of DataRegions expected to be managed by each DataNode. # Notice: Since each Database requires at least two DataRegionGroups to manage its data, # this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases. # Datatype: Double -# data_region_per_processor=1.0 +# data_region_per_data_node=5.0 # Whether to enable the DataPartition inherit policy. diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java index e78060ce3a..2097a05478 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java @@ -114,7 +114,7 @@ public class ColumnHeaderConstant { public static final String SERIES_SLOT_EXECUTOR_CLASS = "SeriesSlotExecutorClass"; public static final String DEFAULT_TTL = "DefaultTTL(ms)"; public static final String SCHEMA_REGION_PER_DATA_NODE = "SchemaRegionPerDataNode"; - public static final String DATA_REGION_PER_PROCESSOR = "DataRegionPerProcessor"; + public static final String DATA_REGION_PER_DATA_NODE = "DataRegionPerDataNode"; public static final String READ_CONSISTENCY_LEVEL = "ReadConsistencyLevel"; public static final String DISK_SPACE_WARNING_THRESHOLD = "DiskSpaceWarningThreshold"; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java index 90e3ffd166..a43715fe9c 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java @@ -103,8 +103,8 @@ public class ShowVariablesTask implements IConfigTask { new Binary(String.valueOf(clusterParameters.getSchemaRegionPerDataNode()))); buildTSBlock( builder, - new Binary(ColumnHeaderConstant.DATA_REGION_PER_PROCESSOR), - new Binary(String.valueOf(clusterParameters.getDataRegionPerProcessor()))); + new Binary(ColumnHeaderConstant.DATA_REGION_PER_DATA_NODE), + new Binary(String.valueOf(clusterParameters.getDataRegionPerDataNode()))); buildTSBlock( builder, new Binary(ColumnHeaderConstant.SERIES_SLOT_NUM), diff --git a/thrift-confignode/src/main/thrift/confignode.thrift b/thrift-confignode/src/main/thrift/confignode.thrift index fc866714f2..ac5ab69268 100644 --- a/thrift-confignode/src/main/thrift/confignode.thrift +++ b/thrift-confignode/src/main/thrift/confignode.thrift @@ -333,7 +333,7 @@ struct TClusterParameters { 8: required i64 defaultTTL 9: required string readConsistencyLevel 10: required double schemaRegionPerDataNode - 11: required double dataRegionPerProcessor + 11: required double dataRegionPerDataNode 12: required i32 seriesPartitionSlotNum 13: required string seriesPartitionExecutorClass 14: required double diskSpaceWarningThreshold
