This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch region-multi-database
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/region-multi-database by this
push:
new 9d39c710e27 manual test template
9d39c710e27 is described below
commit 9d39c710e2726b37bc40b20735e7373a007afd58
Author: YongzaoDan <[email protected]>
AuthorDate: Wed Mar 13 11:26:18 2024 +0800
manual test template
---
.../manager/load/balancer/RegionBalancer.java | 8 +
.../region/CopySetRegionGroupAllocator.java | 18 +-
.../region/GreedyCopySetRegionGroupAllocator.java | 33 +++-
.../region/TieredReplicationAllocator.java | 19 +-
...orAndLeaderBalancerCombinatorialManualTest.java | 6 +-
...ualTest.java => RegionAllocatorManualTest.java} | 197 ++++++++++++---------
6 files changed, 190 insertions(+), 91 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java
index 0ec62ee99c4..94889430304 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java
@@ -29,9 +29,11 @@ import
org.apache.iotdb.confignode.exception.DatabaseNotExistsException;
import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException;
import org.apache.iotdb.confignode.manager.IManager;
import org.apache.iotdb.confignode.manager.load.LoadManager;
+import
org.apache.iotdb.confignode.manager.load.balancer.region.CopySetRegionGroupAllocator;
import
org.apache.iotdb.confignode.manager.load.balancer.region.GreedyCopySetRegionGroupAllocator;
import
org.apache.iotdb.confignode.manager.load.balancer.region.GreedyRegionGroupAllocator;
import
org.apache.iotdb.confignode.manager.load.balancer.region.IRegionGroupAllocator;
+import
org.apache.iotdb.confignode.manager.load.balancer.region.TieredReplicationAllocator;
import org.apache.iotdb.confignode.manager.node.NodeManager;
import org.apache.iotdb.confignode.manager.partition.PartitionManager;
import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
@@ -55,6 +57,12 @@ public class RegionBalancer {
case GREEDY:
this.regionGroupAllocator = new GreedyRegionGroupAllocator();
break;
+ case TIERED_REPLICATION:
+ this.regionGroupAllocator = new TieredReplicationAllocator();
+ break;
+ case COPY_SET:
+ this.regionGroupAllocator = new CopySetRegionGroupAllocator();
+ break;
case GREEDY_COPY_SET:
default:
this.regionGroupAllocator = new GreedyCopySetRegionGroupAllocator();
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/CopySetRegionGroupAllocator.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/CopySetRegionGroupAllocator.java
index 7580a398e72..37bec35adcb 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/CopySetRegionGroupAllocator.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/CopySetRegionGroupAllocator.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.confignode.manager.load.balancer.region;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import java.util.ArrayList;
import java.util.BitSet;
@@ -35,9 +36,11 @@ public class CopySetRegionGroupAllocator implements
IRegionGroupAllocator {
private final Random RANDOM = new Random();
private final Map<Integer, List<List<Integer>>> COPY_SETS = new TreeMap<>();
- private final int dataNodeNum;
+ private int dataNodeNum = -1;
- public CopySetRegionGroupAllocator(int dataNodeNum, int replicationFactor,
int loadFactor) {
+ public CopySetRegionGroupAllocator() {}
+
+ private void init(int dataNodeNum, int replicationFactor, int loadFactor) {
this.dataNodeNum = dataNodeNum;
BitSet bitSet = new BitSet(dataNodeNum + 1);
for (int p = 0; p < loadFactor || bitSet.cardinality() < dataNodeNum; p++)
{
@@ -51,7 +54,7 @@ public class CopySetRegionGroupAllocator implements
IRegionGroupAllocator {
permutation.set(i, permutation.get(pos));
permutation.set(pos, tmp);
}
- for (int i = 0; i + replicationFactor < permutation.size(); i +=
replicationFactor) {
+ for (int i = 0; i + replicationFactor <= permutation.size(); i +=
replicationFactor) {
List<Integer> copySet = new ArrayList<>();
for (int j = 0; j < replicationFactor; j++) {
int e = permutation.get(i + j);
@@ -73,6 +76,15 @@ public class CopySetRegionGroupAllocator implements
IRegionGroupAllocator {
List<TRegionReplicaSet> databaseAllocatedRegionGroups,
int replicationFactor,
TConsensusGroupId consensusGroupId) {
+ if (this.dataNodeNum == -1) {
+ init(
+ availableDataNodeMap.size(),
+ replicationFactor,
+
ConfigNodeDescriptor.getInstance().getConf().getDefaultDataRegionGroupNumPerDatabase()
+ * replicationFactor
+ / availableDataNodeMap.size());
+ }
+
TRegionReplicaSet result = new TRegionReplicaSet();
Map<Integer, Integer> regionCounter = new TreeMap<>();
for (int i = 1; i <= dataNodeNum; i++) {
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/GreedyCopySetRegionGroupAllocator.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/GreedyCopySetRegionGroupAllocator.java
index 3066f64d38c..945c6378d58 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/GreedyCopySetRegionGroupAllocator.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/GreedyCopySetRegionGroupAllocator.java
@@ -49,6 +49,7 @@ public class GreedyCopySetRegionGroupAllocator implements
IRegionGroupAllocator
private int[] databaseRegionCounter;
// The number of 2-Region combinations in current cluster
private int[][] combinationCounter;
+ private int maxDataNodeId;
// First Key: the sum of Regions at the DataNodes within the same Database
in the allocation
// result is minimal
@@ -58,6 +59,7 @@ public class GreedyCopySetRegionGroupAllocator implements
IRegionGroupAllocator
// Third Key: the sum of overlapped 2-Region combination Regions with other
allocated
// RegionGroups is minimal
int optimalCombinationSum;
+ int optimalIdXorSum;
List<int[]> optimalReplicaSets;
private static final int MAX_OPTIMAL_PLAN_NUM = 1000;
@@ -157,6 +159,7 @@ public class GreedyCopySetRegionGroupAllocator implements
IRegionGroupAllocator
.mapToInt(TDataNodeLocation::getDataNodeId)
.max()
.orElse(0));
+ this.maxDataNodeId = maxDataNodeId;
// Compute regionCounter, databaseRegionCounter and combinationCounter
regionCounter = new int[maxDataNodeId + 1];
@@ -221,6 +224,7 @@ public class GreedyCopySetRegionGroupAllocator implements
IRegionGroupAllocator
optimalDatabaseRegionSum = Integer.MAX_VALUE;
optimalRegionSum = Integer.MAX_VALUE;
optimalCombinationSum = Integer.MAX_VALUE;
+ optimalIdXorSum = Integer.MAX_VALUE;
optimalReplicaSets = new ArrayList<>();
}
@@ -262,17 +266,42 @@ public class GreedyCopySetRegionGroupAllocator implements
IRegionGroupAllocator
combinationSum +=
combinationCounter[currentReplicaSet[i]][currentReplicaSet[j]];
}
}
+ if (databaseRegionSum == optimalDatabaseRegionSum
+ && regionSum == optimalRegionSum
+ && combinationSum > optimalCombinationSum) {
+ return;
+ }
+
+ if (maxDataNodeId < 160) {
+ if (databaseRegionSum < optimalDatabaseRegionSum
+ || regionSum < optimalRegionSum
+ || combinationSum < optimalCombinationSum) {
+ // Reset the optimal result when a better one is found
+ optimalDatabaseRegionSum = databaseRegionSum;
+ optimalRegionSum = regionSum;
+ optimalCombinationSum = combinationSum;
+ optimalReplicaSets.clear();
+ }
+ optimalReplicaSets.add(Arrays.copyOf(currentReplicaSet,
replicationFactor));
+ return;
+ }
+ int idXorSum = 0;
+ for (int id : currentReplicaSet) {
+ idXorSum ^= id;
+ }
if (databaseRegionSum < optimalDatabaseRegionSum
|| regionSum < optimalRegionSum
- || combinationSum < optimalCombinationSum) {
+ || combinationSum < optimalCombinationSum
+ || (optimalIdXorSum > 0 && idXorSum == 0)) {
// Reset the optimal result when a better one is found
optimalDatabaseRegionSum = databaseRegionSum;
optimalRegionSum = regionSum;
optimalCombinationSum = combinationSum;
+ optimalIdXorSum = idXorSum;
optimalReplicaSets.clear();
}
- if (combinationSum == optimalCombinationSum) {
+ if (idXorSum == 0 || (idXorSum > 0 && optimalIdXorSum > 0)) {
optimalReplicaSets.add(Arrays.copyOf(currentReplicaSet,
replicationFactor));
}
return;
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/TieredReplicationAllocator.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/TieredReplicationAllocator.java
index ab349e6f215..c18e8081a50 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/TieredReplicationAllocator.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/TieredReplicationAllocator.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.confignode.manager.load.balancer.region;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import java.util.ArrayList;
import java.util.BitSet;
@@ -33,7 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class TieredReplicationAllocator implements IRegionGroupAllocator {
- private final int dataNodeNum;
+ private int dataNodeNum = -1;
private final Random RANDOM = new Random();
private final Map<Integer, List<List<Integer>>> COPY_SETS = new TreeMap<>();
@@ -56,13 +57,13 @@ public class TieredReplicationAllocator implements
IRegionGroupAllocator {
}
}
- public TieredReplicationAllocator(int dataNodeNum, int replicationFactor,
int loadFactor) {
+ private void init(int dataNodeNum, int replicationFactor, int loadFactor) {
this.dataNodeNum = dataNodeNum;
Map<Integer, BitSet> scatterWidthMap = new TreeMap<>();
for (int i = 1; i <= dataNodeNum; i++) {
scatterWidthMap.put(i, new BitSet(dataNodeNum + 1));
}
- int targetScatterWidth = loadFactor * (replicationFactor - 1);
+ int targetScatterWidth = Math.min(loadFactor * (replicationFactor - 1),
dataNodeNum - 1);
while (existScatterWidthUnsatisfied(scatterWidthMap, targetScatterWidth)) {
for (int firstRegion = 1; firstRegion <= dataNodeNum; firstRegion++) {
if (scatterWidthMap.get(firstRegion).cardinality() <
targetScatterWidth) {
@@ -109,11 +110,14 @@ public class TieredReplicationAllocator implements
IRegionGroupAllocator {
for (int e : copySet) {
COPY_SETS.computeIfAbsent(e, k -> new ArrayList<>()).add(copySet);
}
+ break;
}
}
}
}
+ public TieredReplicationAllocator() {}
+
private boolean existScatterWidthUnsatisfied(
Map<Integer, BitSet> scatterWidthMap, int targetScatterWidth) {
AtomicBoolean result = new AtomicBoolean(false);
@@ -134,6 +138,15 @@ public class TieredReplicationAllocator implements
IRegionGroupAllocator {
List<TRegionReplicaSet> databaseAllocatedRegionGroups,
int replicationFactor,
TConsensusGroupId consensusGroupId) {
+ if (this.dataNodeNum == -1) {
+ init(
+ availableDataNodeMap.size(),
+ replicationFactor,
+
ConfigNodeDescriptor.getInstance().getConf().getDefaultDataRegionGroupNumPerDatabase()
+ * replicationFactor
+ / availableDataNodeMap.size());
+ }
+
TRegionReplicaSet result = new TRegionReplicaSet();
Map<Integer, Integer> regionCounter = new TreeMap<>();
for (int i = 1; i <= dataNodeNum; i++) {
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorAndLeaderBalancerCombinatorialManualTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorAndLeaderBalancerCombinatorialManualTest.java
index d8822f33079..edff2abe674 100644
---
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorAndLeaderBalancerCombinatorialManualTest.java
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorAndLeaderBalancerCombinatorialManualTest.java
@@ -31,8 +31,8 @@ public class
RegionAllocatorAndLeaderBalancerCombinatorialManualTest {
LoggerFactory.getLogger(RegionAllocatorAndLeaderBalancerCombinatorialManualTest.class);
private static final int TEST_LOOP = 1;
- private static final int TEST_DATA_NODE_NUM = 100;
- private static final int DATA_REGION_PER_DATA_NODE = 4;
+ private static final int TEST_DATA_NODE_NUM = 8;
+ private static final int DATA_REGION_PER_DATA_NODE = 6;
private static final int DATA_REPLICATION_FACTOR = 3;
private static final String DATABASE = "root.db";
@@ -41,8 +41,6 @@ public class
RegionAllocatorAndLeaderBalancerCombinatorialManualTest {
private static final Map<Integer, Double> FREE_SPACE_MAP = new TreeMap<>();
private static final IRegionGroupAllocator ALLOCATOR = new
GreedyCopySetRegionGroupAllocator();
- // new TieredReplicationAllocator(
- // TEST_DATA_NODE_NUM, DATA_REPLICATION_FACTOR,
DATA_REGION_PER_DATA_NODE);
private static final ILeaderBalancer BALANCER = new
MinCostFlowLeaderBalancer();
@BeforeClass
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorAndLeaderBalancerCombinatorialManualTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorManualTest.java
similarity index 57%
copy from
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorAndLeaderBalancerCombinatorialManualTest.java
copy to
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorManualTest.java
index d8822f33079..c61e580045c 100644
---
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorAndLeaderBalancerCombinatorialManualTest.java
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionAllocatorManualTest.java
@@ -5,64 +5,125 @@ import
org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-import
org.apache.iotdb.confignode.manager.load.balancer.region.GreedyCopySetRegionGroupAllocator;
+import
org.apache.iotdb.confignode.manager.load.balancer.region.GreedyRegionGroupAllocator;
import
org.apache.iotdb.confignode.manager.load.balancer.region.IRegionGroupAllocator;
import
org.apache.iotdb.confignode.manager.load.balancer.router.leader.ILeaderBalancer;
-import
org.apache.iotdb.confignode.manager.load.balancer.router.leader.MinCostFlowLeaderBalancer;
+import
org.apache.iotdb.confignode.manager.load.balancer.router.leader.RandomLeaderBalancer;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
-public class RegionAllocatorAndLeaderBalancerCombinatorialManualTest {
+public class RegionAllocatorManualTest {
- private static final Logger LOGGER =
-
LoggerFactory.getLogger(RegionAllocatorAndLeaderBalancerCombinatorialManualTest.class);
+ private Logger LOGGER =
LoggerFactory.getLogger(RegionAllocatorManualTest.class);
- private static final int TEST_LOOP = 1;
- private static final int TEST_DATA_NODE_NUM = 100;
- private static final int DATA_REGION_PER_DATA_NODE = 4;
+ private static final int TEST_LOOP = 10;
+ private static final int MIN_DATA_NODE_NUM = 3;
+ private static final int MAX_DATA_NODE_NUM = 100;
+ private static final int MIN_DATA_REGION_PER_DATA_NODE = 6;
+ private static final int MAX_DATA_REGION_PER_DATA_NODE = 6;
private static final int DATA_REPLICATION_FACTOR = 3;
+ private static final double EXAM_LOOP = 1000;
private static final String DATABASE = "root.db";
private static final Map<Integer, TDataNodeConfiguration>
AVAILABLE_DATA_NODE_MAP =
new TreeMap<>();
private static final Map<Integer, Double> FREE_SPACE_MAP = new TreeMap<>();
- private static final IRegionGroupAllocator ALLOCATOR = new
GreedyCopySetRegionGroupAllocator();
- // new TieredReplicationAllocator(
- // TEST_DATA_NODE_NUM, DATA_REPLICATION_FACTOR,
DATA_REGION_PER_DATA_NODE);
- private static final ILeaderBalancer BALANCER = new
MinCostFlowLeaderBalancer();
+ // private static final IRegionGroupAllocator ALLOCATOR = new
GreedyRegionGroupAllocator();
+ private static final ILeaderBalancer BALANCER = new RandomLeaderBalancer();
+
+ private static class DataEntry {
+ public final Integer countRange;
+ public final Integer minScatterWidth;
+ public final List<Double> disabledPercent;
+ public final Integer leaderRange;
+
+ private DataEntry(
+ int countRange, int minScatterWidth, List<Double> disabledPercent, int
leaderRange) {
+ this.countRange = countRange;
+ this.minScatterWidth = minScatterWidth;
+ this.disabledPercent = disabledPercent;
+ this.leaderRange = leaderRange;
+ }
+ }
+
+ @Test
+ public void loopTest() throws IOException {
+ List<DataEntry> testResult = new ArrayList<>();
+ for (int dataNodeNum = MIN_DATA_NODE_NUM; dataNodeNum <=
MAX_DATA_NODE_NUM; dataNodeNum++) {
+ for (int dataRegionPerDataNode = MIN_DATA_REGION_PER_DATA_NODE;
+ dataRegionPerDataNode <= MAX_DATA_REGION_PER_DATA_NODE;
+ dataRegionPerDataNode++) {
+ testResult.add(test(dataNodeNum, dataRegionPerDataNode));
+ }
+ LOGGER.info("{}, finish", dataNodeNum);
+ }
+
+ final String path = "/Users/yongzaodan/Desktop/simulation/";
+ String allocatorPath = "Copyset";
+ // FileWriter countW = new FileWriter(path + "count/" + allocatorPath +
".txt");
+ FileWriter scatterW = new FileWriter(path + "scatter/" + allocatorPath);
+ // FileWriter percentW = new FileWriter(path + "percent/" + allocatorPath
+ ".txt");
+ // FileWriter leaderW = new FileWriter(path + "leader/" +
BALANCER.getClass().getSimpleName() +
+ // ".txt");
+
+ for (DataEntry entry : testResult) {
+ // countW.write(entry.countRange.toString()); countW.write("\n");
countW.flush();
+ scatterW.write(entry.minScatterWidth.toString());
+ scatterW.write("\n");
+ scatterW.flush();
+ // for (Double percent : entry.disabledPercent) {
+ // percentW.write(percent.toString());
+ // percentW.write("\n");
+ // percentW.flush();
+ // }
+ // leaderW.write(entry.leaderRange.toString());
+ // leaderW.write("\n");
+ // leaderW.flush();
+ }
+
+ // countW.close();
+ scatterW.close();
+ // percentW.close();
+ // leaderW.close();
+ }
+
+ public DataEntry test(int TEST_DATA_NODE_NUM, int DATA_REGION_PER_DATA_NODE)
{
+ IRegionGroupAllocator ALLOCATOR = new GreedyRegionGroupAllocator();
- @BeforeClass
- public static void setUp() {
// Construct TEST_DATA_NODE_NUM DataNodes
Random random = new Random();
+ AVAILABLE_DATA_NODE_MAP.clear();
+ FREE_SPACE_MAP.clear();
for (int i = 1; i <= TEST_DATA_NODE_NUM; i++) {
AVAILABLE_DATA_NODE_MAP.put(
i, new TDataNodeConfiguration().setLocation(new
TDataNodeLocation().setDataNodeId(i)));
FREE_SPACE_MAP.put(i, random.nextDouble());
}
- }
- @Test
- public void manualTest() {
final int dataRegionGroupNum =
DATA_REGION_PER_DATA_NODE * TEST_DATA_NODE_NUM /
DATA_REPLICATION_FACTOR;
List<Integer> regionCountList = new ArrayList<>();
List<Integer> scatterWidthList = new ArrayList<>();
List<Integer> leaderCountList = new ArrayList<>();
+ int[] hitList = new int[20];
+ Arrays.fill(hitList, 0);
for (int loop = 1; loop <= TEST_LOOP; loop++) {
/* Allocate RegionGroup */
List<TRegionReplicaSet> allocateResult = new ArrayList<>();
@@ -77,6 +138,36 @@ public class
RegionAllocatorAndLeaderBalancerCombinatorialManualTest {
new TConsensusGroupId(TConsensusGroupType.DataRegion, index)));
}
+ for (int M = 1; M <= 0.1 * TEST_DATA_NODE_NUM; M++) {
+ for (int exam = 0; exam < EXAM_LOOP; exam++) {
+ Set<Integer> examSet = new TreeSet<>();
+ while (examSet.size() < DATA_REPLICATION_FACTOR) {
+ int id = random.nextInt(TEST_DATA_NODE_NUM) + 1;
+ while (examSet.contains(id)) {
+ id = random.nextInt(TEST_DATA_NODE_NUM) + 1;
+ }
+ examSet.add(id);
+ }
+ for (TRegionReplicaSet result : allocateResult) {
+ Set<Integer> scheme =
+ result.getDataNodeLocations().stream()
+ .map(TDataNodeLocation::getDataNodeId)
+ .collect(Collectors.toSet());
+ boolean isHit = true;
+ for (int id : examSet) {
+ if (!scheme.contains(id)) {
+ isHit = false;
+ break;
+ }
+ }
+ if (isHit) {
+ ++hitList[M];
+ break;
+ }
+ }
+ }
+ }
+
/* Count Region in each DataNode */
// Map<DataNodeId, RegionGroup Count>
Map<Integer, Integer> regionCounter = new TreeMap<>();
@@ -116,19 +207,6 @@ public class
RegionAllocatorAndLeaderBalancerCombinatorialManualTest {
regionCountList.add(regionCounter.getOrDefault(i, 0));
scatterWidthList.add(scatterWidth);
}
- // LOGGER.info(
- // "Loop: {}, Test :{}, {}",
- // loop,
- // ALLOCATOR.getClass().getSimpleName(),
- // BALANCER.getClass().getSimpleName());
- // LOGGER.info(
- // "Allocate {} DataRegionGroups for {} DataNodes",
dataRegionGroupNum,
- // TEST_DATA_NODE_NUM);
- // LOGGER.info(
- // "Scatter width avg: {}, min: {}, max: {}",
- // (double) scatterWidthSum / TEST_DATA_NODE_NUM,
- // minScatterWidth,
- // maxScatterWidth);
/* Balance Leader */
Map<String, List<TConsensusGroupId>> databaseRegionGroupMap =
@@ -149,57 +227,18 @@ public class
RegionAllocatorAndLeaderBalancerCombinatorialManualTest {
int minLeaderCount =
leaderCounter.values().stream().min(Integer::compareTo).orElse(0);
int maxLeaderCount =
leaderCounter.values().stream().max(Integer::compareTo).orElse(0);
leaderCounter.forEach((dataNodeId, leaderCount) ->
leaderCountList.add(leaderCount));
- // LOGGER.info("Leader count min: {}, max: {}", minLeaderCount,
maxLeaderCount);
}
- LOGGER.info("All tests done.");
- double regionCountAvg =
-
regionCountList.stream().mapToInt(Integer::intValue).average().orElse(0);
- double regionCountVariance =
- regionCountList.stream()
- .mapToInt(Integer::intValue)
- .mapToDouble(i -> Math.pow(i - regionCountAvg, 2))
- .sum()
- / regionCountList.size();
- int regionCountRange =
+ List<Double> percentList = new ArrayList<>();
+ for (int M = 1; M <= 10; M++) {
+ percentList.add(hitList[M] / EXAM_LOOP / (double) TEST_LOOP);
+ }
+ return new DataEntry(
regionCountList.stream().mapToInt(Integer::intValue).max().orElse(0)
- -
regionCountList.stream().mapToInt(Integer::intValue).min().orElse(0);
- LOGGER.info(
- "Region count avg: {}, var: {}, range: {}",
- regionCountAvg,
- regionCountVariance,
- regionCountRange);
- double scatterWidthAvg =
-
scatterWidthList.stream().mapToInt(Integer::intValue).average().orElse(0);
- double scatterWidthVariance =
- scatterWidthList.stream()
- .mapToInt(Integer::intValue)
- .mapToDouble(i -> Math.pow(i - scatterWidthAvg, 2))
- .sum()
- / scatterWidthList.size();
- int scatterWidthRange =
- scatterWidthList.stream().mapToInt(Integer::intValue).max().orElse(0)
- -
scatterWidthList.stream().mapToInt(Integer::intValue).min().orElse(0);
- LOGGER.info(
- "Scatter width avg: {}, var: {}, range: {}",
- scatterWidthAvg,
- scatterWidthVariance,
- scatterWidthRange);
- double leaderCountAvg =
-
leaderCountList.stream().mapToInt(Integer::intValue).average().orElse(0);
- double leaderCountVariance =
- leaderCountList.stream()
- .mapToInt(Integer::intValue)
- .mapToDouble(i -> Math.pow(i - leaderCountAvg, 2))
- .sum()
- / leaderCountList.size();
- int leaderCountRange =
+ -
regionCountList.stream().mapToInt(Integer::intValue).min().orElse(0),
+ scatterWidthList.stream().mapToInt(Integer::intValue).min().orElse(0),
+ percentList,
leaderCountList.stream().mapToInt(Integer::intValue).max().orElse(0)
- -
leaderCountList.stream().mapToInt(Integer::intValue).min().orElse(0);
- LOGGER.info(
- "Leader count avg: {}, var: {}, range: {}",
- leaderCountAvg,
- leaderCountVariance,
- leaderCountRange);
+ -
leaderCountList.stream().mapToInt(Integer::intValue).min().orElse(0));
}
}
