This is an automated email from the ASF dual-hosted git repository. tanxinyu pushed a commit to branch Computing-resource-balancing_cp in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2e323d011f40d30b941c08c8c2e38932e091e8fd Author: YongzaoDan <[email protected]> AuthorDate: Fri Jul 21 22:15:29 2023 +0800 Fix UT and IT --- .../it/partition/IoTDBPartitionCreationIT.java | 176 --------------------- .../manager/load/balancer/PartitionBalancer.java | 12 +- .../load/balancer/partition/DataAllotTable.java | 18 +-- .../statemachine/CreateRegionGroupsProcedure.java | 6 +- .../request/ConfigPhysicalPlanSerDeTest.java | 6 +- .../impl/CreateRegionGroupsProcedureTest.java | 5 +- .../commons/partition/SeriesPartitionTable.java | 21 +-- 7 files changed, 38 insertions(+), 206 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java index 33844dcf48c..dba7b812814 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java @@ -419,182 +419,6 @@ public class IoTDBPartitionCreationIT { 4 + testTimePartitionBatchSize, testTimePartitionInterval, dataPartitionTableResp.getDataPartitionTable()); - - // RegionGroup statistics: - // 0: 1 Removing, 1 partition - // 1: 1 ReadOnly, 1 partition - // 2: 1 Unknown, 1 partition - // 3: All Running, 1 partition - // Least Region Group number per storageGroup = 4, match the current Region Group number - // Will allocate the new partition to Running RegionGroup 3, DataNodes: [1, 2, 6] - showRegionResp = client.showRegion(new TShowRegionReq()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); - for (TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { - if (regionInfo.getDataNodeId() == 6) { - Assert.assertEquals(regionInfo.getTimeSlots(), 2); - } - } - - partitionSlotsMap = - ConfigNodeTestUtils.constructPartitionSlotsMap( - sg, - 5, - 5 + testSeriesPartitionBatchSize, - 5, - 5 + testTimePartitionBatchSize, - testTimePartitionInterval); - dataPartitionReq = new TDataPartitionReq(partitionSlotsMap); - for (int retry = 0; retry < 5; retry++) { - // Build new Client since it's unstable in Win8 environment - try (SyncConfigNodeIServiceClient configNodeClient = - (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - dataPartitionTableResp = configNodeClient.getOrCreateDataPartitionTable(dataPartitionReq); - if (dataPartitionTableResp != null) { - break; - } - } catch (Exception e) { - // Retry sometimes in order to avoid request timeout - LOGGER.error(e.getMessage()); - TimeUnit.SECONDS.sleep(1); - } - } - Assert.assertNotNull(dataPartitionTableResp); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - dataPartitionTableResp.getStatus().getCode()); - Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable()); - ConfigNodeTestUtils.checkDataPartitionTable( - sg, - 5, - 5 + testSeriesPartitionBatchSize, - 5, - 5 + testTimePartitionBatchSize, - testTimePartitionInterval, - dataPartitionTableResp.getDataPartitionTable()); - - // RegionGroup statistics: - // 0: 1 Removing, 1 partition - // 1: 1 ReadOnly, 1 partition - // 2: 1 Unknown, 1 partition - // 3: All Running, 2 partition - // Least Region Group number per storageGroup = 4, match the current Region Group number - // Will allocate the new partition to available RegionGroup 2, DataNodes: [1, 2, 5] - showRegionResp = client.showRegion(new TShowRegionReq()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); - for (TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { - if (regionInfo.getDataNodeId() == 5) { - Assert.assertEquals(regionInfo.getTimeSlots(), 2); - } - } - - partitionSlotsMap = - ConfigNodeTestUtils.constructPartitionSlotsMap( - sg, - 6, - 6 + testSeriesPartitionBatchSize, - 6, - 6 + testTimePartitionBatchSize, - testTimePartitionInterval); - dataPartitionReq = new TDataPartitionReq(partitionSlotsMap); - for (int retry = 0; retry < 5; retry++) { - // Build new Client since it's unstable in Win8 environment - try (SyncConfigNodeIServiceClient configNodeClient = - (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - dataPartitionTableResp = configNodeClient.getOrCreateDataPartitionTable(dataPartitionReq); - if (dataPartitionTableResp != null) { - break; - } - } catch (Exception e) { - // Retry sometimes in order to avoid request timeout - LOGGER.error(e.getMessage()); - TimeUnit.SECONDS.sleep(1); - } - } - Assert.assertNotNull(dataPartitionTableResp); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - dataPartitionTableResp.getStatus().getCode()); - Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable()); - ConfigNodeTestUtils.checkDataPartitionTable( - sg, - 6, - 6 + testSeriesPartitionBatchSize, - 6, - 6 + testTimePartitionBatchSize, - testTimePartitionInterval, - dataPartitionTableResp.getDataPartitionTable()); - - // RegionGroup statistics: - // 0: 1 Removing, 1 partition - // 1: 1 ReadOnly, 1 partition - // 2: 1 Unknown, 2 partition - // 3: All Running, 2 partition - // Least Region Group number per storageGroup = 4, match the current Region Group number - // Will allocate the new partition to Discouraged RegionGroup 2, DataNodes: [1, 2, 4] - showRegionResp = client.showRegion(new TShowRegionReq()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); - for (TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { - if (regionInfo.getDataNodeId() == 4) { - Assert.assertEquals(regionInfo.getTimeSlots(), 2); - } - } - - partitionSlotsMap = - ConfigNodeTestUtils.constructPartitionSlotsMap( - sg, - 7, - 7 + testSeriesPartitionBatchSize, - 7, - 7 + testTimePartitionBatchSize, - testTimePartitionInterval); - dataPartitionReq = new TDataPartitionReq(partitionSlotsMap); - for (int retry = 0; retry < 5; retry++) { - // Build new Client since it's unstable in Win8 environment - try (SyncConfigNodeIServiceClient configNodeClient = - (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - dataPartitionTableResp = configNodeClient.getOrCreateDataPartitionTable(dataPartitionReq); - if (dataPartitionTableResp != null) { - break; - } - } catch (Exception e) { - // Retry sometimes in order to avoid request timeout - LOGGER.error(e.getMessage()); - TimeUnit.SECONDS.sleep(1); - } - } - Assert.assertNotNull(dataPartitionTableResp); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - dataPartitionTableResp.getStatus().getCode()); - Assert.assertNotNull(dataPartitionTableResp.getDataPartitionTable()); - ConfigNodeTestUtils.checkDataPartitionTable( - sg, - 7, - 7 + testSeriesPartitionBatchSize, - 7, - 7 + testTimePartitionBatchSize, - testTimePartitionInterval, - dataPartitionTableResp.getDataPartitionTable()); - - // RegionGroup statistics: - // 0: 1 Removing, 1 partition - // 1: 1 ReadOnly, 2 partition - // 2: 1 Unknown, 2 partition - // 3: All Running, 2 partition - // Least Region Group number per storageGroup = 4, match the current Region Group number - // Will allocate the new partition to Running RegionGroup 3, DataNodes: [0, 1, 5] - // Because RegionGroup 1 is Disabled - showRegionResp = client.showRegion(new TShowRegionReq()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); - for (TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { - if (regionInfo.getDataNodeId() == 6) { - Assert.assertEquals(regionInfo.getTimeSlots(), 3); - } - } } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/PartitionBalancer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/PartitionBalancer.java index 963ec39edd8..c0c617cd795 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/PartitionBalancer.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/PartitionBalancer.java @@ -215,7 +215,7 @@ public class PartitionBalancer { } /** - * Update the DataPartitionCount in DataAllotTable + * Update the DataPartitionCount in DataAllotTable. * * @param database Database name * @param dataPartitionTable new created DataPartitionTable @@ -231,13 +231,13 @@ public class PartitionBalancer { .updateCurrentTimePartition( getPartitionManager().getRegionGroupCount(database, TConsensusGroupType.DataRegion)); } catch (DatabaseNotExistsException e) { - LOGGER.error("Database {} not exists", database); + LOGGER.error("Database {} not exists when updateDataPartitionCount", database); return false; } } /** - * Update the DataAllotTable + * Update the DataAllotTable. * * @param database Database name */ @@ -265,7 +265,7 @@ public class PartitionBalancer { getPartitionManager().getAllRegionGroupIds(database, TConsensusGroupType.DataRegion), allocatedTable); } catch (DatabaseNotExistsException e) { - LOGGER.error("Database {} not exists", database); + LOGGER.error("Database {} not exists when updateDataAllotTable", database); } } @@ -302,10 +302,10 @@ public class PartitionBalancer { currentTimePartition.getStartTime() - TIME_PARTITION_INTERVAL); } - dataAllotTable.setDataAllotTable( + dataAllotTable.setDataAllotMap( getPartitionManager().getLastDataAllotTable(database)); } catch (DatabaseNotExistsException e) { - LOGGER.error("Database {} not exists", database); + LOGGER.error("Database {} not exists when setupPartitionBalancer", database); } finally { dataAllotTable.releaseWriteLock(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataAllotTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataAllotTable.java index 0507272f6c3..dec450d20d9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataAllotTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataAllotTable.java @@ -50,19 +50,19 @@ public class DataAllotTable { private final TreeMap<TTimePartitionSlot, AtomicInteger> dataPartitionCounter; // Map<SeriesPartitionSlot, RegionGroupId> // The optimal allocation of SeriesSlots to RegionGroups in the currentTimePartition - private final Map<TSeriesPartitionSlot, TConsensusGroupId> dataAllotTable; + private final Map<TSeriesPartitionSlot, TConsensusGroupId> dataAllotMap; public DataAllotTable() { this.dataAllotTableLock = new ReentrantReadWriteLock(); this.currentTimePartition = new AtomicReference<>(new TTimePartitionSlot(0)); this.dataPartitionCounter = new TreeMap<>(); - this.dataAllotTable = new HashMap<>(); + this.dataAllotMap = new HashMap<>(); } public boolean isEmpty() { dataAllotTableLock.readLock().lock(); try { - return dataAllotTable.isEmpty(); + return dataAllotMap.isEmpty(); } finally { dataAllotTableLock.readLock().unlock(); } @@ -109,7 +109,7 @@ public class DataAllotTable { continue; } - TConsensusGroupId oldRegionGroupId = dataAllotTable.get(seriesPartitionSlot); + TConsensusGroupId oldRegionGroupId = dataAllotMap.get(seriesPartitionSlot); if (oldRegionGroupId != null && counter.containsKey(oldRegionGroupId) && counter.get(oldRegionGroupId) < mu) { @@ -125,8 +125,8 @@ public class DataAllotTable { counter.put(newRegionGroupId, counter.get(newRegionGroupId) + 1); } - dataAllotTable.clear(); - dataAllotTable.putAll(newAllotTable); + dataAllotMap.clear(); + dataAllotMap.putAll(newAllotTable); } finally { dataAllotTableLock.writeLock().unlock(); } @@ -187,7 +187,7 @@ public class DataAllotTable { public TConsensusGroupId getRegionGroupId(TSeriesPartitionSlot seriesPartitionSlot) { dataAllotTableLock.readLock().lock(); try { - return dataAllotTable.get(seriesPartitionSlot); + return dataAllotMap.get(seriesPartitionSlot); } finally { dataAllotTableLock.readLock().unlock(); } @@ -199,8 +199,8 @@ public class DataAllotTable { } /** Only use this interface when init PartitionBalancer. */ - public void setDataAllotTable(Map<TSeriesPartitionSlot, TConsensusGroupId> dataAllotTable) { - this.dataAllotTable.putAll(dataAllotTable); + public void setDataAllotMap(Map<TSeriesPartitionSlot, TConsensusGroupId> dataAllotMap) { + this.dataAllotMap.putAll(dataAllotMap); } public static int timePartitionThreshold(int regionGroupNum) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java index 279a5e8c872..0aeb4f056af 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/statemachine/CreateRegionGroupsProcedure.java @@ -53,7 +53,7 @@ public class CreateRegionGroupsProcedure private TConsensusGroupType consensusGroupType; private CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); - private CreateRegionGroupsPlan persistPlan; + private CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); /** key: TConsensusGroupId value: Failed RegionReplicas */ private Map<TConsensusGroupId, TRegionReplicaSet> failedRegionReplicaSets = new HashMap<>(); @@ -66,18 +66,18 @@ public class CreateRegionGroupsProcedure TConsensusGroupType consensusGroupType, CreateRegionGroupsPlan createRegionGroupsPlan) { this.consensusGroupType = consensusGroupType; this.createRegionGroupsPlan = createRegionGroupsPlan; - this.persistPlan = new CreateRegionGroupsPlan(); } @TestOnly public CreateRegionGroupsProcedure( TConsensusGroupType consensusGroupType, CreateRegionGroupsPlan createRegionGroupsPlan, + CreateRegionGroupsPlan persistPlan, Map<TConsensusGroupId, TRegionReplicaSet> failedRegionReplicaSets) { this.consensusGroupType = consensusGroupType; this.createRegionGroupsPlan = createRegionGroupsPlan; + this.persistPlan = persistPlan; this.failedRegionReplicaSets = failedRegionReplicaSets; - this.persistPlan = new CreateRegionGroupsPlan(); } @Override diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index e1a212c7f50..f30d46dd117 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.confignode.consensus.request; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; @@ -818,9 +819,12 @@ public class ConfigPhysicalPlanSerDeTest { CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); + CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); + persistPlan.addRegionGroup("root.sg0", dataRegionSet); + persistPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsProcedure procedure0 = new CreateRegionGroupsProcedure( - TConsensusGroupType.DataRegion, createRegionGroupsPlan, failedRegions); + TConsensusGroupType.DataRegion, createRegionGroupsPlan, persistPlan, failedRegions); updateProcedurePlan0.setProcedure(procedure0); updateProcedurePlan1 = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java index 0642bf6d6ab..9cd9d41301c 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java @@ -93,10 +93,13 @@ public class CreateRegionGroupsProcedureTest { CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); + CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); + persistPlan.addRegionGroup("root.sg0", dataRegionSet); + persistPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsProcedure procedure0 = new CreateRegionGroupsProcedure( - TConsensusGroupType.DataRegion, createRegionGroupsPlan, failedRegions0); + TConsensusGroupType.DataRegion, createRegionGroupsPlan, persistPlan, failedRegions0); PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java index 00edd8bcb89..d72157a5037 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.commons.partition; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; @@ -67,7 +68,7 @@ public class SeriesPartitionTable { } /** - * Thread-safely get DataPartition within the specific StorageGroup + * Thread-safely get DataPartition within the specific Database. * * @param partitionSlotList TimePartitionSlotList * @param seriesPartitionTable Store the matched SeriesPartitions @@ -126,7 +127,7 @@ public class SeriesPartitionTable { } /** - * Check and return the specified DataPartition's successor + * Check and return the specified DataPartition's successor. * * @param timePartitionSlot Corresponding TimePartitionSlot * @return The specified DataPartition's successor if exists, null otherwise @@ -137,7 +138,7 @@ public class SeriesPartitionTable { } /** - * Query a timePartition's corresponding dataRegionIds + * Query a timePartition's corresponding dataRegionIds. * * @param timeSlotId Time partition's timeSlotId * @return the timePartition's corresponding dataRegionIds @@ -170,7 +171,7 @@ public class SeriesPartitionTable { } /** - * Create DataPartition within the specific SeriesPartitionSlot + * Create DataPartition within the specific SeriesPartitionSlot. * * @param assignedSeriesPartitionTable Assigned result * @param seriesPartitionSlot Corresponding TSeriesPartitionSlot @@ -197,7 +198,7 @@ public class SeriesPartitionTable { /** * Only Leader use this interface. And this interface is synchronized. Thread-safely filter no - * assigned DataPartitionSlots within the specific SeriesPartitionSlot + * assigned DataPartitionSlots within the specific SeriesPartitionSlot. * * @param partitionSlots TimePartitionSlots * @return Unassigned PartitionSlots @@ -259,7 +260,7 @@ public class SeriesPartitionTable { } /** - * Check whether the specified DataPartition exists + * Check whether the specified DataPartition exists. * * @param timePartitionSlot Corresponding TimePartitionSlot * @return 1 if exists, 0 otherwise @@ -269,7 +270,7 @@ public class SeriesPartitionTable { } /** - * Get the last DataPartition's ConsensusGroupId + * Get the last DataPartition's ConsensusGroupId. * * @return The last DataPartition's ConsensusGroupId, null if there are no DataPartitions yet */ @@ -283,7 +284,7 @@ public class SeriesPartitionTable { } /** - * Get the number of DataPartitions in each TimePartitionSlot + * Get the number of DataPartitions in each TimePartitionSlot. * * @return Map<TimePartitionSlot, the number of DataPartitions> */ @@ -307,7 +308,7 @@ public class SeriesPartitionTable { } } - /** Only for ConsensusRequest */ + /** Only for ConsensusRequest. */ public void deserialize(ByteBuffer buffer) { int timePartitionSlotNum = buffer.getInt(); for (int i = 0; i < timePartitionSlotNum; i++) { @@ -324,7 +325,7 @@ public class SeriesPartitionTable { } } - /** Only for Snapshot */ + /** Only for Snapshot. */ public void deserialize(InputStream inputStream, TProtocol protocol) throws IOException, TException { int timePartitionSlotNum = ReadWriteIOUtils.readInt(inputStream);
