This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new d6e531505 Fix Empty IdealState Calculation with Disabled Nodes in
AutoRebalance… (#2877)
d6e531505 is described below
commit d6e53150583dfc3066ac69f7a25811f72cf0c37f
Author: Xiaxuan Gao <[email protected]>
AuthorDate: Thu Sep 5 09:15:58 2024 -0700
Fix Empty IdealState Calculation with Disabled Nodes in AutoRebalance…
(#2877)
Fix IdealState Calculation in AutoRebalanceStrategy for Disabled Nodes
---
.../rebalancer/strategy/AutoRebalanceStrategy.java | 95 +++--
.../controller/strategy/AutoRebalanceStrategy.java | 1 -
.../src/test/java/org/apache/helix/TestHelper.java | 29 ++
.../rebalancer/TestAutoRebalanceStrategy.java | 416 +++++++++++++++++++--
...stAutoRebalanceStrategyImbalanceAssignment.java | 7 +-
.../integration/rebalancer/TestAutoRebalance.java | 2 +-
6 files changed, 490 insertions(+), 60 deletions(-)
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/strategy/AutoRebalanceStrategy.java
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/strategy/AutoRebalanceStrategy.java
index 68279c81f..410d5ba59 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/strategy/AutoRebalanceStrategy.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/strategy/AutoRebalanceStrategy.java
@@ -34,6 +34,8 @@ import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.helix.HelixManager;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
import org.slf4j.Logger;
@@ -82,7 +84,6 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
@Override
public ZNRecord computePartitionAssignment(final List<String> allNodes,
final List<String> liveNodes,
final Map<String, Map<String, String>> currentMapping,
ResourceControllerDataProvider clusterData) {
- int numReplicas = countStateReplicas();
ZNRecord znRecord = new ZNRecord(_resourceName);
if (liveNodes.size() == 0) {
return znRecord;
@@ -97,9 +98,7 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
List<String> sortedLiveNodes = new ArrayList<String>(liveNodes);
Collections.sort(sortedLiveNodes, currentStateNodeComparator);
- int distRemainder = (numReplicas * _partitions.size()) %
sortedLiveNodes.size();
- int distFloor = (numReplicas * _partitions.size()) /
sortedLiveNodes.size();
- _nodeMap = new HashMap<String, Node>();
+ _nodeMap = new HashMap<>();
_liveNodesList = new ArrayList<Node>();
for (String id : sortedAllNodes) {
@@ -108,6 +107,10 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
node.hasCeilingCapacity = false;
_nodeMap.put(id, node);
}
+
+ int numReplicas = calculateExpectedReplicaCount(clusterData);
+ int distRemainder = (numReplicas * _partitions.size()) %
sortedLiveNodes.size();
+ int distFloor = (numReplicas * _partitions.size()) /
sortedLiveNodes.size();
for (int i = 0; i < sortedLiveNodes.size(); i++) {
boolean usingCeiling = false;
int targetSize = (_maximumPerNode > 0) ? Math.min(distFloor,
_maximumPerNode) : distFloor;
@@ -116,7 +119,8 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
distRemainder = distRemainder - 1;
usingCeiling = true;
}
- Node node = _nodeMap.get(sortedLiveNodes.get(i));
+ String nodeName = sortedLiveNodes.get(i);
+ Node node = _nodeMap.get(nodeName);
node.isAlive = true;
node.capacity = targetSize;
node.hasCeilingCapacity = usingCeiling;
@@ -127,7 +131,7 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
_stateMap = generateStateMap();
// compute the preferred mapping if all nodes were up
- _preferredAssignment = computePreferredPlacement(sortedAllNodes);
+ _preferredAssignment = computePreferredPlacement(sortedAllNodes,
clusterData);
// logger.info("preferred mapping:"+ preferredAssignment);
// from current mapping derive the ones in preferred location
@@ -135,7 +139,8 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
_existingPreferredAssignment =
computeExistingPreferredPlacement(currentMapping);
// from current mapping derive the ones not in preferred location
- _existingNonPreferredAssignment =
computeExistingNonPreferredPlacement(currentMapping);
+ _existingNonPreferredAssignment =
+ computeExistingNonPreferredPlacement(currentMapping, clusterData);
// compute orphaned replicas that are not assigned to any node
_orphaned = computeOrphaned();
@@ -152,7 +157,7 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
forceToAssignOrphans();
}
- prepareResult(znRecord);
+ prepareResult(znRecord, clusterData);
return znRecord;
}
@@ -301,7 +306,7 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
* Update a ZNRecord with the results of the rebalancing.
* @param znRecord
*/
- private void prepareResult(ZNRecord znRecord) {
+ private void prepareResult(ZNRecord znRecord, ResourceControllerDataProvider
clusterData) {
// The map fields are keyed on partition name to a pair of node and state,
i.e. it
// indicates that the partition with given state is served by that node
//
@@ -336,7 +341,10 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
}
}
}
- normalizePreferenceLists(znRecord.getListFields(), newPreferences);
+ normalizePreferenceLists(znRecord.getListFields(), newPreferences,
clusterData);
+
+ String stateModelDef =
clusterData.getIdealState(_resourceName).getStateModelDefRef();
+ StateModelDefinition stateModel =
clusterData.getStateModelDef(stateModelDef);
// generate preference maps based on the preference lists
for (String partition : _partitions) {
@@ -359,6 +367,9 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
&& receiver.currentlyAssigned < _maximumPerNode && receiver
.canAddIfCapacity(replica)) {
nodeToAssign = receiver;
+ // Should update the minOverloadedCapacity to find the node with
minimum overloaded capacity
+ minOverloadedCapacity =
+ Math.min(receiver.currentlyAssigned - receiver.capacity,
minOverloadedCapacity);
}
}
@@ -380,15 +391,15 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
* assignment
*/
private void normalizePreferenceLists(Map<String, List<String>>
preferenceLists,
- Map<String, List<String>> newPreferences) {
+ Map<String, List<String>> newPreferences, ResourceControllerDataProvider
clusterData) {
Map<String, Map<String, Integer>> nodeReplicaCounts =
new HashMap<String, Map<String, Integer>>();
for (String partition : preferenceLists.keySet()) {
- normalizePreferenceList(preferenceLists.get(partition),
nodeReplicaCounts);
+ normalizePreferenceList(preferenceLists.get(partition),
nodeReplicaCounts, clusterData);
}
for (String partition : newPreferences.keySet()) {
- normalizePreferenceList(newPreferences.get(partition),
nodeReplicaCounts);
+ normalizePreferenceList(newPreferences.get(partition),
nodeReplicaCounts, clusterData);
preferenceLists.get(partition).addAll(newPreferences.get(partition));
}
}
@@ -399,9 +410,13 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
* @param nodeReplicaCounts map of (node --> state --> count)
*/
private void normalizePreferenceList(List<String> preferenceList,
- Map<String, Map<String, Integer>> nodeReplicaCounts) {
+ Map<String, Map<String, Integer>> nodeReplicaCounts,
+ ResourceControllerDataProvider clusterData) {
List<String> newPreferenceList = new ArrayList<String>();
- int replicas = Math.min(countStateReplicas(), preferenceList.size());
+ // Use the expected replica count instead of relying on the _states map.
+ // This prevents the preference list from being truncated when
ANY_LIVEINSTANCE
+ // is used as the replication factor.
+ int replicas = Math.min(calculateExpectedReplicaCount(clusterData),
preferenceList.size());
// make this a LinkedHashSet to preserve iteration order
Set<String> notAssigned = new LinkedHashSet<String>(preferenceList);
@@ -463,14 +478,14 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
/**
* Compute the subset of the current mapping where replicas are not mapped
according to their
- * preferred assignment.
+ * existing preferred assignment.
* @param currentMapping Current mapping of replicas to nodes
* @return The current assignments that do not conform to the preferred
assignment
*/
private Map<Replica, Node> computeExistingNonPreferredPlacement(
- Map<String, Map<String, String>> currentMapping) {
+ Map<String, Map<String, String>> currentMapping,
ResourceControllerDataProvider clusterData) {
Map<Replica, Node> existingNonPreferredAssignment = new TreeMap<Replica,
Node>();
- int count = countStateReplicas();
+ int count = calculateExpectedReplicaCount(clusterData);
for (String partition : currentMapping.keySet()) {
Map<String, String> nodeStateMap = currentMapping.get(partition);
nodeStateMap.keySet().retainAll(_nodeMap.keySet());
@@ -496,12 +511,11 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
throw new IllegalArgumentException("partition: " + replica + " is
in currentMapping but not in partitions");
}
- if (_preferredAssignment.get(replica).id != node.id
+ if (!_preferredAssignment.get(replica).id.equals(node.id)
&& !_existingPreferredAssignment.containsKey(replica)
&& !existingNonPreferredAssignment.containsKey(replica)) {
existingNonPreferredAssignment.put(replica, node);
node.nonPreferred.add(replica);
-
break;
}
}
@@ -548,7 +562,7 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
private Map<Replica, Node> computeExistingPreferredPlacement(
final Map<String, Map<String, String>> currentMapping) {
Map<Replica, Node> existingPreferredAssignment = new TreeMap<Replica,
Node>();
- int count = countStateReplicas();
+ int count = calculateStatesReplicaCount();
for (String partition : currentMapping.keySet()) {
Map<String, String> nodeStateMap = currentMapping.get(partition);
nodeStateMap.keySet().retainAll(_nodeMap.keySet());
@@ -560,7 +574,7 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
Replica replica = new Replica(partition, replicaId);
if (_preferredAssignment.containsKey(replica)
&& !existingPreferredAssignment.containsKey(replica)
- && _preferredAssignment.get(replica).id == node.id) {
+ && _preferredAssignment.get(replica).id.equals(node.id)) {
existingPreferredAssignment.put(replica, node);
node.preferred.add(replica);
break;
@@ -576,16 +590,18 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
* Given a predefined set of all possible nodes, compute an assignment of
replicas to
* nodes that evenly assigns all replicas to nodes.
* @param allNodes Identifiers to all nodes, live and non-live
+ * @param clusterData
* @return Preferred assignment of replicas
*/
- private Map<Replica, Node> computePreferredPlacement(final List<String>
allNodes) {
+ private Map<Replica, Node> computePreferredPlacement(final List<String>
allNodes,
+ ResourceControllerDataProvider clusterData) {
Map<Replica, Node> preferredMapping;
preferredMapping = new HashMap<Replica, Node>();
int partitionId = 0;
- int numReplicas = countStateReplicas();
- int count = countStateReplicas();
+ // Count the total number of replicas that should be assigned assuming all
nodes are up
+ int numReplicas = calculateExpectedReplicaCount(clusterData);
for (String partition : _partitions) {
- for (int replicaId = 0; replicaId < count; replicaId++) {
+ for (int replicaId = 0; replicaId < numReplicas; replicaId++) {
Replica replica = new Replica(partition, replicaId);
String nodeName =
_placementScheme.getLocation(partitionId, replicaId,
_partitions.size(), numReplicas,
@@ -598,10 +614,11 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
}
/**
- * Counts the total number of replicas given a state-count mapping
+ * Calculates the total number of replicas based on the state-count mapping
+ * which only includes the states of live instances.
* @return
*/
- private int countStateReplicas() {
+ private int calculateStatesReplicaCount() {
int total = 0;
for (Integer count : _states.values()) {
total += count;
@@ -609,6 +626,28 @@ public class AutoRebalanceStrategy implements
RebalanceStrategy<ResourceControll
return total;
}
+ /**
+ * Calculates the expected total number of replicas assuming full cluster
availability.
+ * @param clusterData the cache that stores all cluster data
+ * @return The total number of replicas that should be assigned
+ */
+ private int calculateExpectedReplicaCount(ResourceControllerDataProvider
clusterData) {
+ IdealState currentIdealState = clusterData.getIdealState(_resourceName);
+ // Recompute the total number of replicas because for resources with
ANY_LIVEINSTANCE,
+ // the replica count should match the total number of instances in the
cluster.
+ // The _states map cannot be used for this calculation, as it only
accounts for live instances.
+ int totalReplicaCount =
currentIdealState.getReplicaCount(_nodeMap.keySet().size());
+ StateModelDefinition stateModelDef =
+ clusterData.getStateModelDef(currentIdealState.getStateModelDefRef());
+ LinkedHashMap<String, Integer> stateToCountMap =
+ stateModelDef.getStateCountMap(_nodeMap.keySet().size(),
totalReplicaCount);
+ int total = 0;
+ for (Integer count : stateToCountMap.values()) {
+ total += count;
+ }
+ return total;
+ }
+
/**
* Compute a map of replica ids to state names
* @return Map: replica id -> state name
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
b/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
index b6c02a818..47ddea079 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
@@ -118,7 +118,6 @@ public class AutoRebalanceStrategy {
// compute the preferred mapping if all nodes were up
_preferredAssignment = computePreferredPlacement(allNodes);
- // logger.info("preferred mapping:"+ preferredAssignment);
// from current mapping derive the ones in preferred location
// this will update the nodes with their current fill status
_existingPreferredAssignment =
computeExistingPreferredPlacement(currentMapping);
diff --git a/helix-core/src/test/java/org/apache/helix/TestHelper.java
b/helix-core/src/test/java/org/apache/helix/TestHelper.java
index 9dbba3476..206617861 100644
--- a/helix-core/src/test/java/org/apache/helix/TestHelper.java
+++ b/helix-core/src/test/java/org/apache/helix/TestHelper.java
@@ -40,6 +40,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.helix.PropertyKey.Builder;
+import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import
org.apache.helix.controller.rebalancer.constraint.MonitoredAbnormalResolver;
import
org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
import org.apache.helix.integration.manager.ZkTestManager;
@@ -48,8 +50,10 @@ import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.CurrentState;
import org.apache.helix.model.ExternalView;
+import org.apache.helix.model.IdealState;
import org.apache.helix.model.IdealState.RebalanceMode;
import org.apache.helix.model.Message;
import org.apache.helix.model.Message.MessageType;
@@ -73,6 +77,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
public class TestHelper {
private static final Logger LOG = LoggerFactory.getLogger(TestHelper.class);
@@ -864,4 +871,26 @@ public class TestHelper {
}
System.out.println("}");
}
+
+ public static ResourceControllerDataProvider buildMockDataCache(String
resourceName,
+ String numOfReplicas, String stateModelDef, StateModelDefinition
stateModel,
+ Set<String> disabledInstances) {
+ ClusterConfig config = new ClusterConfig("cluster");
+ config.setRebalanceDelayTime(0);
+ IdealState idealState = new IdealState(resourceName);
+ idealState.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO);
+ idealState.setReplicas(numOfReplicas);
+ idealState.setStateModelDefRef(stateModelDef);
+ idealState.setRebalanceStrategy(
+ "org.apache.helix.controller.rebalancer.strategy." +
"AutoRebalanceStrategy");
+ ResourceControllerDataProvider dataCache =
mock(ResourceControllerDataProvider.class);
+ when(dataCache.getStateModelDef(stateModelDef)).thenReturn(stateModel);
+ when(dataCache.getIdealState(resourceName)).thenReturn(idealState);
+ when(dataCache.getDisabledInstances()).thenReturn(disabledInstances);
+ when(dataCache.getClusterConfig()).thenReturn(config);
+ when(dataCache.getAbnormalStateResolver(any()))
+ .thenReturn(MonitoredAbnormalResolver.DUMMY_STATE_RESOLVER);
+ when(dataCache.getDisabledInstancesForPartition(any(),
any())).thenReturn(disabledInstances);
+ return dataCache;
+ }
}
diff --git
a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategy.java
b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategy.java
index 905c5552b..55fe690c4 100644
---
a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategy.java
+++
b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategy.java
@@ -19,6 +19,7 @@ package org.apache.helix.controller.rebalancer;
* under the License.
*/
+import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -41,6 +42,7 @@ import com.google.common.collect.Sets;
import org.apache.helix.HelixDefinedState;
import org.apache.helix.MockAccessor;
import org.apache.helix.PropertyKey.Builder;
+import org.apache.helix.TestHelper;
import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
import
org.apache.helix.controller.rebalancer.constraint.MonitoredAbnormalResolver;
import org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy;
@@ -48,8 +50,14 @@ import
org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
import org.apache.helix.controller.stages.CurrentStateOutput;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.IdealState;
+import org.apache.helix.model.LeaderStandbySMD;
import org.apache.helix.model.LiveInstance;
+import org.apache.helix.model.MasterSlaveSMD;
+import org.apache.helix.model.OnlineOfflineSMD;
import org.apache.helix.model.Partition;
+import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
+import org.apache.helix.model.ResourceConfig;
import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.tools.StateModelConfigGenerator;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
@@ -58,8 +66,12 @@ import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
public class TestAutoRebalanceStrategy {
private static Logger logger =
LoggerFactory.getLogger(TestAutoRebalanceStrategy.class);
+ private static final String DEFAULT_STATE_MODEL = "OnlineOffline";
/**
* Sanity test for a basic Master-Slave model
@@ -117,11 +129,12 @@ public class TestAutoRebalanceStrategy {
for (int i = 0; i < Math.min(stateNames.length, stateCounts.length); i++) {
states.put(stateNames[i], stateCounts[i]);
}
+ int replicaCount = states.values().stream().mapToInt(i -> i).sum();
StateModelDefinition stateModelDef = getIncompleteStateModelDef(name,
stateNames[0], states);
new AutoRebalanceTester(partitions, states, liveNodes, currentMapping,
allNodes, maxPerNode,
- stateModelDef).runRepeatedly(numIterations);
+ replicaCount + "", stateModelDef).runRepeatedly(numIterations);
}
/**
@@ -160,12 +173,14 @@ public class TestAutoRebalanceStrategy {
private Map<String, Map<String, String>> _currentMapping;
private List<String> _allNodes;
private int _maxPerNode;
+ private String _numOfReplica;
private StateModelDefinition _stateModelDef;
private Random _random;
public AutoRebalanceTester(List<String> partitions, LinkedHashMap<String,
Integer> states,
List<String> liveNodes, Map<String, Map<String, String>>
currentMapping,
- List<String> allNodes, int maxPerNode, StateModelDefinition
stateModelDef) {
+ List<String> allNodes, int maxPerNode, String numOfReplica,
+ StateModelDefinition stateModelDef) {
_partitions = partitions;
_states = states;
_liveNodes = liveNodes;
@@ -183,6 +198,7 @@ public class TestAutoRebalanceStrategy {
}
}
_maxPerNode = maxPerNode;
+ _numOfReplica = numOfReplica;
_stateModelDef = stateModelDef;
_random = new Random();
}
@@ -194,16 +210,19 @@ public class TestAutoRebalanceStrategy {
*/
public void runRepeatedly(int numIterations) {
logger.info("~~~~ Initial State ~~~~~");
+ ResourceControllerDataProvider dataProvider =
+ TestHelper.buildMockDataCache(RESOURCE_NAME, _numOfReplica,
"MasterSlave", _stateModelDef,
+ Collections.emptySet());
RebalanceStrategy strategy =
new AutoRebalanceStrategy(RESOURCE_NAME, _partitions, _states,
_maxPerNode);
ZNRecord initialResult =
- strategy.computePartitionAssignment(_allNodes, _liveNodes,
_currentMapping, null);
+ strategy.computePartitionAssignment(_allNodes, _liveNodes,
_currentMapping, dataProvider);
_currentMapping = getMapping(initialResult.getListFields());
logger.info(_currentMapping.toString());
getRunResult(_currentMapping, initialResult.getListFields());
for (int i = 0; i < numIterations; i++) {
logger.info("~~~~ Iteration " + i + " ~~~~~");
- ZNRecord znRecord = runOnceRandomly();
+ ZNRecord znRecord = runOnceRandomly(dataProvider);
if (znRecord != null) {
final Map<String, List<String>> listResult =
znRecord.getListFields();
final Map<String, Map<String, String>> mapResult =
getMapping(listResult);
@@ -387,6 +406,13 @@ public class TestAutoRebalanceStrategy {
Set<String> nodeSet = new HashSet<String>(partitionEntry.getValue());
int numUniques = nodeSet.size();
int total = partitionEntry.getValue().size();
+ int expectedPreferenceListSize =
_numOfReplica.equals("ANY_LIVEINSTANCE") ? _allNodes.size()
+ : Integer.parseInt(_numOfReplica);
+ if (nodeSet.size() != expectedPreferenceListSize) {
+ logger.error("ERROR: Partition " + partitionEntry.getKey() + "
expect " + expectedPreferenceListSize
+ + " of replicas, but the preference list has " +
listFields.size() + " nodes!");
+ return false;
+ }
if (numUniques < total) {
logger.error("ERROR: Partition " + partitionEntry.getKey() + " is
assigned to " + total
+ " nodes, but only " + numUniques + " are unique!");
@@ -480,15 +506,15 @@ public class TestAutoRebalanceStrategy {
* Randomly choose between killing, adding, or resurrecting a single node
* @return (Partition -> (Node -> State)) ZNRecord
*/
- public ZNRecord runOnceRandomly() {
+ public ZNRecord runOnceRandomly(ResourceControllerDataProvider
dataProvider) {
double choose = _random.nextDouble();
ZNRecord result = null;
if (choose < P_KILL) {
- result = removeSingleNode(null);
+ result = removeSingleNode(null, dataProvider);
} else if (choose < P_KILL + P_ADD) {
- result = addSingleNode(null);
+ result = addSingleNode(null, dataProvider);
} else if (choose < P_KILL + P_ADD + P_RESURRECT) {
- result = resurrectSingleNode(null);
+ result = resurrectSingleNode(null, dataProvider);
}
return result;
}
@@ -499,7 +525,7 @@ public class TestAutoRebalanceStrategy {
* Optional String to add
* @return ZNRecord result returned by the rebalancer
*/
- public ZNRecord addSingleNode(String node) {
+ public ZNRecord addSingleNode(String node, ResourceControllerDataProvider
dataProvider) {
logger.info("=================== add node =================");
if (_nonLiveSet.size() == 0) {
logger.warn("Cannot add node because there are no nodes left to add.");
@@ -516,7 +542,7 @@ public class TestAutoRebalanceStrategy {
_nonLiveSet.remove(node);
return new AutoRebalanceStrategy(RESOURCE_NAME, _partitions, _states,
_maxPerNode).
- computePartitionAssignment(_allNodes, _liveNodes, _currentMapping,
null);
+ computePartitionAssignment(_allNodes, _liveNodes, _currentMapping,
dataProvider);
}
/**
@@ -525,7 +551,7 @@ public class TestAutoRebalanceStrategy {
* Optional String to remove
* @return ZNRecord result returned by the rebalancer
*/
- public ZNRecord removeSingleNode(String node) {
+ public ZNRecord removeSingleNode(String node,
ResourceControllerDataProvider dataProvider) {
logger.info("=================== remove node =================");
if (_liveSet.size() == 0) {
logger.warn("Cannot remove node because there are no nodes left to
remove.");
@@ -550,7 +576,7 @@ public class TestAutoRebalanceStrategy {
}
return new AutoRebalanceStrategy(RESOURCE_NAME, _partitions, _states,
_maxPerNode)
- .computePartitionAssignment(_allNodes, _liveNodes, _currentMapping,
null);
+ .computePartitionAssignment(_allNodes, _liveNodes, _currentMapping,
dataProvider);
}
/**
@@ -559,7 +585,7 @@ public class TestAutoRebalanceStrategy {
* Optional String to resurrect
* @return ZNRecord result returned by the rebalancer
*/
- public ZNRecord resurrectSingleNode(String node) {
+ public ZNRecord resurrectSingleNode(String node,
ResourceControllerDataProvider dataProvider) {
logger.info("=================== resurrect node =================");
if (_removedSet.size() == 0) {
logger.warn("Cannot remove node because there are no nodes left to
resurrect.");
@@ -576,7 +602,7 @@ public class TestAutoRebalanceStrategy {
_liveSet.add(node);
return new AutoRebalanceStrategy(RESOURCE_NAME, _partitions, _states,
_maxPerNode)
- .computePartitionAssignment(_allNodes, _liveNodes, _currentMapping,
null);
+ .computePartitionAssignment(_allNodes, _liveNodes, _currentMapping,
dataProvider);
}
private <T> T getRandomSetElement(Set<T> source) {
@@ -618,18 +644,23 @@ public class TestAutoRebalanceStrategy {
currentMapping.put(partition, new HashMap<String, String>());
}
+ ResourceControllerDataProvider dataCache =
+ TestHelper.buildMockDataCache(RESOURCE_NAME, REPLICA_COUNT + "",
"MasterSlave",
+ MasterSlaveSMD.build(), Collections.emptySet());
+
// make sure that when the first node joins, a single replica is assigned
fairly
List<String> partitions = ImmutableList.copyOf(PARTITIONS);
LinkedHashMap<String, Integer> stateCount =
STATE_MODEL.getStateCountMap(liveNodes.size(), REPLICA_COUNT);
ZNRecord znRecord =
new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
Map<String, List<String>> preferenceLists = znRecord.getListFields();
for (String partition : currentMapping.keySet()) {
// make sure these are all MASTER
List<String> preferenceList = preferenceLists.get(partition);
Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ // Since there is only 1 node in the cluster, it constructs the whole
preference list
Assert.assertEquals(preferenceList.size(), 1, "invalid preference list
for " + partition);
}
@@ -642,12 +673,15 @@ public class TestAutoRebalanceStrategy {
}
znRecord =
new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
preferenceLists = znRecord.getListFields();
for (String partition : currentMapping.keySet()) {
List<String> preferenceList = preferenceLists.get(partition);
Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
- Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ // Since we have enough nodes to achieve the replica count, the
preference list size should be
+ // equal to the replica count
+ Assert.assertEquals(preferenceList.size(), REPLICA_COUNT,
+ "invalid preference list for " + partition);
Assert.assertEquals(preferenceList.get(0), NODES[0], "invalid preference
list for "
+ partition);
Assert.assertEquals(preferenceList.get(1), NODES[1], "invalid preference
list for "
@@ -660,7 +694,7 @@ public class TestAutoRebalanceStrategy {
}
znRecord =
new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
preferenceLists = znRecord.getListFields();
Set<String> firstNodes = Sets.newHashSet();
for (String partition : currentMapping.keySet()) {
@@ -682,13 +716,14 @@ public class TestAutoRebalanceStrategy {
currentMapping.get(PARTITIONS[1]).put(NODES[1], "MASTER");
znRecord =
new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
preferenceLists = znRecord.getListFields();
boolean newNodeUsed = false;
for (String partition : currentMapping.keySet()) {
List<String> preferenceList = preferenceLists.get(partition);
Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
- Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ Assert.assertEquals(preferenceList.size(), REPLICA_COUNT,
+ "invalid preference list for " + partition);
if (preferenceList.contains(NODES[2])) {
newNodeUsed = true;
Assert.assertEquals(preferenceList.get(1), NODES[2],
@@ -710,14 +745,15 @@ public class TestAutoRebalanceStrategy {
currentMapping.get(PARTITIONS[2]).put(NODES[2], "SLAVE");
znRecord =
new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
preferenceLists = znRecord.getListFields();
firstNodes.clear();
Set<String> secondNodes = Sets.newHashSet();
for (String partition : currentMapping.keySet()) {
List<String> preferenceList = preferenceLists.get(partition);
Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
- Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ Assert.assertEquals(preferenceList.size(), REPLICA_COUNT,
+ "invalid preference list for " + partition);
firstNodes.add(preferenceList.get(0));
secondNodes.add(preferenceList.get(1));
}
@@ -738,12 +774,13 @@ public class TestAutoRebalanceStrategy {
currentMapping.get(PARTITIONS[2]).put(NODES[2], "MASTER");
znRecord =
new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
preferenceLists = znRecord.getListFields();
for (String partition : currentMapping.keySet()) {
List<String> preferenceList = preferenceLists.get(partition);
Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
- Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ Assert.assertEquals(preferenceList.size(), REPLICA_COUNT,
+ "invalid preference list for " + partition);
Map<String, String> stateMap = currentMapping.get(partition);
for (String participant : stateMap.keySet()) {
Assert.assertTrue(preferenceList.contains(participant), "minimal
movement violated for "
@@ -769,13 +806,14 @@ public class TestAutoRebalanceStrategy {
currentMapping.get(PARTITIONS[2]).put(NODES[2], "MASTER");
znRecord =
new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
preferenceLists = znRecord.getListFields();
firstNodes.clear();
for (String partition : currentMapping.keySet()) {
List<String> preferenceList = preferenceLists.get(partition);
Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
- Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ Assert.assertEquals(preferenceList.size(), REPLICA_COUNT,
+ "invalid preference list for " + partition);
firstNodes.add(preferenceList.get(0));
}
Assert.assertEquals(firstNodes.size(), 2, "masters not evenly
distributed");
@@ -794,6 +832,9 @@ public class TestAutoRebalanceStrategy {
for (int i = 0; i < nPartitions; i++) {
partitions.add(Integer.toString(i));
}
+ ResourceControllerDataProvider dataCache =
+ TestHelper.buildMockDataCache(resourceName, nReplicas + "",
DEFAULT_STATE_MODEL,
+ OnlineOfflineSMD.build(), Collections.emptySet());
LinkedHashMap<String, Integer> states = new LinkedHashMap<String,
Integer>(2);
states.put("OFFLINE", 0);
@@ -801,7 +842,7 @@ public class TestAutoRebalanceStrategy {
AutoRebalanceStrategy strategy = new AutoRebalanceStrategy(resourceName,
partitions, states);
ZNRecord znRecord = strategy.computePartitionAssignment(instanceNames,
instanceNames,
- new HashMap<String, Map<String, String>>(0), null);
+ new HashMap<String, Map<String, String>>(0), dataCache);
for (List p : znRecord.getListFields().values()) {
Assert.assertEquals(p.size(), nReplicas);
@@ -837,9 +878,13 @@ public class TestAutoRebalanceStrategy {
upperBounds.put(state, STATE_MODEL.getNumInstancesPerState(state));
}
+ ResourceControllerDataProvider dataCache =
+ TestHelper.buildMockDataCache(RESOURCE, 1 + "", DEFAULT_STATE_MODEL,
OnlineOfflineSMD.build(),
+ Collections.emptySet());
+
ZNRecord znRecord =
new AutoRebalanceStrategy(RESOURCE, partitions, stateCount,
Integer.MAX_VALUE)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
Map<String, List<String>> preferenceLists = znRecord.getListFields();
List<String> preferenceList = preferenceLists.get(partition.toString());
Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
@@ -854,7 +899,7 @@ public class TestAutoRebalanceStrategy {
znRecord =
new AutoRebalanceStrategy(RESOURCE, partitions, stateCount,
Integer.MAX_VALUE)
- .computePartitionAssignment(allNodes, liveNodes, currentMapping,
null);
+ .computePartitionAssignment(allNodes, liveNodes, currentMapping,
dataCache);
preferenceLists = znRecord.getListFields();
preferenceList = preferenceLists.get(partition.toString());
@@ -866,4 +911,317 @@ public class TestAutoRebalanceStrategy {
// finally, make sure we haven't moved it.
Assert.assertEquals(finalPreferredNode, otherNode);
}
+
+ @Test
+ public void testAutoRebalanceStrategyWorkWithDisabledInstances() {
+ final String RESOURCE_NAME = "resource";
+ final String[] PARTITIONS = {"resource_0", "resource_1", "resource_2"};
+ final StateModelDefinition STATE_MODEL = LeaderStandbySMD.build();
+ final int REPLICA_COUNT = 2;
+ final String[] NODES = {"n0", "n1"};
+
+ ResourceControllerDataProvider dataCache =
TestHelper.buildMockDataCache(RESOURCE_NAME,
+ ResourceConfig.ResourceConfigConstants.ANY_LIVEINSTANCE.toString(),
"LeaderStandby",
+ STATE_MODEL, Collections.emptySet());
+
+ // initial state, 2 nodes, no mapping
+ List<String> allNodes = Lists.newArrayList(NODES[0], NODES[1]);
+ List<String> liveNodes = Lists.newArrayList(NODES[0], NODES[1]);
+ Map<String, Map<String, String>> currentMapping = Maps.newHashMap();
+ for (String partition : PARTITIONS) {
+ currentMapping.put(partition, new HashMap<String, String>());
+ }
+
+ // make sure that when the first node joins, a single replica is assigned
fairly
+ List<String> partitions = ImmutableList.copyOf(PARTITIONS);
+ LinkedHashMap<String, Integer> stateCount =
+ STATE_MODEL.getStateCountMap(liveNodes.size(), REPLICA_COUNT);
+ ZNRecord znRecord =
+ new AutoRebalanceStrategy(RESOURCE_NAME, partitions,
stateCount).computePartitionAssignment(
+ allNodes, liveNodes, currentMapping, dataCache);
+ Map<String, List<String>> preferenceLists = znRecord.getListFields();
+ for (String partition : currentMapping.keySet()) {
+ List<String> preferenceList = preferenceLists.get(partition);
+ Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ }
+
+ // now disable node 1, and make sure that it is not in the preference list
+ allNodes = new ArrayList<>(allNodes);
+ liveNodes = new ArrayList<>(liveNodes);
+ liveNodes.remove(NODES[0]);
+ for (String partition : PARTITIONS) {
+ Map<String, String> idealStateMap = znRecord.getMapField(partition);
+ currentMapping.put(partition, idealStateMap);
+ }
+
+ stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(), 1);
+ znRecord =
+ new AutoRebalanceStrategy(RESOURCE_NAME, partitions,
stateCount).computePartitionAssignment(
+ allNodes, liveNodes, currentMapping, dataCache);
+ preferenceLists = znRecord.getListFields();
+ for (String partition : currentMapping.keySet()) {
+ // make sure the master is transferred to the other node
+ List<String> preferenceList = preferenceLists.get(partition);
+ Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ Assert.assertEquals(preferenceList.size(), 1, "invalid preference list
for " + partition);
+ // Since node 0 is disabled, node 1 should be the only node in the
preference list and it
+ // should be in the top state for every partition
+ Assert.assertTrue(znRecord.getListField(partition).contains(NODES[1]),
+ "invalid preference list for " + partition);
+ Assert.assertEquals(znRecord.getMapField(partition).get(NODES[1]),
STATE_MODEL.getTopState());
+ }
+ }
+
+ @Test
+ public void testRebalanceWithErrorPartition() {
+ final String RESOURCE_NAME = "resource";
+ final String[] PARTITIONS = {"resource_0", "resource_1", "resource_2"};
+ final StateModelDefinition STATE_MODEL = LeaderStandbySMD.build();
+ final String[] NODES = {"n0", "n1", "n2", "n3", "n4", "n5", "n6", "n7",
"n8", "n9"};
+
+ ResourceControllerDataProvider dataCache =
TestHelper.buildMockDataCache(RESOURCE_NAME,
+ ResourceConfig.ResourceConfigConstants.ANY_LIVEINSTANCE.toString(),
"LeaderStandby",
+ STATE_MODEL, Collections.emptySet());
+ // initial state, 10 node, no mapping
+ List<String> allNodes = Lists.newArrayList(NODES);
+ List<String> liveNodes = Lists.newArrayList(NODES);
+ Map<String, Map<String, String>> currentMapping = Maps.newHashMap();
+ for (String partition : PARTITIONS) {
+ currentMapping.put(partition, new HashMap<String, String>());
+ }
+
+ // make sure that when nodes join, all partitions is assigned fairly
+ List<String> partitions = ImmutableList.copyOf(PARTITIONS);
+ LinkedHashMap<String, Integer> stateCount =
+ STATE_MODEL.getStateCountMap(liveNodes.size(), allNodes.size());
+ ZNRecord znRecord =
+ new AutoRebalanceStrategy(RESOURCE_NAME, partitions,
stateCount).computePartitionAssignment(
+ allNodes, liveNodes, currentMapping, dataCache);
+ Map<String, List<String>> preferenceLists = znRecord.getListFields();
+ for (String partition : currentMapping.keySet()) {
+ List<String> preferenceList = preferenceLists.get(partition);
+ Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ Assert.assertEquals(preferenceList.size(), allNodes.size(),
+ "invalid preference list for " + partition);
+ }
+
+ // Suppose that one replica of partition 0 is in n0, and it has been in
the ERROR state.
+ for (String partition : PARTITIONS) {
+ Map<String, String> idealStateMap = znRecord.getMapField(partition);
+ currentMapping.put(partition, idealStateMap);
+ }
+ currentMapping.get(PARTITIONS[0]).put(NODES[0], "ERROR");
+
+ // Recalculate the ideal state, n0 shouldn't be dropped from the
preference list.
+ stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(),
allNodes.size());
+ znRecord =
+ new AutoRebalanceStrategy(RESOURCE_NAME, partitions,
stateCount).computePartitionAssignment(
+ allNodes, liveNodes, currentMapping, dataCache);
+ preferenceLists = znRecord.getListFields();
+ for (String partition : currentMapping.keySet()) {
+ // make sure the size is equal to the number of all nodes
+ List<String> preferenceList = preferenceLists.get(partition);
+ Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ Assert.assertEquals(preferenceList.size(), allNodes.size(),
+ "invalid preference list for " + partition);
+ // Even if n0 is in ERROR state, it should appear in the IDEAL state
+ Assert.assertTrue(znRecord.getListField(partition).contains(NODES[0]),
+ "invalid preference list for " + partition);
+ Assert.assertTrue(znRecord.getMapField(partition).containsKey(NODES[0]),
+ "invalid ideal state mapping for " + partition);
+ }
+
+ // now disable node 0, and make sure the dataCache provides it. And add
another node n10 to the
+ // cluster. We want to make sure the n10 can pick up another replica of
partition 0,1,2.
+ allNodes = new ArrayList<>(allNodes);
+ liveNodes = new ArrayList<>(liveNodes);
+ liveNodes.remove(NODES[0]);
+ allNodes.add("n10");
+ liveNodes.add("n10");
+
+ dataCache = TestHelper.buildMockDataCache(RESOURCE_NAME,
+ ResourceConfig.ResourceConfigConstants.ANY_LIVEINSTANCE.toString(),
"LeaderStandby",
+ STATE_MODEL, Collections.emptySet());
+
+ // Even though we had 11 nodes, we only have 10 nodes in the liveNodes
list. So the state
+ // count map should have 10 entries instead of 11 when using
ANY_LIVEINSTANCE .
+ stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(), 10);
+ znRecord =
+ new AutoRebalanceStrategy(RESOURCE_NAME, partitions,
stateCount).computePartitionAssignment(
+ allNodes, liveNodes, currentMapping, dataCache);
+ preferenceLists = znRecord.getListFields();
+ for (String partition : currentMapping.keySet()) {
+ // make sure the size is equal to the number of live nodes
+ List<String> preferenceList = preferenceLists.get(partition);
+ Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ Assert.assertEquals(preferenceList.size(), liveNodes.size(),
+ "invalid preference list for " + partition);
+ // Since node 0 is disabled with ERROR state, it shouldn't appear in the
IDEAL state
+ Assert.assertFalse(znRecord.getListField(partition).contains(NODES[0]),
+ "invalid preference list for " + partition);
+ Assert.assertFalse(znRecord.getMapField(partition).containsKey(NODES[0]),
+ "invalid ideal state mapping for " + partition);
+ }
+ }
+
+ @Test
+ public void testAutoRebalanceStrategyWorkWithDisabledButActiveInstances() {
+ final String RESOURCE_NAME = "resource";
+ final String[] PARTITIONS = {"resource_0", "resource_1", "resource_2"};
+ final StateModelDefinition STATE_MODEL = LeaderStandbySMD.build();
+ final int REPLICA_COUNT = 2;
+ final String[] NODES = {"n0", "n1"};
+
+ ResourceControllerDataProvider dataCache =
TestHelper.buildMockDataCache(RESOURCE_NAME,
+ ResourceConfig.ResourceConfigConstants.ANY_LIVEINSTANCE.toString(),
"LeaderStandby",
+ STATE_MODEL, Collections.emptySet());
+ Map<String, LiveInstance> liveInstances = new HashMap<>();
+ liveInstances.put(NODES[0], new LiveInstance(NODES[0]));
+ liveInstances.put(NODES[1], new LiveInstance(NODES[1]));
+ when(dataCache.getLiveInstances()).thenReturn(liveInstances);
+ // initial state, 2 node, no mapping
+ List<String> allNodes = Lists.newArrayList(NODES[0], NODES[1]);
+ List<String> liveNodes = Lists.newArrayList(NODES[0], NODES[1]);
+ Map<String, Map<String, String>> currentMapping = Maps.newHashMap();
+ for (String partition : PARTITIONS) {
+ currentMapping.put(partition, new HashMap<String, String>());
+ }
+
+ // make sure that when nodes join, all partitions is assigned fairly
+ List<String> partitions = ImmutableList.copyOf(PARTITIONS);
+ LinkedHashMap<String, Integer> stateCount =
+ STATE_MODEL.getStateCountMap(liveNodes.size(), REPLICA_COUNT);
+ ZNRecord znRecord =
+ new AutoRebalanceStrategy(RESOURCE_NAME, partitions,
stateCount).computePartitionAssignment(
+ allNodes, liveNodes, currentMapping, dataCache);
+ Map<String, List<String>> preferenceLists = znRecord.getListFields();
+ for (String partition : currentMapping.keySet()) {
+ List<String> preferenceList = preferenceLists.get(partition);
+ Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ }
+
+ // now disable node 0, and make sure the dataCache provides it
+ for (String partition : PARTITIONS) {
+ Map<String, String> idealStateMap = znRecord.getMapField(partition);
+ currentMapping.put(partition, idealStateMap);
+ }
+ dataCache = TestHelper.buildMockDataCache(RESOURCE_NAME,
+ ResourceConfig.ResourceConfigConstants.ANY_LIVEINSTANCE.toString(),
"LeaderStandby",
+ STATE_MODEL, Sets.newHashSet(NODES[0]));
+ liveInstances.put(NODES[0], new LiveInstance(NODES[0]));
+ liveInstances.put(NODES[1], new LiveInstance(NODES[1]));
+ when(dataCache.getLiveInstances()).thenReturn(liveInstances);
+
+ stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(), 2);
+ znRecord =
+ new AutoRebalanceStrategy(RESOURCE_NAME, partitions,
stateCount).computePartitionAssignment(
+ allNodes, liveNodes, currentMapping, dataCache);
+ preferenceLists = znRecord.getListFields();
+
+ for (String partition : currentMapping.keySet()) {
+ // make sure the size is equal to the number of active nodes
+ List<String> preferenceList = preferenceLists.get(partition);
+ Assert.assertNotNull(preferenceList, "invalid preference list for " +
partition);
+ Assert.assertEquals(preferenceList.size(), 2, "invalid preference list
for " + partition);
+ Assert.assertTrue(znRecord.getListField(partition).contains(NODES[1]),
+ "invalid preference list for " + partition);
+ Assert.assertTrue(znRecord.getListField(partition).contains(NODES[0]),
+ "invalid preference list for " + partition);
+ }
+
+ // Genera the new ideal state
+ IdealState currentIdealState = dataCache.getIdealState(RESOURCE_NAME);
+ IdealState newIdealState = new IdealState(RESOURCE_NAME);
+
newIdealState.getRecord().setSimpleFields(currentIdealState.getRecord().getSimpleFields());
+ newIdealState.setRebalanceMode(currentIdealState.getRebalanceMode());
+ newIdealState.getRecord().setListFields(znRecord.getListFields());
+
+ // Mimic how the Rebalancer would react to the new ideal state and update
the current mapping
+ Resource resource = new Resource(RESOURCE_NAME);
+ for (String partition : PARTITIONS) {
+ resource.addPartition(partition);
+ }
+ CurrentStateOutput currentStateOutput = new CurrentStateOutput();
+ currentStateOutput.setCurrentState(RESOURCE_NAME,
resource.getPartition(PARTITIONS[0]), NODES[0], "LEADER");
+ currentStateOutput.setCurrentState(RESOURCE_NAME,
resource.getPartition(PARTITIONS[0]), NODES[1], "STANDBY");
+ currentStateOutput.setCurrentState(RESOURCE_NAME,
resource.getPartition(PARTITIONS[1]), NODES[0], "STANDBY");
+ currentStateOutput.setCurrentState(RESOURCE_NAME,
resource.getPartition(PARTITIONS[1]), NODES[1], "LEADER");
+ currentStateOutput.setCurrentState(RESOURCE_NAME,
resource.getPartition(PARTITIONS[2]), NODES[0], "LEADER");
+ currentStateOutput.setCurrentState(RESOURCE_NAME,
resource.getPartition(PARTITIONS[2]), NODES[1], "STANDBY");
+
+ DelayedAutoRebalancer autoRebalancer = new DelayedAutoRebalancer();
+ ResourceAssignment assignment =
autoRebalancer.computeBestPossiblePartitionState(dataCache, newIdealState,
resource,
+ currentStateOutput);
+
+ // Assert that the new assignment will move the node 0 as the OFFLINE
state. And the node 1 as
+ // the top state LEADER.
+ for (String partition : PARTITIONS) {
+
Assert.assertEquals(assignment.getReplicaMap(resource.getPartition(partition)).get(NODES[0]),
"OFFLINE");
+
Assert.assertEquals(assignment.getReplicaMap(resource.getPartition(partition)).get(NODES[1]),
"LEADER");
+ }
+ }
+
+ @Test
+ public void testSlowlyBootstrapping() {
+ // Resource setup
+ final String RESOURCE_NAME = "resource";
+ final int PARTITIONS = 100;
+ final int NUM_NODES = 5;
+ final StateModelDefinition STATE_MODEL = LeaderStandbySMD.build();
+ ArrayList<String> partitions = new ArrayList<String>();
+ for (int i = 0; i < PARTITIONS; i++) {
+ partitions.add("resource_" + i);
+ }
+ ArrayList<String> allNodes = new ArrayList<String>();
+ ArrayList<String> liveNodes = new ArrayList<String>();
+ for (int i = 0; i < NUM_NODES; i++) {
+ allNodes.add("node-" + i);
+ }
+
+ ResourceControllerDataProvider dataCache =
TestHelper.buildMockDataCache(RESOURCE_NAME,
+ "1", "LeaderStandby",
+ STATE_MODEL, Collections.emptySet());
+ // initial state, 10 node, no mapping
+ Map<String, Map<String, String>> currentMapping = Maps.newHashMap();
+ for (String partition : partitions) {
+ currentMapping.put(partition, new HashMap<String, String>());
+ }
+
+ // Run rebalance with 5 nodes, 1 live instances
+ liveNodes.add(allNodes.get(0));
+ LinkedHashMap<String, Integer> stateCount =
+ STATE_MODEL.getStateCountMap(liveNodes.size(), 1);
+ RebalanceStrategy strategy = new AutoRebalanceStrategy(RESOURCE_NAME,
partitions, stateCount, 25);
+ ZNRecord znRecord = strategy.computePartitionAssignment(allNodes,
liveNodes, currentMapping,
+ dataCache);
+
+ // Suppose that we could only bootstrap a portion of the ideal state
replicas and update the
+ // current state mapping
+ int i = 0;
+ for (String partition : partitions) {
+ List<String> preferenceList = znRecord.getListField(partition);
+ if (!preferenceList.isEmpty()) {
+ if (i % 2 == 0) {
+ currentMapping.get(partition).put(preferenceList.get(0), "LEADER");
+ }
+ }
+ i++;
+ }
+
+ // The result of the assignment should be the same as the previous
assignment
+ int countOfNonEmptyPreferenceList = 0;
+ ZNRecord newRecord = strategy.computePartitionAssignment(allNodes,
liveNodes, currentMapping, dataCache);
+ for (String partition : partitions) {
+ List<String> preferenceList = newRecord.getListField(partition);
+ Assert.assertEquals(newRecord.getMapField(partition),
znRecord.getMapField(partition),
+ "The partition " + partition + " should have the same ideal state
mapping");
+ if (!preferenceList.isEmpty()) {
+ countOfNonEmptyPreferenceList++;
+ }
+ }
+ // The number of non-empty preference list should be 25 because we set the
MAX_PARTITION_PER_NODE = 25
+ Assert.assertEquals(countOfNonEmptyPreferenceList, 25);
+ }
}
diff --git
a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategyImbalanceAssignment.java
b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategyImbalanceAssignment.java
index f8f0b9060..cd34d91f7 100644
---
a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategyImbalanceAssignment.java
+++
b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestAutoRebalanceStrategyImbalanceAssignment.java
@@ -20,11 +20,14 @@ package org.apache.helix.controller.rebalancer;
*/
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.helix.TestHelper;
+import org.apache.helix.model.OnlineOfflineSMD;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
import org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy;
@@ -68,7 +71,9 @@ public class TestAutoRebalanceStrategyImbalanceAssignment {
AutoRebalanceStrategy strategy = new AutoRebalanceStrategy(resourceName,
partitions, states);
ZNRecord record = strategy.computePartitionAssignment(instanceNames,
instanceNames,
- new HashMap<String, Map<String, String>>(0), new
ResourceControllerDataProvider());
+ new HashMap<String, Map<String, String>>(0),
+ TestHelper.buildMockDataCache(resourceName, nReplicas + "",
"OnlineOffline",
+ OnlineOfflineSMD.build(), Collections.emptySet()));
for (Map<String, String> stateMapping : record.getMapFields().values()) {
Assert.assertEquals(stateMapping.size(), nReplicas);
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestAutoRebalance.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestAutoRebalance.java
index 3c389ac3a..091da7fac 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestAutoRebalance.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestAutoRebalance.java
@@ -164,7 +164,7 @@ public class TestAutoRebalance extends
ZkStandAloneCMTestBase {
ZK_ADDR);
}
- @Test()
+ @Test(dependsOnMethods = "testDropResourceAutoRebalance")
public void testAutoRebalance() throws Exception {
// kill 1 node
_participants[0].syncStop();