This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new eda45fc58 Add support for ALL_RESOURCES key to disabled partitions
(#2848)
eda45fc58 is described below
commit eda45fc58f4e6fd19a96eb6bcf33f0e7813e0a53
Author: Grant Paláu Spencer <[email protected]>
AuthorDate: Thu Jul 25 13:01:50 2024 -0700
Add support for ALL_RESOURCES key to disabled partitions (#2848)
Add support for ALL_RESOURCES key to disabled partitions
---
.../apache/helix/constants/InstanceConstants.java | 2 +
.../dataproviders/BaseControllerDataProvider.java | 5 +-
.../ResourceControllerDataProvider.java | 23 +++
.../constraints/ReplicaActivateConstraint.java | 5 +-
.../helix/monitoring/mbeans/InstanceMonitor.java | 13 ++
.../helix/integration/TestDisablePartition.java | 83 ++++++++++
.../rebalancer/TestInstanceOperation.java | 183 +++++++++++++++++++--
.../monitoring/mbeans/TestInstanceMonitor.java | 7 +-
8 files changed, 304 insertions(+), 17 deletions(-)
diff --git
a/helix-common/src/main/java/org/apache/helix/constants/InstanceConstants.java
b/helix-common/src/main/java/org/apache/helix/constants/InstanceConstants.java
index 22f6c7c76..df6c37bb6 100644
---
a/helix-common/src/main/java/org/apache/helix/constants/InstanceConstants.java
+++
b/helix-common/src/main/java/org/apache/helix/constants/InstanceConstants.java
@@ -100,4 +100,6 @@ public class InstanceConstants {
*/
UNKNOWN
}
+
+ public static final String ALL_RESOURCES_DISABLED_PARTITION_KEY =
"ALL_RESOURCES";
}
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
index ce5d3de8c..885e789f6 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/BaseControllerDataProvider.java
@@ -791,6 +791,7 @@ public class BaseControllerDataProvider implements
ControlContextProvider {
*/
public Set<String> getDisabledInstancesForPartition(String resource, String
partition) {
Set<String> disabledInstancesForPartition = new
HashSet<>(_disabledInstanceSet);
+
if (_disabledInstanceForPartitionMap.containsKey(resource)
&& _disabledInstanceForPartitionMap
.get(resource).containsKey(partition)) {
@@ -1080,8 +1081,8 @@ public class BaseControllerDataProvider implements
ControlContextProvider {
_disabledInstanceSet.clear();
for (InstanceConfig config : allInstanceConfigs) {
Map<String, List<String>> disabledPartitionMap =
config.getDisabledPartitionsMap();
- if (config.getInstanceOperation().getOperation()
- .equals(InstanceConstants.InstanceOperation.DISABLE)) {
+ // Treat instance as disabled if it has "DISABLE" operation or
"ALL_RESOURCES" in the disabled partition map
+ if
(config.getInstanceOperation().getOperation().equals(InstanceConstants.InstanceOperation.DISABLE))
{
_disabledInstanceSet.add(config.getInstanceName());
}
for (String resource : disabledPartitionMap.keySet()) {
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
index bc0cbf530..021aab6a8 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
@@ -20,6 +20,7 @@ package org.apache.helix.controller.dataproviders;
*/
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -36,6 +37,7 @@ import org.apache.helix.common.caches.AbstractDataCache;
import org.apache.helix.common.caches.CustomizedStateCache;
import org.apache.helix.common.caches.CustomizedViewCache;
import org.apache.helix.common.caches.PropertyCache;
+import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.controller.LogUtil;
import org.apache.helix.controller.common.CapacityNode;
import org.apache.helix.controller.pipeline.Pipeline;
@@ -48,6 +50,7 @@ import org.apache.helix.model.CustomizedStateConfig;
import org.apache.helix.model.CustomizedView;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
+import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.ResourceAssignment;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.slf4j.Logger;
@@ -85,6 +88,7 @@ public class ResourceControllerDataProvider extends
BaseControllerDataProvider {
private Set<HelixConstants.ChangeType> _refreshedChangeTypes;
private Set<String> _aggregationEnabledTypes = new HashSet<>();
private Set<CapacityNode> _simpleCapacitySet;
+ private final Set<String> _disabledInstancesForAllPartitionsSet = new
HashSet<>();
// CrushEd strategy needs to have a stable partition list input. So this
cached list persist the
@@ -176,6 +180,7 @@ public class ResourceControllerDataProvider extends
BaseControllerDataProvider {
// TODO: remove the workaround once we are able to apply the simple fix
without majorly
// TODO: impacting user's clusters.
refreshStablePartitionList(getIdealStates());
+ refreshDisabledInstancesForAllPartitionsSet();
if (getClusterConfig().getGlobalMaxPartitionAllowedPerInstance() != -1) {
buildSimpleCapacityMap(getClusterConfig().getGlobalMaxPartitionAllowedPerInstance());
@@ -551,4 +556,22 @@ public class ResourceControllerDataProvider extends
BaseControllerDataProvider {
public Set<CapacityNode> getSimpleCapacitySet() {
return _simpleCapacitySet;
}
+
+ private void refreshDisabledInstancesForAllPartitionsSet() {
+ _disabledInstancesForAllPartitionsSet.clear();
+ Collection<InstanceConfig> allConfigs = getInstanceConfigMap().values();
+ for (InstanceConfig config : allConfigs) {
+ Map<String, List<String>> disabledPartitionMap =
config.getDisabledPartitionsMap();
+ if
(disabledPartitionMap.containsKey(InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY))
{
+ _disabledInstancesForAllPartitionsSet.add(config.getInstanceName());
+ }
+ }
+ }
+
+ @Override
+ public Set<String> getDisabledInstancesForPartition(String resource, String
partition) {
+ Set<String> disabledInstances =
super.getDisabledInstancesForPartition(resource, partition);
+ disabledInstances.addAll(_disabledInstancesForAllPartitionsSet);
+ return disabledInstances;
+ }
}
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/constraints/ReplicaActivateConstraint.java
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/constraints/ReplicaActivateConstraint.java
index efee2a219..d4c42aeb6 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/constraints/ReplicaActivateConstraint.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/constraints/ReplicaActivateConstraint.java
@@ -21,6 +21,7 @@ package
org.apache.helix.controller.rebalancer.waged.constraints;
import java.util.List;
+import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.controller.rebalancer.waged.model.AssignableNode;
import org.apache.helix.controller.rebalancer.waged.model.AssignableReplica;
import org.apache.helix.controller.rebalancer.waged.model.ClusterContext;
@@ -36,8 +37,10 @@ class ReplicaActivateConstraint extends HardConstraint {
boolean isAssignmentValid(AssignableNode node, AssignableReplica replica,
ClusterContext clusterContext) {
List<String> disabledPartitions =
node.getDisabledPartitionsMap().get(replica.getResourceName());
+ boolean allResourcesDisabled = node.getDisabledPartitionsMap()
+ .containsKey(InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY);
- if (disabledPartitions != null &&
disabledPartitions.contains(replica.getPartitionName())) {
+ if (allResourcesDisabled || (disabledPartitions != null &&
disabledPartitions.contains(replica.getPartitionName()))) {
if (enableLogging) {
LOG.info("Cannot assign the inactive replica: {}",
replica.getPartitionName());
}
diff --git
a/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/InstanceMonitor.java
b/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/InstanceMonitor.java
index 6b7d6c087..a1d574b82 100644
---
a/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/InstanceMonitor.java
+++
b/helix-core/src/main/java/org/apache/helix/monitoring/mbeans/InstanceMonitor.java
@@ -30,6 +30,7 @@ import javax.management.ObjectName;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.monitoring.mbeans.dynamicMBeans.DynamicMBeanProvider;
import org.apache.helix.monitoring.mbeans.dynamicMBeans.DynamicMetric;
import org.apache.helix.monitoring.mbeans.dynamicMBeans.SimpleDynamicMetric;
@@ -48,6 +49,7 @@ public class InstanceMonitor extends DynamicMBeanProvider {
ENABLED_STATUS_GAUGE("Enabled"),
ONLINE_STATUS_GAUGE("Online"),
DISABLED_PARTITIONS_GAUGE("DisabledPartitions"),
+ ALL_PARTITIONS_DISABLED_GAUGE("AllPartitionsDisabled"),
MAX_CAPACITY_USAGE_GAUGE("MaxCapacityUsageGauge"),
MESSAGE_QUEUE_SIZE_GAUGE("MessageQueueSizeGauge"),
PASTDUE_MESSAGE_GAUGE("PastDueMessageGauge");
@@ -75,6 +77,7 @@ public class InstanceMonitor extends DynamicMBeanProvider {
// Gauges
private SimpleDynamicMetric<Long> _enabledStatusGauge;
private SimpleDynamicMetric<Long> _disabledPartitionsGauge;
+ private SimpleDynamicMetric<Long> _allPartitionsDisabledGauge;
private SimpleDynamicMetric<Long> _onlineStatusGauge;
private SimpleDynamicMetric<Double> _maxCapacityUsageGauge;
private SimpleDynamicMetric<Long> _messageQueueSizeGauge;
@@ -105,6 +108,8 @@ public class InstanceMonitor extends DynamicMBeanProvider {
_disabledPartitionsGauge =
new
SimpleDynamicMetric<>(InstanceMonitorMetric.DISABLED_PARTITIONS_GAUGE.metricName(),
0L);
+ _allPartitionsDisabledGauge = new SimpleDynamicMetric<>(
+ InstanceMonitorMetric.ALL_PARTITIONS_DISABLED_GAUGE.metricName(), 0L);
_enabledStatusGauge =
new
SimpleDynamicMetric<>(InstanceMonitorMetric.ENABLED_STATUS_GAUGE.metricName(),
0L);
_onlineStatusGauge =
@@ -124,6 +129,7 @@ public class InstanceMonitor extends DynamicMBeanProvider {
List<DynamicMetric<?, ?>> attributeList = Lists.newArrayList(
_totalMessagedReceivedCounter,
_disabledPartitionsGauge,
+ _allPartitionsDisabledGauge,
_enabledStatusGauge,
_onlineStatusGauge,
_maxCapacityUsageGauge,
@@ -157,6 +163,7 @@ public class InstanceMonitor extends DynamicMBeanProvider {
protected long getDisabledPartitions() {
return _disabledPartitionsGauge.getValue();
}
+ protected long getAllPartitionsDisabled() { return
_allPartitionsDisabledGauge.getValue(); }
protected long getMessageQueueSizeGauge() { return
_messageQueueSizeGauge.getValue(); }
@@ -191,10 +198,15 @@ public class InstanceMonitor extends DynamicMBeanProvider
{
Collections.sort(_tags);
}
long numDisabledPartitions = 0L;
+ boolean allPartitionsDisabled = false;
if (disabledPartitions != null) {
for (List<String> partitions : disabledPartitions.values()) {
if (partitions != null) {
numDisabledPartitions += partitions.size();
+ if
(partitions.contains(InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY)) {
+ allPartitionsDisabled = true;
+ numDisabledPartitions -= 1;
+ }
}
}
}
@@ -206,6 +218,7 @@ public class InstanceMonitor extends DynamicMBeanProvider {
_onlineStatusGauge.updateValue(isLive ? 1L : 0L);
_enabledStatusGauge.updateValue(isEnabled ? 1L : 0L);
_disabledPartitionsGauge.updateValue(numDisabledPartitions);
+ _allPartitionsDisabledGauge.updateValue(allPartitionsDisabled ? 1L : 0L);
}
/**
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/TestDisablePartition.java
b/helix-core/src/test/java/org/apache/helix/integration/TestDisablePartition.java
index 08ac69204..f9d9a6422 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/TestDisablePartition.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/TestDisablePartition.java
@@ -19,6 +19,7 @@ package org.apache.helix.integration;
* under the License.
*/
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -28,16 +29,20 @@ import java.util.Set;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
+import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.controller.rebalancer.AutoRebalancer;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
+import
org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
import org.apache.helix.integration.manager.ClusterControllerManager;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.model.BuiltInStateModelDefinitions;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.IdealState.RebalanceMode;
@@ -209,4 +214,82 @@ public class TestDisablePartition extends
ZkStandAloneCMTestBase {
deleteCluster(clusterName);
}
+
+ @Test
+ public void testDisableAllPartitions() throws Exception {
+ final int NUM_PARTITIONS = 8;
+ final int NUM_REPLICAS = 3;
+ final long DELAY_WINDOW = 200000;
+ final String CRUSHED_RESOURCE = "TEST_DB0_CRUSHED";
+ final String WAGED_RESOURCE = "TEST_DB1_WAGED";
+
+ List<String> resources = Arrays.asList(CRUSHED_RESOURCE, WAGED_RESOURCE);
+ createResourceWithDelayedRebalance(CLUSTER_NAME, CRUSHED_RESOURCE,
+ BuiltInStateModelDefinitions.LeaderStandby.name(), NUM_PARTITIONS,
NUM_REPLICAS,
+ NUM_REPLICAS - 1, DELAY_WINDOW,
CrushEdRebalanceStrategy.class.getName());
+
+ createResourceWithWagedRebalance(CLUSTER_NAME, WAGED_RESOURCE,
"MasterSlave",
+ NUM_PARTITIONS, NUM_REPLICAS, NUM_REPLICAS - 1);
+ Assert.assertTrue(_clusterVerifier.verifyByPolling());
+
+ // Disable all partitions on a first participant
+ MockParticipantManager disabledPartitionInstance = _participants[0];
+ _gSetupTool.getClusterManagementTool().enablePartition(false,
CLUSTER_NAME, disabledPartitionInstance.getInstanceName(),
+ InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY,
Collections.singletonList("foobar"));
+
+ Assert.assertTrue(_clusterVerifier.verifyByPolling());
+ verifier(() -> {
+ for (String resource : resources) {
+ ExternalView ev =
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
resource);
+ // EV should not have disabled instance above the lowest state
(OFFLINE)
+ for (String partition : ev.getPartitionSet()) {
+ if
(ev.getStateMap(partition).containsKey(disabledPartitionInstance.getInstanceName())
&&
+
!ev.getStateMap(partition).get(disabledPartitionInstance.getInstanceName()).equals("OFFLINE"))
{
+ return false;
+ }
+ }
+ }
+ return true;
+ }, 5000);
+
+ _gSetupTool.getClusterManagementTool().enablePartition(true, CLUSTER_NAME,
disabledPartitionInstance.getInstanceName(),
+ InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY,
Collections.singletonList("foobar"));
+
+ // Assert node still has CRUSHED assignment and that it is not in OFFLINE
state
+ // Cannot assert for WAGED resource as we can't guarantee that the node
will be in subsequent assignments
+ Assert.assertTrue(_clusterVerifier.verifyByPolling());
+ verifier(() -> {
+ ExternalView ev =
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
CRUSHED_RESOURCE);
+ boolean assignmentSeen = false;
+ for (String partition : ev.getPartitionSet()) {
+ if
(ev.getStateMap(partition).containsKey(disabledPartitionInstance.getInstanceName()))
{
+ assignmentSeen = true;
+ if
(ev.getStateMap(partition).get(disabledPartitionInstance.getInstanceName()).equals("OFFLINE"))
{
+ return false;
+ }
+ }
+ }
+ return assignmentSeen;
+ }, 5000);
+
+ // Cleanup test resources
+ for (String resource : resources) {
+ _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME,
resource);
+ }
+ }
+
+ private static void verifier(TestHelper.Verifier verifier, long timeout)
throws Exception {
+ Assert.assertTrue(TestHelper.verify(() -> {
+ try {
+ boolean result = verifier.verify();
+ if (!result) {
+ LOG.error("Verifier returned false, retrying...");
+ }
+ return result;
+ } catch (AssertionError e) {
+ LOG.error("Caught AssertionError on verifier attempt: ", e);
+ return false;
+ }
+ }, timeout));
+ }
}
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestInstanceOperation.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestInstanceOperation.java
index 67b575f0c..cb29c5849 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestInstanceOperation.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestInstanceOperation.java
@@ -540,7 +540,7 @@ public class TestInstanceOperation extends ZkTestBase {
swapOutInstancesToSwapInInstances.put(instanceToSwapOutName,
instanceToSwapInName);
addParticipant(instanceToSwapInName,
instanceToSwapOutInstanceConfig.getLogicalId(LOGICAL_ID),
instanceToSwapOutInstanceConfig.getDomainAsMap().get(ZONE),
- InstanceConstants.InstanceOperation.SWAP_IN, -1,
instanceToSwapInInstanceConfigListener);
+ InstanceConstants.InstanceOperation.SWAP_IN, -1,
instanceToSwapInInstanceConfigListener, null);
// Validate that the throttles are off since the InstanceOperation is set
to SWAP_IN
Assert.assertFalse(instanceToSwapInInstanceConfigListener.isThrottlesEnabled());
@@ -1344,10 +1344,10 @@ public class TestInstanceOperation extends ZkTestBase {
v -> v.equals("MASTER") || v.equals("LEADER") ||
v.equals("SLAVE") || v.equals(
"FOLLOWER") || v.equals("STANDBY"))
.forEach(v -> activeReplicaCount.getAndIncrement());
- if (activeReplicaCount.get() < REPLICA - 1 || (
- ev.getStateMap(partition).containsKey(evacuateInstanceName) &&
ev.getStateMap(
- partition).get(evacuateInstanceName).equals("MASTER") &&
ev.getStateMap(
- partition).get(evacuateInstanceName).equals("LEADER"))) {
+ // If min active replicas violated OR if instance is evacuating and
is top state for partition
+ if (activeReplicaCount.get() < REPLICA - 1 ||
(ev.getStateMap(partition).containsKey(evacuateInstanceName) &&
+
(ev.getStateMap(partition).get(evacuateInstanceName).equals("MASTER") ||
+
ev.getStateMap(partition).get(evacuateInstanceName).equals("LEADER")))) {
return false;
}
}
@@ -1361,6 +1361,99 @@ public class TestInstanceOperation extends ZkTestBase {
dropTestDBs(ImmutableSet.of("TEST_DB3_DELAYED_CRUSHED",
"TEST_DB4_DELAYED_WAGED"));
}
+ @Test(dependsOnMethods = "testEvacuationWithOfflineInstancesInCluster")
+ public void testEvacuateWithDisabledPartition() throws Exception {
+ System.out.println(
+ "START TestInstanceOperation.testEvacuateWithDisabledPartition() at "
+ new Date(
+ System.currentTimeMillis()));
+ StateTransitionCountStateModelFactory
stateTransitionCountStateModelFactory = new
StateTransitionCountStateModelFactory();
+ removeOfflineOrInactiveInstances();
+ String testCrushedDBName = "testEvacuateWithDisabledPartition_CRUSHED_DB0";
+ String testWagedDBName = "testEvacuateWithDisabledPartition_WAGED_DB1";
+ String toDisableThenEvacuateInstanceName = "disable_then_evacuate_host";
+ addParticipant(toDisableThenEvacuateInstanceName,
stateTransitionCountStateModelFactory);
+ MockParticipantManager toDisableThenEvacuateParticipant =
_participants.get(_participants.size() - 1);
+
+ List<String> testResources = Arrays.asList(testCrushedDBName,
testWagedDBName);
+ createResourceWithDelayedRebalance(CLUSTER_NAME, testCrushedDBName,
"MasterSlave",
+ PARTITIONS, REPLICA, REPLICA-1, 200000,
CrushEdRebalanceStrategy.class.getName());
+ createResourceWithWagedRebalance(CLUSTER_NAME, testWagedDBName,
"MasterSlave", PARTITIONS,
+ REPLICA, REPLICA-1);
+
+ Assert.assertTrue(_clusterVerifier.verifyByPolling());
+ int upwardSTCountBeforeDisableThenEvacuate =
stateTransitionCountStateModelFactory.getUpwardStateTransitionCounter();
+ int downwardSTCountBeforeDisableThenEvacuate =
stateTransitionCountStateModelFactory.getDownwardStateTransitionCounter();
+
+
+ InstanceConfig instanceConfig =
_gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME,
+ toDisableThenEvacuateInstanceName);
+
instanceConfig.setInstanceEnabledForPartition(InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY,
"", false);
+ _gSetupTool.getClusterManagementTool().setInstanceConfig(CLUSTER_NAME,
toDisableThenEvacuateInstanceName, instanceConfig);
+ Assert.assertTrue(_clusterVerifier.verifyByPolling());
+ // EV should not have disabled instance above the lowest state (OFFLINE)
+ verifier(() -> {
+ for (String resource : testResources) {
+ ExternalView ev =
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
resource);
+ for (String partition : ev.getPartitionSet()) {
+ if
(ev.getStateMap(partition).containsKey(toDisableThenEvacuateInstanceName) &&
!ev.getStateMap(partition).
+ get(toDisableThenEvacuateInstanceName).equals("OFFLINE")) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }, 5000);
+
+ // Assert node received downward state transitions and no upward
transitions
+
Assert.assertEquals(stateTransitionCountStateModelFactory.getUpwardStateTransitionCounter(),
+ upwardSTCountBeforeDisableThenEvacuate, "Upward state transitions
should not have been received");
+
Assert.assertTrue(stateTransitionCountStateModelFactory.getDownwardStateTransitionCounter()
>
+ downwardSTCountBeforeDisableThenEvacuate, "Should have received
downward state transitions");
+
+ _gSetupTool.getClusterManagementTool().setInstanceOperation(CLUSTER_NAME,
+ toDisableThenEvacuateInstanceName,
InstanceConstants.InstanceOperation.EVACUATE);
+
+ verifier(() -> _admin.isEvacuateFinished(CLUSTER_NAME,
toDisableThenEvacuateInstanceName), 30000);
+ int downwardSTCountAfterEvacuateComplete =
stateTransitionCountStateModelFactory.getDownwardStateTransitionCounter();
+
+ // Assert node received no upward state transitions after evacuation was
called on already disabled node
+
Assert.assertEquals(stateTransitionCountStateModelFactory.getUpwardStateTransitionCounter(),
+ upwardSTCountBeforeDisableThenEvacuate, "Upward state transitions
should not have been received");
+
+ // Re-enable all partitions for the instance
+ instanceConfig =
_gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME,
+ toDisableThenEvacuateInstanceName);
+
instanceConfig.setInstanceEnabledForPartition(InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY,
"", true);
+ _gSetupTool.getClusterManagementTool().setInstanceConfig(CLUSTER_NAME,
toDisableThenEvacuateInstanceName, instanceConfig);
+ Assert.assertTrue(_clusterVerifier.verifyByPolling());
+
+ // Assert node received no upward state transitions after re-enabled
partitions
+
Assert.assertEquals(stateTransitionCountStateModelFactory.getUpwardStateTransitionCounter(),
+ upwardSTCountBeforeDisableThenEvacuate, "Upward state transitions
should not have been received");
+
+ // Disable all partitions for the instance again
+ instanceConfig =
_gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME,
+ toDisableThenEvacuateInstanceName);
+
instanceConfig.setInstanceEnabledForPartition(InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY,
"", true);
+ _gSetupTool.getClusterManagementTool().setInstanceConfig(CLUSTER_NAME,
toDisableThenEvacuateInstanceName, instanceConfig);
+ Assert.assertTrue(_clusterVerifier.verifyByPolling());
+
+ // Assert node received no upward state transitions after disabling
already evacuated node
+
Assert.assertEquals(stateTransitionCountStateModelFactory.getUpwardStateTransitionCounter(),
+ upwardSTCountBeforeDisableThenEvacuate, "Upward state transitions
should not have been received");
+
Assert.assertEquals(stateTransitionCountStateModelFactory.getDownwardStateTransitionCounter(),
+ downwardSTCountAfterEvacuateComplete, "Downward state transitions
should not have been received");
+
+
+ // Clean up test resources
+ for (String resource : testResources) {
+ _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME,
resource);
+ }
+ // Clean up test participant
+ toDisableThenEvacuateParticipant.syncStop();
+ removeOfflineOrInactiveInstances();
+ }
+
/**
* Verifies that the given verifier returns true within the given timeout.
Handles AssertionError
* by returning false, which TestHelper.verify will not do. Asserts that
return value from
@@ -1408,14 +1501,14 @@ public class TestInstanceOperation extends ZkTestBase {
}
}
- private MockParticipantManager createParticipant(String participantName)
throws Exception {
+ private MockParticipantManager createParticipant(String participantName,
StateModelFactory stateModelFactory) throws Exception {
// start dummy participants
MockParticipantManager participant =
new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, participantName, 10,
null);
StateMachineEngine stateMachine = participant.getStateMachineEngine();
- // Using a delayed state model
- StDelayMSStateModelFactory delayFactory = new StDelayMSStateModelFactory();
- stateMachine.registerStateModelFactory("MasterSlave", delayFactory);
+ // Default to delayed statemodel if stateModel not provided
+ stateMachine.registerStateModelFactory("MasterSlave", stateModelFactory !=
null ?
+ stateModelFactory : new StDelayMSStateModelFactory());
return participant;
}
@@ -1424,15 +1517,20 @@ public class TestInstanceOperation extends ZkTestBase {
"zone_" + _participants.size() % ZONE_COUNT, null, -1);
}
+ private void addParticipant(String participantName, StateModelFactory
stateModelFactory) throws Exception {
+ addParticipant(participantName, UUID.randomUUID().toString(),
+ "zone_" + _participants.size() % ZONE_COUNT, null, -1, null,
stateModelFactory);
+ }
+
private void addParticipant(String participantName, String logicalId, String
zone,
InstanceConstants.InstanceOperation instanceOperation, int capacity)
throws Exception {
- addParticipant(participantName, logicalId, zone, instanceOperation,
capacity, null);
+ addParticipant(participantName, logicalId, zone, instanceOperation,
capacity, null, null);
}
private void addParticipant(String participantName, String logicalId, String
zone,
InstanceConstants.InstanceOperation instanceOperation, int capacity,
- InstanceConfigChangeListener listener) throws Exception {
+ InstanceConfigChangeListener listener, StateModelFactory
stateModelFactory) throws Exception {
InstanceConfig config = new InstanceConfig.Builder().setDomain(
String.format("%s=%s, %s=%s, %s=%s", ZONE, zone, HOST,
participantName, LOGICAL_ID,
logicalId)).setInstanceOperation(instanceOperation)
@@ -1443,7 +1541,7 @@ public class TestInstanceOperation extends ZkTestBase {
}
_gSetupTool.getClusterManagementTool().addInstance(CLUSTER_NAME, config);
- MockParticipantManager participant = createParticipant(participantName);
+ MockParticipantManager participant = createParticipant(participantName,
stateModelFactory);
participant.syncStart();
if (listener != null) {
@@ -1719,4 +1817,65 @@ public class TestInstanceOperation extends ZkTestBase {
}
}
}
+
+ // State Transition Factory that has counters to track number of state
transitions. The counters are shared across
+ // all state models. You can register this state model for a single
participant if you need to isolate the counter.
+ public class StateTransitionCountStateModelFactory extends
StateModelFactory<StateTransitionCountStateModel> {
+
+ AtomicInteger _upwardStateTransitionCounter = new AtomicInteger(0);
+ AtomicInteger _downwardStateTransitionCounter = new AtomicInteger(0);
+
+ @Override
+ public StateTransitionCountStateModel createNewStateModel(String
resourceName, String partitionKey) {
+ StateTransitionCountStateModel model =
+ new StateTransitionCountStateModel(_upwardStateTransitionCounter,
_downwardStateTransitionCounter);
+ return model;
+ }
+
+ public int getUpwardStateTransitionCounter() {
+ return _upwardStateTransitionCounter.get();
+ }
+
+ public int getDownwardStateTransitionCounter() {
+ return _downwardStateTransitionCounter.get();
+ }
+ }
+
+ @StateModelInfo(initialState = "OFFLINE", states = {"MASTER", "SLAVE",
"ERROR"})
+ public class StateTransitionCountStateModel extends StateModel {
+ AtomicInteger _upwardStateTransitionCounter;
+ AtomicInteger _downwardStateTransitionCounter;
+ public StateTransitionCountStateModel(AtomicInteger
upwardStateTransitionCounter, AtomicInteger downwardStateTransitionCounter) {
+ _upwardStateTransitionCounter = upwardStateTransitionCounter;
+ _downwardStateTransitionCounter = downwardStateTransitionCounter;
+ }
+
+ @Transition(to = "SLAVE", from = "OFFLINE")
+ public void onBecomeSlaveFromOffline(Message message, NotificationContext
context) {
+ _upwardStateTransitionCounter.incrementAndGet();
+ }
+
+ @Transition(to = "MASTER", from = "SLAVE")
+ public void onBecomeMasterFromSlave(Message message, NotificationContext
context) {
+ _upwardStateTransitionCounter.incrementAndGet();
+ }
+
+ @Transition(to = "SLAVE", from = "MASTER")
+ public void onBecomeSlaveFromMaster(Message message, NotificationContext
context) {
+ _downwardStateTransitionCounter.incrementAndGet();
+ }
+
+ @Transition(to = "OFFLINE", from = "SLAVE")
+ public void onBecomeOfflineFromSlave(Message message, NotificationContext
context) {
+ _downwardStateTransitionCounter.incrementAndGet();
+ }
+
+ @Transition(to = "DROPPED", from = "OFFLINE")
+ public void onBecomeDroppedFromOffline(Message message,
NotificationContext context) {
+ _downwardStateTransitionCounter.incrementAndGet();
+ }
+ }
+
+
+
}
diff --git
a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestInstanceMonitor.java
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestInstanceMonitor.java
index 7cac92eb4..619bdfe6c 100644
---
a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestInstanceMonitor.java
+++
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestInstanceMonitor.java
@@ -29,6 +29,7 @@ import javax.management.ObjectName;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import org.apache.helix.constants.InstanceConstants;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -40,8 +41,8 @@ public class TestInstanceMonitor {
String testInstance = "testInstance";
String testDomain = "testDomain:key=value";
Set<String> tags = ImmutableSet.of("test", "DEFAULT");
- Map<String, List<String>> disabledPartitions =
- ImmutableMap.of("instance1", ImmutableList.of("partition1",
"partition2"));
+ Map<String, List<String>> disabledPartitions = ImmutableMap.of("instance1",
+ ImmutableList.of("partition1", "partition2",
InstanceConstants.ALL_RESOURCES_DISABLED_PARTITION_KEY));
InstanceMonitor monitor =
new InstanceMonitor(testCluster, testInstance, new
ObjectName(testDomain));
@@ -53,6 +54,7 @@ public class TestInstanceMonitor {
Assert.assertEquals(monitor.getEnabled(), 0L);
Assert.assertEquals(monitor.getTotalMessageReceived(), 0L);
Assert.assertEquals(monitor.getDisabledPartitions(), 0L);
+ Assert.assertEquals(monitor.getAllPartitionsDisabled(), 0L);
Assert.assertEquals(monitor.getMaxCapacityUsageGauge(), 0.0d);
// Update metrics.
@@ -70,6 +72,7 @@ public class TestInstanceMonitor {
Assert.assertEquals(monitor.getOnline(), 1L);
Assert.assertEquals(monitor.getEnabled(), 1L);
Assert.assertEquals(monitor.getDisabledPartitions(), 2L);
+ Assert.assertEquals(monitor.getAllPartitionsDisabled(), 1L);
Assert.assertEquals(monitor.getMaxCapacityUsageGauge(), 0.5d);
Assert.assertEquals(monitor.getMessageQueueSizeGauge(), 100L);
Assert.assertEquals(monitor.getPastDueMessageGauge(), 50L);