This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new aaf2294c8 Change stoppable to perform min active check sequentially in
mz (#2886)
aaf2294c8 is described below
commit aaf2294c884edc71f7eb815c774c6a9fad835856
Author: Grant Paláu Spencer <[email protected]>
AuthorDate: Wed Aug 28 14:27:35 2024 -0700
Change stoppable to perform min active check sequentially in mz (#2886)
Batch stoppable check now performs min_active check sequentially in zone
---
.../clusterMaintenanceService/HealthCheck.java | 5 +-
.../MaintenanceManagementService.java | 43 ++++++++++++++++
.../server/resources/helix/InstancesAccessor.java | 2 +
.../TestMaintenanceManagementService.java | 13 +++--
.../helix/rest/server/TestInstancesAccessor.java | 58 ++++++++++++++++++++++
5 files changed, 115 insertions(+), 6 deletions(-)
diff --git
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
index 992fe767c..0c0a3d37c 100644
---
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
+++
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
@@ -60,9 +60,8 @@ public enum HealthCheck {
* constraint if this instance is shutdown
*/
MIN_ACTIVE_REPLICA_CHECK_FAILED;
-
/**
- * Pre-defined list of checks to test if an instance can be stopped at
runtime
+ * Pre-defined list of checks to test if an instance can be stopped at
runtime.
*/
public static List<HealthCheck> STOPPABLE_CHECK_LIST =
Arrays.asList(HealthCheck.values());
/**
@@ -71,4 +70,4 @@ public enum HealthCheck {
public static List<HealthCheck> STARTED_AND_HEALTH_CHECK_LIST = ImmutableList
.of(INVALID_CONFIG, INSTANCE_NOT_ALIVE, INSTANCE_NOT_ENABLED,
INSTANCE_NOT_STABLE,
EMPTY_RESOURCE_ASSIGNMENT);
-}
\ No newline at end of file
+}
diff --git
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
index 52377e612..063e7107f 100644
---
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
+++
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
@@ -472,9 +472,15 @@ public class MaintenanceManagementService {
private List<String> batchHelixInstanceStoppableCheck(String clusterId,
Collection<String> instances, Map<String, StoppableCheck>
finalStoppableChecks,
Set<String> toBeStoppedInstances) {
+
+ // Perform all but min_active replicas check in parallel
Map<String, Future<StoppableCheck>> helixInstanceChecks =
instances.stream().collect(
Collectors.toMap(Function.identity(), instance -> POOL.submit(
() -> performHelixOwnInstanceCheck(clusterId, instance,
toBeStoppedInstances))));
+
+ // Perform min_active replicas check sequentially
+ addMinActiveReplicaChecks(clusterId, helixInstanceChecks,
toBeStoppedInstances);
+
// finalStoppableChecks contains instances that does not pass this health
check
return filterInstancesForNextCheck(helixInstanceChecks,
finalStoppableChecks);
}
@@ -640,6 +646,8 @@ public class MaintenanceManagementService {
LOG.info("Perform helix own custom health checks for {}/{}", clusterId,
instanceName);
List<HealthCheck> healthChecksToExecute = new
ArrayList<>(HealthCheck.STOPPABLE_CHECK_LIST);
healthChecksToExecute.removeAll(_skipStoppableHealthCheckList);
+ // Min active check is performed sequentially later
+ healthChecksToExecute.remove(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED);
Map<String, Boolean> helixStoppableCheck =
getInstanceHealthStatus(clusterId, instanceName, healthChecksToExecute,
toBeStoppedInstances);
@@ -800,6 +808,41 @@ public class MaintenanceManagementService {
return healthStatus;
}
+ // Adds the result of the min_active replica check for each stoppable check
passed in futureStoppableCheckByInstance
+ private void addMinActiveReplicaChecks(String clusterId, Map<String,
Future<StoppableCheck>> futureStoppableCheckByInstance,
+ Set<String> toBeStoppedInstances) {
+ // Do not perform check if in the skip list
+ if
(_skipStoppableHealthCheckList.contains(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED))
{
+ return;
+ }
+
+ Set<String> possibleToStopInstances = new HashSet<>(toBeStoppedInstances);
+ for (Map.Entry<String, Future<StoppableCheck>> entry :
futureStoppableCheckByInstance.entrySet()) {
+ try {
+ String instanceName = entry.getKey();
+ StoppableCheck stoppableCheck = entry.getValue().get();
+
+ // Check if min active will be violated and add to stoppableCheck. If
instance still stoppable,
+ // add to possibleToStopInstances
+ Map<String, Boolean> helixStoppableCheck =
+ getInstanceHealthStatus(clusterId, instanceName,
+
Collections.singletonList(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED),
+ possibleToStopInstances);
+ stoppableCheck.add(new StoppableCheck(helixStoppableCheck,
StoppableCheck.Category.HELIX_OWN_CHECK));
+
+ if (stoppableCheck.isStoppable()) {
+ possibleToStopInstances.add(instanceName);
+ }
+
+ } catch (Exception e) {
+ String errorMessage = String.format("Failed to get StoppableChecks in
parallel. Instances: %s",
+ futureStoppableCheckByInstance.values());
+ LOG.error(errorMessage, e);
+ throw new HelixException(errorMessage);
+ }
+ }
+ }
+
public static class MaintenanceManagementServiceBuilder {
private ConfigAccessor _configAccessor;
private boolean _skipZKRead;
diff --git
a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
index fcad387dc..d24ad9fce 100644
---
a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
+++
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
@@ -313,6 +313,8 @@ public class InstancesAccessor extends
AbstractHelixResource {
.build();
stoppableInstancesSelector.calculateOrderOfZone(instances, random);
ObjectNode result;
+ // TODO: Add support for clusters that do not have topology set up.
+ // Issue #2893: https://github.com/apache/helix/issues/2893
switch (selectionBase) {
case zone_based:
result =
stoppableInstancesSelector.getStoppableInstancesInSingleZone(instances,
toBeStoppedInstances);
diff --git
a/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
b/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
index a49a95066..6c207bd10 100644
---
a/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
+++
b/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
@@ -83,6 +83,7 @@ public class TestMaintenanceManagementService {
RESTConfig restConfig = new RESTConfig("restConfig");
restConfig.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL,
"http://*:123/path");
when(_configAccessor.getRESTConfig(TEST_CLUSTER)).thenReturn(restConfig);
+ when(_dataAccessorWrapper.keyBuilder()).thenReturn(new
PropertyKey.Builder(TEST_CLUSTER));
}
class MockMaintenanceManagementService extends MaintenanceManagementService {
@@ -124,10 +125,13 @@ public class TestMaintenanceManagementService {
Map<String, Boolean> failedCheck = ImmutableMap.of("FailCheck", false);
MockMaintenanceManagementService service =
new MockMaintenanceManagementService(_dataAccessorWrapper,
_configAccessor,
- _customRestClient, false, false,
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
+ _customRestClient, false, false, null,
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
@Override
protected Map<String, Boolean> getInstanceHealthStatus(String
clusterId,
String instanceName, List<HealthCheck> healthChecks, Set<String>
toBeStoppedInstances) {
+ if
(Collections.singletonList(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED).equals(healthChecks))
{
+ return Collections.emptyMap();
+ }
return failedCheck;
}
};
@@ -144,7 +148,7 @@ public class TestMaintenanceManagementService {
public void testGetInstanceStoppableCheckWhenCustomInstanceCheckFail()
throws IOException {
MockMaintenanceManagementService service =
new MockMaintenanceManagementService(_dataAccessorWrapper,
_configAccessor,
- _customRestClient, false, false,
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
+ _customRestClient, false, false, null,
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
@Override
protected Map<String, Boolean> getInstanceHealthStatus(String
clusterId,
String instanceName, List<HealthCheck> healthChecks, Set<String>
toBeStoppedInstances) {
@@ -243,7 +247,7 @@ public class TestMaintenanceManagementService {
public void testGetInstanceStoppableCheckConnectionRefused() throws
IOException {
MockMaintenanceManagementService service =
new MockMaintenanceManagementService(_dataAccessorWrapper,
_configAccessor,
- _customRestClient, false, false,
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
+ _customRestClient, false, false, null,
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
@Override
protected Map<String, Boolean> getInstanceHealthStatus(String
clusterId,
String instanceName, List<HealthCheck> healthChecks, Set<String>
toBeStoppedInstances) {
@@ -366,6 +370,9 @@ public class TestMaintenanceManagementService {
@Override
protected Map<String, Boolean> getInstanceHealthStatus(String
clusterId,
String instanceName, List<HealthCheck> healthChecks, Set<String>
toBeStoppedInstances) {
+ if
(Collections.singletonList(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED).equals(healthChecks))
{
+ return Collections.emptyMap();
+ }
return instanceHealthFailedCheck;
}
};
diff --git
a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
index 0403083fb..e282df5e5 100644
---
a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
+++
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
@@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.helix.TestHelper;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.rest.server.resources.helix.InstancesAccessor;
import org.apache.helix.rest.server.util.JerseyUriRequestBuilder;
@@ -288,6 +289,10 @@ public class TestInstancesAccessor extends
AbstractTestClass {
ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"),
ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST"));
+
instanceConfig.setInstanceOperation(InstanceConstants.InstanceOperation.ENABLE);
+ _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance0,
instanceConfig);
+
instanceConfig1.setInstanceOperation(InstanceConstants.InstanceOperation.ENABLE);
+ _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance1,
instanceConfig1);
System.out.println("End test :" + TestHelper.getTestMethodName());
}
@@ -519,6 +524,59 @@ public class TestInstancesAccessor extends
AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+ @Test(dependsOnMethods = "testValidateWeightForAllInstances")
+ public void testMultipleReplicasInSameMZ() throws Exception {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ // Create SemiAuto DB so that we can control assignment
+ String testDb = TestHelper.getTestMethodName() + "_resource";
+ _gSetupTool.getClusterManagementTool().addResource(STOPPABLE_CLUSTER2,
testDb, 3, "MasterSlave",
+ IdealState.RebalanceMode.SEMI_AUTO.toString());
+ _gSetupTool.getClusterManagementTool().rebalance(STOPPABLE_CLUSTER2,
testDb, 3);
+
+ // Manually set ideal state to have the 3 replcias assigned to 3 instances
all in the same zone
+ List<String> preferenceList = Arrays.asList("instance0", "instance1",
"instance2");
+ IdealState is =
_gSetupTool.getClusterManagementTool().getResourceIdealState(STOPPABLE_CLUSTER2,
testDb);
+ for (String p : is.getPartitionSet()) {
+ is.setPreferenceList(p, preferenceList);
+ }
+ is.setMinActiveReplicas(2);
+
_gSetupTool.getClusterManagementTool().setResourceIdealState(STOPPABLE_CLUSTER2,
testDb, is);
+
+ // Wait for assignments to take place
+ BestPossibleExternalViewVerifier verifier =
+ new
BestPossibleExternalViewVerifier.Builder(STOPPABLE_CLUSTER2).setZkAddr(ZK_ADDR).build();
+ Assert.assertTrue(verifier.verifyByPolling());
+
+ // Run stoppable check against the 3 instances where SemiAuto DB was
assigned
+ String content =
+ String.format("{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\"]}",
+ InstancesAccessor.InstancesProperties.selection_base.name(),
+ InstancesAccessor.InstanceHealthSelectionBase.zone_based.name(),
+ InstancesAccessor.InstancesProperties.instances.name(),
"instance0", "instance1",
+ "instance2");
+ Response response =
+ new
JerseyUriRequestBuilder("clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format(
+ STOPPABLE_CLUSTER2).post(this, Entity.entity(content,
MediaType.APPLICATION_JSON_TYPE));
+ JsonNode jsonNode =
OBJECT_MAPPER.readTree(response.readEntity(String.class));
+
+ // Resource has 3 replicas with min_active of 2
+ // First instance should be stoppable as min_active still satisfied
+ Set<String> stoppableSet = getStringSet(jsonNode,
+
InstancesAccessor.InstancesProperties.instance_stoppable_parallel.name());
+ Assert.assertTrue(Collections.singleton("instance0").equals(stoppableSet));
+
+ // Next 2 instances should fail stoppable due to
MIN_ACTIVE_REPLICA_CHECK_FAILED
+ JsonNode nonStoppableInstances = jsonNode.get(
+
InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name());
+ Assert.assertFalse(getStringSet(nonStoppableInstances, "instance0")
+ .contains("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+ Assert.assertTrue(getStringSet(nonStoppableInstances, "instance1")
+ .contains("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+ Assert.assertTrue(getStringSet(nonStoppableInstances, "instance2")
+ .contains("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
private Set<String> getStringSet(JsonNode jsonNode, String key) {
Set<String> result = new HashSet<>();
jsonNode.withArray(key).forEach(s -> result.add(s.textValue()));