This is an automated email from the ASF dual-hosted git repository.

jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/master by this push:
     new aaf2294c8 Change stoppable to perform min active check sequentially in 
mz (#2886)
aaf2294c8 is described below

commit aaf2294c884edc71f7eb815c774c6a9fad835856
Author: Grant Paláu Spencer <[email protected]>
AuthorDate: Wed Aug 28 14:27:35 2024 -0700

    Change stoppable to perform min active check sequentially in mz (#2886)
    
    Batch stoppable check now performs min_active check sequentially in zone
---
 .../clusterMaintenanceService/HealthCheck.java     |  5 +-
 .../MaintenanceManagementService.java              | 43 ++++++++++++++++
 .../server/resources/helix/InstancesAccessor.java  |  2 +
 .../TestMaintenanceManagementService.java          | 13 +++--
 .../helix/rest/server/TestInstancesAccessor.java   | 58 ++++++++++++++++++++++
 5 files changed, 115 insertions(+), 6 deletions(-)

diff --git 
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
 
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
index 992fe767c..0c0a3d37c 100644
--- 
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
+++ 
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/HealthCheck.java
@@ -60,9 +60,8 @@ public enum HealthCheck {
    * constraint if this instance is shutdown
    */
   MIN_ACTIVE_REPLICA_CHECK_FAILED;
-
   /**
-   * Pre-defined list of checks to test if an instance can be stopped at 
runtime
+   * Pre-defined list of checks to test if an instance can be stopped at 
runtime.
    */
   public static List<HealthCheck> STOPPABLE_CHECK_LIST = 
Arrays.asList(HealthCheck.values());
   /**
@@ -71,4 +70,4 @@ public enum HealthCheck {
   public static List<HealthCheck> STARTED_AND_HEALTH_CHECK_LIST = ImmutableList
       .of(INVALID_CONFIG, INSTANCE_NOT_ALIVE, INSTANCE_NOT_ENABLED, 
INSTANCE_NOT_STABLE,
           EMPTY_RESOURCE_ASSIGNMENT);
-}
\ No newline at end of file
+}
diff --git 
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
 
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
index 52377e612..063e7107f 100644
--- 
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
+++ 
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
@@ -472,9 +472,15 @@ public class MaintenanceManagementService {
   private List<String> batchHelixInstanceStoppableCheck(String clusterId,
       Collection<String> instances, Map<String, StoppableCheck> 
finalStoppableChecks,
       Set<String> toBeStoppedInstances) {
+
+    // Perform all but min_active replicas check in parallel
     Map<String, Future<StoppableCheck>> helixInstanceChecks = 
instances.stream().collect(
         Collectors.toMap(Function.identity(), instance -> POOL.submit(
             () -> performHelixOwnInstanceCheck(clusterId, instance, 
toBeStoppedInstances))));
+
+    // Perform min_active replicas check sequentially
+    addMinActiveReplicaChecks(clusterId, helixInstanceChecks, 
toBeStoppedInstances);
+
     // finalStoppableChecks contains instances that does not pass this health 
check
     return filterInstancesForNextCheck(helixInstanceChecks, 
finalStoppableChecks);
   }
@@ -640,6 +646,8 @@ public class MaintenanceManagementService {
     LOG.info("Perform helix own custom health checks for {}/{}", clusterId, 
instanceName);
     List<HealthCheck> healthChecksToExecute = new 
ArrayList<>(HealthCheck.STOPPABLE_CHECK_LIST);
     healthChecksToExecute.removeAll(_skipStoppableHealthCheckList);
+    // Min active check is performed sequentially later
+    healthChecksToExecute.remove(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED);
     Map<String, Boolean> helixStoppableCheck =
         getInstanceHealthStatus(clusterId, instanceName, healthChecksToExecute,
             toBeStoppedInstances);
@@ -800,6 +808,41 @@ public class MaintenanceManagementService {
     return healthStatus;
   }
 
+  // Adds the result of the min_active replica check for each stoppable check 
passed in futureStoppableCheckByInstance
+  private void addMinActiveReplicaChecks(String clusterId, Map<String, 
Future<StoppableCheck>> futureStoppableCheckByInstance,
+      Set<String> toBeStoppedInstances) {
+    // Do not perform check if in the skip list
+    if 
(_skipStoppableHealthCheckList.contains(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED))
 {
+      return;
+    }
+
+    Set<String> possibleToStopInstances = new HashSet<>(toBeStoppedInstances);
+    for (Map.Entry<String, Future<StoppableCheck>> entry : 
futureStoppableCheckByInstance.entrySet()) {
+      try {
+        String instanceName = entry.getKey();
+        StoppableCheck stoppableCheck = entry.getValue().get();
+
+        // Check if min active will be violated and add to stoppableCheck. If 
instance still stoppable,
+        // add to possibleToStopInstances
+        Map<String, Boolean> helixStoppableCheck =
+            getInstanceHealthStatus(clusterId, instanceName,
+                
Collections.singletonList(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED),
+                possibleToStopInstances);
+        stoppableCheck.add(new StoppableCheck(helixStoppableCheck, 
StoppableCheck.Category.HELIX_OWN_CHECK));
+
+        if (stoppableCheck.isStoppable()) {
+          possibleToStopInstances.add(instanceName);
+        }
+
+      } catch (Exception e) {
+        String errorMessage = String.format("Failed to get StoppableChecks in 
parallel. Instances: %s",
+            futureStoppableCheckByInstance.values());
+        LOG.error(errorMessage, e);
+        throw new HelixException(errorMessage);
+      }
+    }
+  }
+
   public static class MaintenanceManagementServiceBuilder {
     private ConfigAccessor _configAccessor;
     private boolean _skipZKRead;
diff --git 
a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
 
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
index fcad387dc..d24ad9fce 100644
--- 
a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
+++ 
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
@@ -313,6 +313,8 @@ public class InstancesAccessor extends 
AbstractHelixResource {
               .build();
       stoppableInstancesSelector.calculateOrderOfZone(instances, random);
       ObjectNode result;
+      // TODO: Add support for clusters that do not have topology set up.
+      // Issue #2893: https://github.com/apache/helix/issues/2893
       switch (selectionBase) {
         case zone_based:
           result = 
stoppableInstancesSelector.getStoppableInstancesInSingleZone(instances, 
toBeStoppedInstances);
diff --git 
a/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
 
b/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
index a49a95066..6c207bd10 100644
--- 
a/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
+++ 
b/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java
@@ -83,6 +83,7 @@ public class TestMaintenanceManagementService {
     RESTConfig restConfig = new RESTConfig("restConfig");
     restConfig.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, 
"http://*:123/path";);
     when(_configAccessor.getRESTConfig(TEST_CLUSTER)).thenReturn(restConfig);
+    when(_dataAccessorWrapper.keyBuilder()).thenReturn(new 
PropertyKey.Builder(TEST_CLUSTER));
   }
 
   class MockMaintenanceManagementService extends MaintenanceManagementService {
@@ -124,10 +125,13 @@ public class TestMaintenanceManagementService {
     Map<String, Boolean> failedCheck = ImmutableMap.of("FailCheck", false);
     MockMaintenanceManagementService service =
         new MockMaintenanceManagementService(_dataAccessorWrapper, 
_configAccessor,
-            _customRestClient, false, false, 
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
+            _customRestClient, false, false, null, 
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String 
clusterId,
               String instanceName, List<HealthCheck> healthChecks, Set<String> 
toBeStoppedInstances) {
+            if 
(Collections.singletonList(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED).equals(healthChecks))
 {
+              return Collections.emptyMap();
+            }
             return failedCheck;
           }
         };
@@ -144,7 +148,7 @@ public class TestMaintenanceManagementService {
   public void testGetInstanceStoppableCheckWhenCustomInstanceCheckFail() 
throws IOException {
     MockMaintenanceManagementService service =
         new MockMaintenanceManagementService(_dataAccessorWrapper, 
_configAccessor,
-            _customRestClient, false, false, 
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
+            _customRestClient, false, false, null, 
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String 
clusterId,
               String instanceName, List<HealthCheck> healthChecks, Set<String> 
toBeStoppedInstances) {
@@ -243,7 +247,7 @@ public class TestMaintenanceManagementService {
   public void testGetInstanceStoppableCheckConnectionRefused() throws 
IOException {
     MockMaintenanceManagementService service =
         new MockMaintenanceManagementService(_dataAccessorWrapper, 
_configAccessor,
-            _customRestClient, false, false, 
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
+            _customRestClient, false, false, null, 
HelixRestNamespace.DEFAULT_NAMESPACE_NAME) {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String 
clusterId,
               String instanceName, List<HealthCheck> healthChecks, Set<String> 
toBeStoppedInstances) {
@@ -366,6 +370,9 @@ public class TestMaintenanceManagementService {
           @Override
           protected Map<String, Boolean> getInstanceHealthStatus(String 
clusterId,
               String instanceName, List<HealthCheck> healthChecks, Set<String> 
toBeStoppedInstances) {
+            if 
(Collections.singletonList(HealthCheck.MIN_ACTIVE_REPLICA_CHECK_FAILED).equals(healthChecks))
 {
+              return Collections.emptyMap();
+            }
             return instanceHealthFailedCheck;
           }
         };
diff --git 
a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
 
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
index 0403083fb..e282df5e5 100644
--- 
a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
+++ 
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
@@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableSet;
 import org.apache.helix.TestHelper;
 import org.apache.helix.constants.InstanceConstants;
 import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.IdealState;
 import org.apache.helix.model.InstanceConfig;
 import org.apache.helix.rest.server.resources.helix.InstancesAccessor;
 import org.apache.helix.rest.server.util.JerseyUriRequestBuilder;
@@ -288,6 +289,10 @@ public class TestInstancesAccessor extends 
AbstractTestClass {
         ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
     Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"),
         ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST"));
+    
instanceConfig.setInstanceOperation(InstanceConstants.InstanceOperation.ENABLE);
+    _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance0, 
instanceConfig);
+    
instanceConfig1.setInstanceOperation(InstanceConstants.InstanceOperation.ENABLE);
+    _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance1, 
instanceConfig1);
     System.out.println("End test :" + TestHelper.getTestMethodName());
   }
 
@@ -519,6 +524,59 @@ public class TestInstancesAccessor extends 
AbstractTestClass {
     System.out.println("End test :" + TestHelper.getTestMethodName());
   }
 
+  @Test(dependsOnMethods = "testValidateWeightForAllInstances")
+  public void testMultipleReplicasInSameMZ() throws Exception {
+    System.out.println("Start test :" + TestHelper.getTestMethodName());
+    // Create SemiAuto DB so that we can control assignment
+    String testDb = TestHelper.getTestMethodName() + "_resource";
+    _gSetupTool.getClusterManagementTool().addResource(STOPPABLE_CLUSTER2, 
testDb, 3, "MasterSlave",
+        IdealState.RebalanceMode.SEMI_AUTO.toString());
+    _gSetupTool.getClusterManagementTool().rebalance(STOPPABLE_CLUSTER2, 
testDb, 3);
+
+    // Manually set ideal state to have the 3 replcias assigned to 3 instances 
all in the same zone
+    List<String> preferenceList = Arrays.asList("instance0", "instance1", 
"instance2");
+    IdealState is = 
_gSetupTool.getClusterManagementTool().getResourceIdealState(STOPPABLE_CLUSTER2,
 testDb);
+    for (String p : is.getPartitionSet()) {
+      is.setPreferenceList(p, preferenceList);
+    }
+    is.setMinActiveReplicas(2);
+    
_gSetupTool.getClusterManagementTool().setResourceIdealState(STOPPABLE_CLUSTER2,
 testDb, is);
+
+    // Wait for assignments to take place
+    BestPossibleExternalViewVerifier verifier =
+        new 
BestPossibleExternalViewVerifier.Builder(STOPPABLE_CLUSTER2).setZkAddr(ZK_ADDR).build();
+    Assert.assertTrue(verifier.verifyByPolling());
+
+    // Run stoppable check against the 3 instances where SemiAuto DB was 
assigned
+    String content =
+        String.format("{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\"]}",
+            InstancesAccessor.InstancesProperties.selection_base.name(),
+            InstancesAccessor.InstanceHealthSelectionBase.zone_based.name(),
+            InstancesAccessor.InstancesProperties.instances.name(), 
"instance0", "instance1",
+            "instance2");
+    Response response =
+        new 
JerseyUriRequestBuilder("clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format(
+            STOPPABLE_CLUSTER2).post(this, Entity.entity(content, 
MediaType.APPLICATION_JSON_TYPE));
+    JsonNode jsonNode = 
OBJECT_MAPPER.readTree(response.readEntity(String.class));
+
+    // Resource has 3 replicas with min_active of 2
+    // First instance should be stoppable as min_active still satisfied
+    Set<String> stoppableSet = getStringSet(jsonNode,
+        
InstancesAccessor.InstancesProperties.instance_stoppable_parallel.name());
+    Assert.assertTrue(Collections.singleton("instance0").equals(stoppableSet));
+
+    // Next 2 instances should fail stoppable due to 
MIN_ACTIVE_REPLICA_CHECK_FAILED
+    JsonNode nonStoppableInstances = jsonNode.get(
+        
InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name());
+    Assert.assertFalse(getStringSet(nonStoppableInstances, "instance0")
+        .contains("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+    Assert.assertTrue(getStringSet(nonStoppableInstances, "instance1")
+        .contains("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+    Assert.assertTrue(getStringSet(nonStoppableInstances, "instance2")
+        .contains("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+    System.out.println("End test :" + TestHelper.getTestMethodName());
+  }
+
   private Set<String> getStringSet(JsonNode jsonNode, String key) {
     Set<String> result = new HashSet<>();
     jsonNode.withArray(key).forEach(s -> result.add(s.textValue()));

Reply via email to