This is an automated email from the ASF dual-hosted git repository.

xyuanlu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git


The following commit(s) were added to refs/heads/master by this push:
     new 359b8f5cb Fix new resource causing pipeline NPE when rebalance 
overwrites required (#2914)
359b8f5cb is described below

commit 359b8f5cbb3863f4f3ccf60c58e4f010d04c60c9
Author: Grant Paláu Spencer <[email protected]>
AuthorDate: Tue Sep 17 14:14:10 2024 -0700

    Fix new resource causing pipeline NPE when rebalance overwrites required 
(#2914)
    
    *  Fix new resource causing pipeline NPE when rebalance overwrites required
---
 .../rebalancer/util/DelayedRebalanceUtil.java      |   4 +-
 ...sourceWhenRequireDelayedRebalanceOverwrite.java | 146 +++++++++++++++++++++
 2 files changed, 148 insertions(+), 2 deletions(-)

diff --git 
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/DelayedRebalanceUtil.java
 
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/DelayedRebalanceUtil.java
index 90da408b1..d45aa971b 100644
--- 
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/DelayedRebalanceUtil.java
+++ 
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/DelayedRebalanceUtil.java
@@ -38,7 +38,6 @@ import org.apache.helix.model.InstanceConfig;
 import org.apache.helix.model.Partition;
 import org.apache.helix.model.ResourceAssignment;
 import org.apache.helix.model.ResourceConfig;
-import org.apache.helix.util.InstanceValidationUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -321,7 +320,8 @@ public class DelayedRebalanceUtil {
     Set<AssignableReplica> toBeAssignedReplicas = new HashSet<>();
 
     for (String resourceName : resources) {
-      ResourceAssignment resourceAssignment = 
currentAssignment.get(resourceName);
+      ResourceAssignment resourceAssignment = 
currentAssignment.getOrDefault(resourceName,
+          new ResourceAssignment(resourceName));
       IdealState idealState = clusterData.getIdealState(resourceName);
       String modelDef = idealState.getStateModelDefRef();
       Map<String, Integer> statePriorityMap = 
clusterData.getStateModelDef(modelDef).getStatePriorityMap();
diff --git 
a/helix-core/src/test/java/org/apache/helix/integration/TestAddResourceWhenRequireDelayedRebalanceOverwrite.java
 
b/helix-core/src/test/java/org/apache/helix/integration/TestAddResourceWhenRequireDelayedRebalanceOverwrite.java
new file mode 100644
index 000000000..7c93c33b6
--- /dev/null
+++ 
b/helix-core/src/test/java/org/apache/helix/integration/TestAddResourceWhenRequireDelayedRebalanceOverwrite.java
@@ -0,0 +1,146 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.helix.ConfigAccessor;
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.TestHelper;
+import org.apache.helix.common.ZkTestBase;
+import org.apache.helix.constants.InstanceConstants;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.ExternalView;
+import 
org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+// This test was created to reproduce issue seen in 
https://github.com/apache/helix/issues/2909
+public class TestAddResourceWhenRequireDelayedRebalanceOverwrite extends 
ZkTestBase {
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(TestForceKillInstance.class);
+  protected final String CLASS_NAME = getShortClassName();
+  protected final String CLUSTER_NAME = CLUSTER_PREFIX + "_" + CLASS_NAME;
+  private static final int NUM_REPLICAS = 3;
+  private static final int NUM_PARTITIONS = 10;
+  private static final int NUM_MIN_ACTIVE_REPLICAS = 2;
+  private static final int NUM_NODES = 5;
+  private BestPossibleExternalViewVerifier _bestPossibleClusterVerifier;
+  private HelixAdmin _admin;
+  private List<MockParticipantManager> _participants = new ArrayList<>();
+  private ClusterControllerManager _controller;
+  private ArrayList<String> _resources = new ArrayList<>();
+
+  @BeforeClass
+  public void beforeClass() throws Exception {
+    System.out.println("START " + CLASS_NAME + " at " + new 
Date(System.currentTimeMillis()));
+    _gSetupTool.addCluster(CLUSTER_NAME, true);
+    enablePersistIntermediateAssignment(_gZkClient, CLUSTER_NAME, true);
+
+    ConfigAccessor configAccessor = new ConfigAccessor(_gZkClient);
+    ClusterConfig clusterConfig = 
configAccessor.getClusterConfig(CLUSTER_NAME);
+    clusterConfig.setRebalanceDelayTime(2000000L);
+    configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
+
+    for (int i = 0; i < NUM_NODES; i++) {
+      String instanceName = "localhost_" + i;
+      addParticipant(CLUSTER_NAME, instanceName);
+    }
+
+    // start controller
+    String controllerName = CONTROLLER_PREFIX + "_0";
+    _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, 
controllerName);
+    _controller.syncStart();
+
+    _admin = new ZKHelixAdmin(_gZkClient);
+
+    _bestPossibleClusterVerifier = new 
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
+        
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME).build();
+  }
+
+  // Control test to verify that the cluster is in a good state and is not 
failing for other reasons
+  @Test
+  public void testAddResourceNewCluster() {
+    System.out.println("START " + TestHelper.getTestClassName() + "." + 
TestHelper.getTestMethodName() + " at "
+        + new Date(System.currentTimeMillis()));
+
+    String resourceName = "TestWagedDB";
+    _resources.add(resourceName);
+    createResourceWithWagedRebalance(CLUSTER_NAME, resourceName, 
"MasterSlave", NUM_PARTITIONS, NUM_REPLICAS,
+        NUM_MIN_ACTIVE_REPLICAS);
+
+    Assert.assertTrue(_bestPossibleClusterVerifier.verify());
+    Assert.assertTrue(isMinActiveSatisfied());
+  }
+
+  // This test was created to reproduce issue seen in 
https://github.com/apache/helix/issues/2909
+  // When delay rebalance overwrites are required, adding a new resource to 
the cluster will cause an NPE
+  // and rebalance pipeline will fail until delay rebalance overwrites are no 
longer needed
+  // Delay rebalance overwrites are typically needed when sufficient number of 
instances are disabled within the delay window
+  @Test (dependsOnMethods = "testAddResourceNewCluster")
+  public void testAddResourceWhenInstancesDisabledWithinWindow() {
+    System.out.println("START " + TestHelper.getTestClassName() + "." + 
TestHelper.getTestMethodName() + " at "
+        + new Date(System.currentTimeMillis()));
+
+    // Disable 2 instances to require delay rebalance overwrites
+    for (int i = 0; i < 2; i++) {
+      _admin.setInstanceOperation(CLUSTER_NAME, 
_participants.get(i).getInstanceName(),
+          InstanceConstants.InstanceOperation.DISABLE);
+    }
+    // Wait for cluster to converge
+    Assert.assertTrue(_bestPossibleClusterVerifier.verify());
+    Assert.assertTrue(isMinActiveSatisfied());
+
+    // Add resource
+    String resourceName = "TestWagedDB2";
+    _resources.add(resourceName);
+    createResourceWithWagedRebalance(CLUSTER_NAME, resourceName, 
"MasterSlave", NUM_PARTITIONS, NUM_REPLICAS,
+        NUM_MIN_ACTIVE_REPLICAS);
+
+    // Cluster should be able to converge
+    Assert.assertTrue(_bestPossibleClusterVerifier.verify());
+    System.out.println("END " + TestHelper.getTestClassName() + "." + 
TestHelper.getTestMethodName() + " at "
+        + new Date(System.currentTimeMillis()));
+  }
+
+  private MockParticipantManager addParticipant(String cluster, String 
instanceName) {
+    _gSetupTool.addInstanceToCluster(cluster, instanceName);
+    MockParticipantManager toAddParticipant =
+        new MockParticipantManager(ZK_ADDR, cluster, instanceName);
+    _participants.add(toAddParticipant);
+    toAddParticipant.syncStart();
+    return toAddParticipant;
+  }
+
+  private Map<String, ExternalView> getEVs() {
+    Map<String, ExternalView> externalViews = new HashMap<String, 
ExternalView>();
+    for (String resource : _resources) {
+      ExternalView ev = 
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, 
resource);
+      externalViews.put(resource, ev);
+    }
+    return externalViews;
+  }
+
+  private boolean isMinActiveSatisfied() {
+    Map<String, ExternalView> externalViews = getEVs();
+    for (ExternalView ev : externalViews.values()) {
+      Assert.assertEquals(ev.getPartitionSet().size(), NUM_PARTITIONS);
+      for (String partition : ev.getPartitionSet()) {
+        long activeReplicas = ev.getStateMap(partition).values().stream()
+            .filter(state -> state.equals("SLAVE") || state.equals("MASTER"))
+            .count();
+        if (activeReplicas < NUM_MIN_ACTIVE_REPLICAS) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+}

Reply via email to