This is an automated email from the ASF dual-hosted git repository.
hzlu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new 12bfbae HelixClusterVerifier verify() with default waitTillVerify
time (#1450)
12bfbae is described below
commit 12bfbae5024d2b24e453921002e11cf71f98d047
Author: kaisun2000 <[email protected]>
AuthorDate: Thu Oct 8 16:37:08 2020 -0700
HelixClusterVerifier verify() with default waitTillVerify time (#1450)
HelixClusterVerifier verify() and related method may return
pre-maturely. The reason is that the verify the converging stable
condition too early before controller has a chance to make
change. Basically the previous stable state is mistaken as the
expected next stable state. Part two.
---
.../src/test/java/org/apache/helix/TestHelper.java | 2 --
.../controller/stages/TestRebalancePipeline.java | 1 +
.../WagedRebalancer/TestWagedNodeSwap.java | 6 ++--
.../WagedRebalancer/TestWagedRebalance.java | 16 +++++++----
.../TestWagedRebalanceFaultZone.java | 10 +++++--
.../spectator/TestRoutingTableProvider.java | 4 ++-
.../TestRoutingTableProviderFromCurrentStates.java | 11 ++++----
.../TestRoutingTableProviderPeriodicRefresh.java | 5 +++-
.../spectator/TestRoutingTableSnapshot.java | 6 ++--
.../helix/integration/task/TestDeleteWorkflow.java | 6 ++--
.../integration/task/TestRebalanceRunningTask.java | 4 ++-
.../handling/TestResourceThreadpoolSize.java | 6 +++-
.../TestClusterStatusMonitorLifecycle.java | 6 +++-
.../mbeans/TestClusterAggregateMetrics.java | 6 ++--
.../mbeans/TestDisableResourceMbean.java | 4 ++-
.../helix/task/TaskSynchronizedTestBase.java | 5 +++-
.../apache/helix/tools/TestClusterVerifier.java | 33 +++++++++++++---------
17 files changed, 87 insertions(+), 44 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/TestHelper.java
b/helix-core/src/test/java/org/apache/helix/TestHelper.java
index 7a6c122..c805158 100644
--- a/helix-core/src/test/java/org/apache/helix/TestHelper.java
+++ b/helix-core/src/test/java/org/apache/helix/TestHelper.java
@@ -76,10 +76,8 @@ import org.testng.Assert;
public class TestHelper {
private static final Logger LOG = LoggerFactory.getLogger(TestHelper.class);
-
public static final long WAIT_DURATION = 60 * 1000L; // 60 seconds
public static final int DEFAULT_REBALANCE_PROCESSING_WAIT_TIME = 1500;
-
/**
* Returns a unused random port.
*/
diff --git
a/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
b/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
index 95b0bdd..66c1771 100644
---
a/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
+++
b/helix-core/src/test/java/org/apache/helix/controller/stages/TestRebalancePipeline.java
@@ -158,6 +158,7 @@ public class TestRebalancePipeline extends ZkUnitTestBase {
Thread.sleep(2 * MessageGenerationPhase.DEFAULT_OBSELETE_MSG_PURGE_DELAY);
runPipeline(event, dataRefresh, false);
+ // Verify the stale message should be deleted
Assert.assertTrue(TestHelper.verify(() -> {
if (dataCache.getStaleMessages().size() != 0) {
return false;
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
index c34851e..e956896 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedNodeSwap.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.helix.ConfigAccessor;
+import org.apache.helix.TestHelper;
import org.apache.helix.common.ZkTestBase;
import org.apache.helix.integration.manager.ClusterControllerManager;
import org.apache.helix.integration.manager.MockParticipantManager;
@@ -120,10 +121,11 @@ public class TestWagedNodeSwap extends ZkTestBase {
_gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db, _replica);
_allDBs.add(db);
}
- Thread.sleep(1000);
_clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(_clusterVerifier.verify(5000));
}
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
index b778b3c..e0714cd 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
@@ -707,8 +707,14 @@ public class TestWagedRebalance extends ZkTestBase {
private void validate(int expectedReplica) {
HelixClusterVerifier _clusterVerifier =
new
StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
- Assert.assertTrue(_clusterVerifier.verify(5000));
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
+ try {
+ Assert.assertTrue(_clusterVerifier.verify(5000));
+ } finally {
+ _clusterVerifier.close();
+ }
for (String db : _allDBs) {
IdealState is =
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
@@ -716,7 +722,6 @@ public class TestWagedRebalance extends ZkTestBase {
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
db);
validateIsolation(is, ev, expectedReplica);
}
- _clusterVerifier.close();
}
/**
@@ -745,10 +750,11 @@ public class TestWagedRebalance extends ZkTestBase {
}
_allDBs.clear();
// waiting for all DB be dropped.
- Thread.sleep(100);
ZkHelixClusterVerifier _clusterVerifier =
new
StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
try {
Assert.assertTrue(_clusterVerifier.verifyByPolling());
} finally {
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
index 4a65ab8..8590a78 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.helix.ConfigAccessor;
+import org.apache.helix.TestHelper;
import org.apache.helix.common.ZkTestBase;
import org.apache.helix.integration.manager.ClusterControllerManager;
import org.apache.helix.integration.manager.MockParticipantManager;
@@ -304,7 +305,9 @@ public class TestWagedRebalanceFaultZone extends ZkTestBase
{
private void validate(int expectedReplica) {
ZkHelixClusterVerifier _clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
for (String db : _allDBs) {
@@ -347,9 +350,10 @@ public class TestWagedRebalanceFaultZone extends
ZkTestBase {
}
_allDBs.clear();
// waiting for all DB be dropped.
- Thread.sleep(100);
ZkHelixClusterVerifier _clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
}
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
index 6f22b19..93c73fb 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
@@ -163,7 +163,9 @@ public class TestRoutingTableProvider extends ZkTestBase {
_routingTableProvider_ev = new RoutingTableProvider(_spectator);
_routingTableProvider_cs = new RoutingTableProvider(_spectator,
PropertyType.CURRENTSTATES);
- _clusterVerifier = new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ _clusterVerifier = new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+ .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
}
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
index b051b89..5503067 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderFromCurrentStates.java
@@ -130,9 +130,10 @@ public class TestRoutingTableProviderFromCurrentStates
extends ZkTestBase {
long startTime = System.currentTimeMillis();
_gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db1, NUM_REPLICAS);
- Thread.sleep(1000L);
ZkHelixClusterVerifier clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(clusterVerifier.verifyByPolling());
validatePropagationLatency(PropertyType.CURRENTSTATES,
System.currentTimeMillis() - startTime);
@@ -148,7 +149,6 @@ public class TestRoutingTableProviderFromCurrentStates
extends ZkTestBase {
startTime = System.currentTimeMillis();
_gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, NUM_REPLICAS);
- Thread.sleep(1000L);
Assert.assertTrue(clusterVerifier.verifyByPolling());
validatePropagationLatency(PropertyType.CURRENTSTATES,
System.currentTimeMillis() - startTime);
@@ -160,7 +160,6 @@ public class TestRoutingTableProviderFromCurrentStates
extends ZkTestBase {
// shutdown an instance
startTime = System.currentTimeMillis();
_participants[0].syncStop();
- Thread.sleep(1000L);
Assert.assertTrue(clusterVerifier.verifyByPolling());
validatePropagationLatency(PropertyType.CURRENTSTATES,
System.currentTimeMillis() - startTime);
@@ -222,7 +221,9 @@ public class TestRoutingTableProviderFromCurrentStates
extends ZkTestBase {
IdealState.RebalanceMode.FULL_AUTO.name(),
CrushEdRebalanceStrategy.class.getName());
_gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db, NUM_REPLICAS);
ZkHelixClusterVerifier clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(clusterVerifier.verifyByPolling(5000, 500));
// 2. Process one event, so the current state will be refreshed with the
new DB partitions
routingTableCS.proceedNewEventHandling();
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
index 78f9ee1..d16165c 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProviderPeriodicRefresh.java
@@ -31,6 +31,7 @@ import org.apache.helix.HelixManagerFactory;
import org.apache.helix.InstanceType;
import org.apache.helix.NotificationContext;
import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
import org.apache.helix.common.ZkTestBase;
import org.apache.helix.integration.manager.ClusterControllerManager;
import org.apache.helix.integration.manager.MockParticipantManager;
@@ -136,7 +137,9 @@ public class TestRoutingTableProviderPeriodicRefresh
extends ZkTestBase {
_spectator_3.addInstanceConfigChangeListener(_routingTableProviderLongPeriodicRefresh);
_clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
}
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
index 470fec2..538c389 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableSnapshot.java
@@ -26,6 +26,7 @@ import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerFactory;
import org.apache.helix.InstanceType;
import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
import org.apache.helix.common.ZkTestBase;
import
org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.manager.ClusterControllerManager;
@@ -106,7 +107,9 @@ public class TestRoutingTableSnapshot extends ZkTestBase {
Thread.sleep(200);
ZkHelixClusterVerifier clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(clusterVerifier.verifyByPolling());
IdealState idealState1 =
@@ -128,7 +131,6 @@ public class TestRoutingTableSnapshot extends ZkTestBase {
// shutdown an instance
_participants[0].syncStop();
- Thread.sleep(200);
Assert.assertTrue(clusterVerifier.verifyByPolling());
// the original snapshot should not change
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
b/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
index 8d11b12..05977c7 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteWorkflow.java
@@ -153,8 +153,10 @@ public class TestDeleteWorkflow extends TaskTestBase {
accessor.removeProperty(keyBuild.workflowContext(jobQueueName));
BestPossibleExternalViewVerifier verifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setZkClient(_gZkClient).build();
+ new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME)
+ .setZkClient(_gZkClient)
+ .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(verifier.verifyByPolling());
// Sometimes it's a ZK write fail - delete one more time to lower test
failure rate
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
b/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
index d8fe719..058e9b4 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/task/TestRebalanceRunningTask.java
@@ -323,7 +323,9 @@ public final class TestRebalanceRunningTask extends
TaskSynchronizedTestBase {
startParticipant(_initialNumNodes);
ZkHelixClusterVerifier clusterVerifier =
new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
- .setResources(Sets.newHashSet(DATABASE)).build();
+ .setResources(Sets.newHashSet(DATABASE))
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(clusterVerifier.verify(10 * 1000));
// Running tasks are also rebalanced
diff --git
a/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
b/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
index f4444de..d550679 100644
---
a/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
+++
b/helix-core/src/test/java/org/apache/helix/messaging/handling/TestResourceThreadpoolSize.java
@@ -27,6 +27,7 @@ import java.util.concurrent.ThreadPoolExecutor;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixManager;
+import org.apache.helix.TestHelper;
import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.integration.task.WorkflowGenerator;
@@ -205,7 +206,10 @@ public class TestResourceThreadpoolSize extends
ZkStandAloneCMTestBase {
Assert.assertTrue(executor.getPoolSize() >= numberOfDbs);
BestPossibleExternalViewVerifier verifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
+ new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME)
+ .setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(verifier.verifyByPolling());
}
diff --git
a/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
b/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
index 207be0e..8668f8a 100644
---
a/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
+++
b/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
@@ -100,6 +100,7 @@ public class TestClusterStatusMonitorLifecycle extends
ZkTestBase {
ZkHelixClusterVerifier controllerClusterVerifier =
new
BestPossibleExternalViewVerifier.Builder(_controllerClusterName).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
.build();
Assert.assertTrue(controllerClusterVerifier.verifyByPolling(),
@@ -116,6 +117,7 @@ public class TestClusterStatusMonitorLifecycle extends
ZkTestBase {
ZkHelixClusterVerifier firstClusterVerifier =
new
BestPossibleExternalViewVerifier.Builder(_firstClusterName).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
.build();
Assert.assertTrue(firstClusterVerifier.verifyByPolling(), "first cluster
NOT in ideal state");
@@ -142,7 +144,9 @@ public class TestClusterStatusMonitorLifecycle extends
ZkTestBase {
for (int i = 1; i < clusterNb; i++) {
ZkHelixClusterVerifier clusterVerifier =
new BestPossibleExternalViewVerifier.Builder(_clusterNamePrefix +
"0_" + i)
- .setZkClient(_gZkClient).build();
+ .setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(clusterVerifier.verifyByPolling(), "Cluster NOT in
ideal state.");
}
}
diff --git
a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
index 6b671ae..ba52601 100644
---
a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
+++
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestClusterAggregateMetrics.java
@@ -145,8 +145,10 @@ public class TestClusterAggregateMetrics extends
ZkTestBase {
@Test
public void testAggregateMetrics() throws Exception {
BestPossibleExternalViewVerifier verifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setZkClient(_gZkClient).build();
+ new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME)
+ .setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
// Everything should be up and running initially with 5 total partitions
updateMetrics();
diff --git
a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
index 8cf12f4..63e1a81 100644
---
a/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
+++
b/helix-core/src/test/java/org/apache/helix/monitoring/mbeans/TestDisableResourceMbean.java
@@ -85,7 +85,9 @@ public class TestDisableResourceMbean extends ZkUnitTestBase {
controller.syncStart();
ZkHelixClusterVerifier clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(clusterName).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(clusterName).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(clusterVerifier.verifyByPolling());
// Verify the bean was created for TestDB0, but not for TestDB1.
diff --git
a/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
b/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
index 191065d..026ee3c 100644
---
a/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
+++
b/helix-core/src/test/java/org/apache/helix/task/TaskSynchronizedTestBase.java
@@ -28,6 +28,7 @@ import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerFactory;
import org.apache.helix.InstanceType;
+import org.apache.helix.TestHelper;
import org.apache.helix.common.ZkTestBase;
import
org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.manager.ClusterControllerManager;
@@ -76,7 +77,9 @@ public class TaskSynchronizedTestBase extends ZkTestBase {
startParticipants();
createManagers();
_clusterVerifier =
- new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
}
@AfterClass
diff --git
a/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
b/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
index 59c695a..bbafe22 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
@@ -126,24 +126,23 @@ public class TestClusterVerifier extends ZkUnitTestBase {
// Just ensure that the entire cluster passes
// ensure that the external view coalesces
HelixClusterVerifier bestPossibleVerifier =
- new
BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(bestPossibleVerifier.verify(10000));
// Disable partition for 1 instance, then Full-Auto ExternalView should
match IdealState.
_admin.enablePartition(false, _clusterName,
_participants[0].getInstanceName(),
FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] +
"_0"));
- Thread.sleep(1000);
Assert.assertTrue(bestPossibleVerifier.verify(3000));
// Enable the partition back
_admin.enablePartition(true, _clusterName,
_participants[0].getInstanceName(),
FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] +
"_0"));
- Thread.sleep(1000);
Assert.assertTrue(bestPossibleVerifier.verify(10000));
// Make 1 instance non-live
_participants[0].syncStop();
- Thread.sleep(1000);
Assert.assertTrue(bestPossibleVerifier.verify(10000));
// Recover the participant before next test
@@ -154,31 +153,32 @@ public class TestClusterVerifier extends ZkUnitTestBase {
HelixClusterVerifier strictMatchVerifier =
new StrictMatchExternalViewVerifier.Builder(_clusterName)
.setResources(Sets.newHashSet(RESOURCES)).setZkClient(_gZkClient)
- .setDeactivatedNodeAwareness(true).build();
+ .setDeactivatedNodeAwareness(true)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(strictMatchVerifier.verify(10000));
// Disable partition for 1 instance, then Full-Auto ExternalView should
match IdealState.
_admin.enablePartition(false, _clusterName,
_participants[0].getInstanceName(),
FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] +
"_0"));
- Thread.sleep(1000);
Assert.assertTrue(strictMatchVerifier.verify(3000));
// Enable the partition back
_admin.enablePartition(true, _clusterName,
_participants[0].getInstanceName(),
FULL_AUTO_RESOURCES[0], Lists.newArrayList(FULL_AUTO_RESOURCES[0] +
"_0"));
- Thread.sleep(1000);
Assert.assertTrue(strictMatchVerifier.verify(10000));
// Make 1 instance non-live
_participants[0].syncStop();
- Thread.sleep(1000);
// Semi-Auto ExternalView matching
for (String resource : SEMI_AUTO_RESOURCES) {
System.out.println("Verify resource: " + resource);
strictMatchVerifier =
new
StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
-
.setResources(Sets.newHashSet(resource)).setDeactivatedNodeAwareness(true).build();
+
.setResources(Sets.newHashSet(resource)).setDeactivatedNodeAwareness(true)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(strictMatchVerifier.verify(3000));
}
@@ -186,6 +186,7 @@ public class TestClusterVerifier extends ZkUnitTestBase {
strictMatchVerifier =
new
StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
.setResources(Sets.newHashSet(FULL_AUTO_RESOURCES)).setDeactivatedNodeAwareness(true)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
.build();
Assert.assertTrue(strictMatchVerifier.verify(10000));
}
@@ -204,10 +205,11 @@ public class TestClusterVerifier extends ZkUnitTestBase {
// Ensure that this passes even when one resource is down
_admin.enableInstance(_clusterName, "localhost_12918", false);
- Thread.sleep(1000);
ZkHelixClusterVerifier verifier =
new
BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
- .setResources(Sets.newHashSet(testDB)).build();
+ .setResources(Sets.newHashSet(testDB))
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(verifier.verifyByPolling());
_admin.enableCluster(_clusterName, false);
@@ -238,12 +240,16 @@ public class TestClusterVerifier extends ZkUnitTestBase {
public void testSleepTransition() throws InterruptedException {
HelixClusterVerifier bestPossibleVerifier =
- new
BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient).build();
+ new
BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(bestPossibleVerifier.verify(10000));
HelixClusterVerifier strictMatchVerifier =
new
StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
- .setDeactivatedNodeAwareness(true).build();
+ .setDeactivatedNodeAwareness(true)
+
.setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+ .build();
Assert.assertTrue(strictMatchVerifier.verify(10000));
// Re-start a new participant with sleeping transition(all state model
transition cannot finish)
@@ -255,7 +261,6 @@ public class TestClusterVerifier extends ZkUnitTestBase {
_participants[0].syncStart();
// The new participant causes rebalance, but the state transitions are all
stuck
- Thread.sleep(1000);
Assert.assertFalse(strictMatchVerifier.verify(3000));
}
}
