This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new 524ce38 Improve WAGED simulation API by throwing exception on
FAILED_TO_CALCULATE (#1701)
524ce38 is described below
commit 524ce38b910c599a3ca2e08f3581d20ecc0a12b0
Author: Neal Sun <[email protected]>
AuthorDate: Thu May 6 14:07:24 2021 -0700
Improve WAGED simulation API by throwing exception on FAILED_TO_CALCULATE
(#1701)
WAGED simulation API now throws HelixException upon encountering
FAILED_TO_CALCULATE.
Co-authored-by: Neal Sun <[email protected]>
---
.../rebalancer/waged/ReadOnlyWagedRebalancer.java | 12 +++++++++++
.../rebalancer/waged/WagedRebalancer.java | 15 +++++++++++---
.../main/java/org/apache/helix/util/HelixUtil.java | 3 ++-
.../WagedRebalancer/TestWagedRebalance.java | 24 ++++++++++++++++++++++
4 files changed, 50 insertions(+), 4 deletions(-)
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/ReadOnlyWagedRebalancer.java
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/ReadOnlyWagedRebalancer.java
index 6808389..44a3c1c 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/ReadOnlyWagedRebalancer.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/ReadOnlyWagedRebalancer.java
@@ -19,9 +19,12 @@ package org.apache.helix.controller.rebalancer.waged;
* under the License.
*/
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.helix.HelixRebalanceException;
import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
@@ -48,6 +51,15 @@ public class ReadOnlyWagedRebalancer extends WagedRebalancer
{
ConstraintBasedAlgorithmFactory.getInstance(preferences),
Optional.empty());
}
+ @Override
+ protected List<HelixRebalanceException.Type> failureTypesToPropagate() {
+ // Also propagate FAILED_TO_CALCULATE for ReadOnlyWagedRebalancer
+ List<HelixRebalanceException.Type> failureTypes =
+ new ArrayList<>(super.failureTypesToPropagate());
+ failureTypes.add(HelixRebalanceException.Type.FAILED_TO_CALCULATE);
+ return failureTypes;
+ }
+
private static class ReadOnlyAssignmentMetadataStore extends
AssignmentMetadataStore {
ReadOnlyAssignmentMetadataStore(String metadataStoreAddress, String
clusterName) {
super(new ZkBucketDataAccessor(metadataStoreAddress), clusterName);
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java
index 4451c1d..44df28c 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java
@@ -20,6 +20,7 @@ package org.apache.helix.controller.rebalancer.waged;
*/
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -90,6 +91,10 @@ public class WagedRebalancer implements
StatefulRebalancer<ResourceControllerDat
private static final RebalanceAlgorithm DEFAULT_REBALANCE_ALGORITHM =
ConstraintBasedAlgorithmFactory
.getInstance(ClusterConfig.DEFAULT_GLOBAL_REBALANCE_PREFERENCE);
+ // These failure types should be propagated to caller of
computeNewIdealStates()
+ private static final List<HelixRebalanceException.Type>
FAILURE_TYPES_TO_PROPAGATE = Collections
+
.unmodifiableList(Arrays.asList(HelixRebalanceException.Type.INVALID_REBALANCER_STATUS,
+ HelixRebalanceException.Type.UNKNOWN_FAILURE));
// To calculate the baseline asynchronously
private final ExecutorService _baselineCalculateExecutor;
@@ -268,12 +273,12 @@ public class WagedRebalancer implements
StatefulRebalancer<ResourceControllerDat
_rebalanceFailureCount.increment(1L);
HelixRebalanceException.Type failureType = ex.getFailureType();
- if
(failureType.equals(HelixRebalanceException.Type.INVALID_REBALANCER_STATUS) ||
failureType
- .equals(HelixRebalanceException.Type.UNKNOWN_FAILURE)) {
+ if (failureTypesToPropagate().contains(failureType)) {
// If the failure is unknown or because of assignment store access
failure, throw the
// rebalance exception.
throw ex;
- } else { // return the previously calculated assignment.
+ } else {
+ // return the previously calculated assignment.
LOG.warn(
"Returning the last known-good best possible assignment from
metadata store due to "
+ "rebalance failure of type: {}", failureType);
@@ -393,6 +398,10 @@ public class WagedRebalancer implements
StatefulRebalancer<ResourceControllerDat
return finalIdealStateMap;
}
+ protected List<HelixRebalanceException.Type> failureTypesToPropagate() {
+ return FAILURE_TYPES_TO_PROPAGATE;
+ }
+
/**
* Global rebalance calculates for a new baseline assignment.
* The new baseline assignment will be persisted and leveraged by the
partial rebalance.
diff --git a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
index 7fb5bc1..38103e9 100644
--- a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
@@ -296,7 +296,8 @@ public final class HelixUtil {
// Convert the resulting BestPossibleStateOutput to Map<String,
ResourceAssignment>
Map<String, ResourceAssignment> result = new HashMap<>();
BestPossibleStateOutput output =
event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
- if (output == null) {
+ if (output == null || (output.getPreferenceLists() == null &&
output.getResourceStatesMap()
+ .isEmpty())) {
throw new HelixException(
"getIdealAssignmentForWagedFullAuto(): Calculation failed: Failed to
compute BestPossibleState!");
}
diff --git
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
index bb58d86..a7250d4 100644
---
a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
+++
b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
@@ -31,6 +31,7 @@ import java.util.Set;
import com.google.common.collect.ImmutableMap;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
import org.apache.helix.TestHelper;
import org.apache.helix.common.ZkTestBase;
import
org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
@@ -275,6 +276,29 @@ public class TestWagedRebalance extends ZkTestBase {
// The newly added instances should contain some partitions
Assert.assertTrue(instancesWithAssignmentsImmediate.contains(instance_0));
Assert.assertTrue(instancesWithAssignmentsImmediate.contains(instance_1));
+
+ // Force FAILED_TO_CALCULATE and ensure that both util functions return
no mappings
+ String testCapacityKey = "key";
+
clusterConfig.setDefaultPartitionWeightMap(Collections.singletonMap(testCapacityKey,
2));
+
clusterConfig.setDefaultInstanceCapacityMap(Collections.singletonMap(testCapacityKey,
1));
+
clusterConfig.setInstanceCapacityKeys(Collections.singletonList(testCapacityKey));
+ try {
+ HelixUtil.getTargetAssignmentForWagedFullAuto(ZK_ADDR, clusterConfig,
instanceConfigs,
+ liveInstances, idealStates, resourceConfigs);
+ Assert.fail("Expected HelixException for calculaation failure");
+ } catch (HelixException e) {
+ Assert.assertEquals(e.getMessage(),
+ "getIdealAssignmentForWagedFullAuto(): Calculation failed: Failed
to compute BestPossibleState!");
+ }
+
+ try {
+ HelixUtil.getImmediateAssignmentForWagedFullAuto(ZK_ADDR,
clusterConfig, instanceConfigs,
+ liveInstances, idealStates, resourceConfigs);
+ Assert.fail("Expected HelixException for calculaation failure");
+ } catch (HelixException e) {
+ Assert.assertEquals(e.getMessage(),
+ "getIdealAssignmentForWagedFullAuto(): Calculation failed: Failed
to compute BestPossibleState!");
+ }
} finally {
// restore the config with async mode
clusterConfigGlobal =
