This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 28aa58eaa1 Remove summary option from TableRebalancer and return
summary by default (#15212)
28aa58eaa1 is described below
commit 28aa58eaa1ea80a7c1c31cd34207d7827e4af2dd
Author: Sonam Mandal <[email protected]>
AuthorDate: Thu Mar 6 13:10:24 2025 -0800
Remove summary option from TableRebalancer and return summary by default
(#15212)
---
.../api/resources/PinotTableRestletResource.java | 6 +-
.../helix/core/rebalance/RebalanceConfig.java | 16 +---
.../helix/core/rebalance/TableRebalancer.java | 42 +++-------
.../TableRebalancerClusterStatelessTest.java | 96 ++++++++--------------
.../tests/OfflineClusterIntegrationTest.java | 31 +++----
5 files changed, 58 insertions(+), 133 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index 8815d20d06..ba088c267d 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -606,9 +606,6 @@ public class PinotTableRestletResource {
@ApiParam(value = "OFFLINE|REALTIME", required = true)
@QueryParam("type") String tableTypeStr,
@ApiParam(value = "Whether to rebalance table in dry-run mode")
@DefaultValue("false") @QueryParam("dryRun")
boolean dryRun,
- @ApiParam(value = "Whether to return dry-run summary instead of full,
dry-run must be enabled to use this")
- @DefaultValue("false") @QueryParam("summary")
- boolean summary,
@ApiParam(value = "Whether to enable pre-checks for table, must be in
dry-run mode to enable")
@DefaultValue("false") @QueryParam("preChecks") boolean preChecks,
@ApiParam(value = "Whether to reassign instances before reassigning
segments") @DefaultValue("false")
@@ -651,7 +648,6 @@ public class PinotTableRestletResource {
String tableNameWithType = constructTableNameWithType(tableName,
tableTypeStr);
RebalanceConfig rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(dryRun);
- rebalanceConfig.setSummary(summary);
rebalanceConfig.setPreChecks(preChecks);
rebalanceConfig.setReassignInstances(reassignInstances);
rebalanceConfig.setIncludeConsuming(includeConsuming);
@@ -672,7 +668,7 @@ public class PinotTableRestletResource {
String rebalanceJobId =
TableRebalancer.createUniqueRebalanceJobIdentifier();
try {
- if (dryRun || summary || preChecks || downtime) {
+ if (dryRun || preChecks || downtime) {
// For dry-run, preChecks or rebalance with downtime, directly return
the rebalance result as it should return
// immediately
return _pinotHelixResourceManager.rebalanceTable(tableNameWithType,
rebalanceConfig, rebalanceJobId, false);
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
index 06a9e171c3..f69f6be3d9 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
@@ -34,11 +34,6 @@ public class RebalanceConfig {
@ApiModelProperty(example = "false")
private boolean _dryRun = false;
- // Whether to return only dry-run summary instead of full dry-run output,
can only be used in dry-run mode
- @JsonProperty("summary")
- @ApiModelProperty(example = "false")
- private boolean _summary = false;
-
// Whether to perform pre-checks for rebalance. This only returns the status
of each pre-check and does not fail
// rebalance
@JsonProperty("preChecks")
@@ -129,14 +124,6 @@ public class RebalanceConfig {
_dryRun = dryRun;
}
- public boolean isSummary() {
- return _summary;
- }
-
- public void setSummary(boolean summary) {
- _summary = summary;
- }
-
public boolean isPreChecks() {
return _preChecks;
}
@@ -259,7 +246,7 @@ public class RebalanceConfig {
@Override
public String toString() {
- return "RebalanceConfig{" + "_dryRun=" + _dryRun + ", _summary=" +
_summary + ", preChecks=" + _preChecks
+ return "RebalanceConfig{" + "_dryRun=" + _dryRun + ", preChecks=" +
_preChecks
+ ", _reassignInstances=" + _reassignInstances + ",
_includeConsuming=" + _includeConsuming + ", _bootstrap="
+ _bootstrap + ", _downtime=" + _downtime + ", _minAvailableReplicas="
+ _minAvailableReplicas
+ ", _bestEfforts=" + _bestEfforts + ",
_externalViewCheckIntervalInMs=" + _externalViewCheckIntervalInMs
@@ -272,7 +259,6 @@ public class RebalanceConfig {
public static RebalanceConfig copy(RebalanceConfig cfg) {
RebalanceConfig rc = new RebalanceConfig();
rc._dryRun = cfg._dryRun;
- rc._summary = cfg._summary;
rc._preChecks = cfg._preChecks;
rc._reassignInstances = cfg._reassignInstances;
rc._includeConsuming = cfg._includeConsuming;
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancer.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancer.java
index b5d56f2ba6..34cab7c29d 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancer.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancer.java
@@ -181,7 +181,6 @@ public class TableRebalancer {
rebalanceJobId = createUniqueRebalanceJobIdentifier();
}
boolean dryRun = rebalanceConfig.isDryRun();
- boolean summary = rebalanceConfig.isSummary();
boolean preChecks = rebalanceConfig.isPreChecks();
boolean reassignInstances = rebalanceConfig.isReassignInstances();
boolean includeConsuming = rebalanceConfig.isIncludeConsuming();
@@ -196,19 +195,14 @@ public class TableRebalancer {
&&
RoutingConfig.STRICT_REPLICA_GROUP_INSTANCE_SELECTOR_TYPE.equalsIgnoreCase(
tableConfig.getRoutingConfig().getInstanceSelectorType());
LOGGER.info(
- "Start rebalancing table: {} with dryRun: {}, summary: {}, preChecks:
{}, reassignInstances: {}, "
- + "includeConsuming: {}, bootstrap: {}, downtime: {},
minReplicasToKeepUpForNoDowntime: {}, "
- + "enableStrictReplicaGroup: {}, lowDiskMode: {}, bestEfforts: {},
externalViewCheckIntervalInMs: {}, "
+ "Start rebalancing table: {} with dryRun: {}, preChecks: {},
reassignInstances: {}, includeConsuming: {},"
+ + "bootstrap: {}, downtime: {}, minReplicasToKeepUpForNoDowntime:
{}, enableStrictReplicaGroup: {},"
+ + "lowDiskMode: {}, bestEfforts: {},
externalViewCheckIntervalInMs: {}, "
+ "externalViewStabilizationTimeoutInMs: {}",
- tableNameWithType, dryRun, summary, preChecks, reassignInstances,
includeConsuming, bootstrap, downtime,
+ tableNameWithType, dryRun, preChecks, reassignInstances,
includeConsuming, bootstrap, downtime,
minReplicasToKeepUpForNoDowntime, enableStrictReplicaGroup,
lowDiskMode, bestEfforts,
externalViewCheckIntervalInMs, externalViewStabilizationTimeoutInMs);
- if (summary && !dryRun) {
- return new RebalanceResult(rebalanceJobId, RebalanceResult.Status.FAILED,
- "Must enable dry-run mode to use summary mode", null, null, null,
null, null);
- }
-
// Perform pre-checks if enabled
Map<String, String> preChecksResult = null;
if (preChecks) {
@@ -313,12 +307,10 @@ public class TableRebalancer {
+ "segmentAssignmentUnchanged: {} for table: {}", rebalanceJobId,
instancePartitionsUnchanged,
tierInstancePartitionsUnchanged, segmentAssignmentUnchanged,
tableNameWithType);
- RebalanceSummaryResult summaryResult = null;
- if (summary) {
- // Calculate summary here itself so that even if the table is already
balanced, the caller can verify whether that
- // is expected or not based on the summary results
- summaryResult = calculateDryRunSummary(currentAssignment,
targetAssignment, tableNameWithType, rebalanceJobId);
- }
+ // Calculate summary here itself so that even if the table is already
balanced, the caller can verify whether that
+ // is expected or not based on the summary results
+ RebalanceSummaryResult summaryResult =
+ calculateDryRunSummary(currentAssignment, targetAssignment,
tableNameWithType, rebalanceJobId);
if (segmentAssignmentUnchanged) {
LOGGER.info("Table: {} is already balanced", tableNameWithType);
@@ -327,33 +319,23 @@ public class TableRebalancer {
String.format("For rebalanceId: %s, instance unchanged and table:
%s is already balanced", rebalanceJobId,
tableNameWithType));
return new RebalanceResult(rebalanceJobId,
RebalanceResult.Status.NO_OP, "Table is already balanced",
- summary ? null : instancePartitionsMap, summary ? null :
tierToInstancePartitionsMap,
- summary ? null : targetAssignment, preChecksResult, summaryResult);
+ instancePartitionsMap, tierToInstancePartitionsMap,
targetAssignment, preChecksResult, summaryResult);
} else {
if (dryRun) {
return new RebalanceResult(rebalanceJobId,
RebalanceResult.Status.DONE,
"Instance reassigned in dry-run mode, table is already balanced",
- summary ? null : instancePartitionsMap, summary ? null :
tierToInstancePartitionsMap,
- summary ? null : targetAssignment, preChecksResult,
summaryResult);
+ instancePartitionsMap, tierToInstancePartitionsMap,
targetAssignment, preChecksResult, summaryResult);
} else {
_tableRebalanceObserver.onSuccess(
String.format("For rebalanceId: %s, instance reassigned but
table: %s is already balanced",
rebalanceJobId, tableNameWithType));
return new RebalanceResult(rebalanceJobId,
RebalanceResult.Status.DONE,
- "Instance reassigned, table is already balanced", summary ? null
: instancePartitionsMap,
- summary ? null : tierToInstancePartitionsMap, summary ? null :
targetAssignment, preChecksResult,
- summaryResult);
+ "Instance reassigned, table is already balanced",
instancePartitionsMap,
+ tierToInstancePartitionsMap, targetAssignment, preChecksResult,
summaryResult);
}
}
}
- if (summary) {
- LOGGER.info("For rebalanceId: {}, rebalancing table: {} in dry-run
summary mode, returning the summary only",
- rebalanceJobId, tableNameWithType);
- return new RebalanceResult(rebalanceJobId, RebalanceResult.Status.DONE,
"Dry-run summary mode", null,
- null, null, preChecksResult, summaryResult);
- }
-
if (dryRun) {
LOGGER.info("For rebalanceId: {}, rebalancing table: {} in dry-run mode,
returning the target assignment",
rebalanceJobId, tableNameWithType);
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
index 2f7ab350bc..5b99e56cc3 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
@@ -108,7 +108,6 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Rebalance with dry-run summary should fail without creating the table
RebalanceConfig rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.FAILED);
assertNull(rebalanceResult.getRebalanceSummaryResult());
@@ -129,7 +128,6 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Rebalance with dry-run summary should return NO_OP status
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.NO_OP);
RebalanceSummaryResult rebalanceSummaryResult =
rebalanceResult.getRebalanceSummaryResult();
@@ -137,11 +135,10 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
0);
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
3);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// Dry-run mode should not change the IdealState
assertEquals(_helixResourceManager.getTableIdealState(OFFLINE_TABLE_NAME).getRecord().getMapFields(),
@@ -173,7 +170,6 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Rebalance in dry-run summary mode with added servers
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
@@ -181,12 +177,11 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
14);
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
3);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
Map<String, RebalanceSummaryResult.ServerSegmentChangeInfo>
serverSegmentChangeInfoMap =
rebalanceSummaryResult.getServerInfo().getServerSegmentChangeInfo();
@@ -269,7 +264,6 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// occur
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceConfig.setPreChecks(true);
rebalanceConfig.setMinAvailableReplicas(3);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
@@ -278,12 +272,11 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
6);
assertNotNull(rebalanceResult.getPreChecksResult());
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// Rebalance with 3 min available replicas should fail as the table only
have 3 replicas
rebalanceConfig = new RebalanceConfig();
@@ -321,7 +314,6 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// No need to reassign instances because instances should be automatically
assigned when updating the table config
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
@@ -329,11 +321,10 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
11);
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
6);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
serverSegmentChangeInfoMap =
rebalanceSummaryResult.getServerInfo().getServerSegmentChangeInfo();
assertNotNull(serverSegmentChangeInfoMap);
@@ -402,20 +393,18 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// no movement should occur
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.NO_OP);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
0);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
0);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// Without instances reassignment, the rebalance should return status
NO_OP, and the existing instance partitions
// should be used
@@ -427,7 +416,6 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode with reassignment
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceConfig.setReassignInstances(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
@@ -435,14 +423,13 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
// No move expected since already balanced
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
0);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
0);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// With instances reassignment, the rebalance should return status DONE,
the existing instance partitions should be
// removed, and the default instance partitions should be used
@@ -476,7 +463,6 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceConfig.setDowntime(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
@@ -484,13 +470,12 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
15);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
3);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// Rebalance with downtime should succeed
rebalanceConfig = new RebalanceConfig();
@@ -519,15 +504,13 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
}
}
- // Try summary mode without dry-run set
+ // Try pre-checks mode without dry-run set
rebalanceConfig = new RebalanceConfig();
- rebalanceConfig.setSummary(true);
+ rebalanceConfig.setPreChecks(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.FAILED);
assertNull(rebalanceResult.getRebalanceSummaryResult());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
+ assertNull(rebalanceResult.getPreChecksResult());
_helixResourceManager.deleteOfflineTable(RAW_TABLE_NAME);
executorService.shutdown();
@@ -574,20 +557,18 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode
RebalanceConfig rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
RebalanceResult rebalanceResult = tableRebalancer.rebalance(tableConfig,
rebalanceConfig, null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.NO_OP);
RebalanceSummaryResult rebalanceSummaryResult =
rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
0);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
0);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// Run actual table rebalance
rebalanceResult = tableRebalancer.rebalance(tableConfig, new
RebalanceConfig(), null);
@@ -608,20 +589,18 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode, should be no-op
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.NO_OP);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
0);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
0);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// rebalance is NOOP and no change in assignment caused by new instances
rebalanceResult = tableRebalancer.rebalance(tableConfig, new
RebalanceConfig(), null);
@@ -644,20 +623,19 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode, some moves should occur
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
15);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
9);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
6);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getTierInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// rebalance should change assignment
rebalanceResult = tableRebalancer.rebalance(tableConfig, new
RebalanceConfig(), null);
@@ -721,20 +699,18 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode
RebalanceConfig rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
RebalanceResult rebalanceResult = tableRebalancer.rebalance(tableConfig,
rebalanceConfig, null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.NO_OP);
RebalanceSummaryResult rebalanceSummaryResult =
rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
0);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
0);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
rebalanceResult = tableRebalancer.rebalance(tableConfig, new
RebalanceConfig(), null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.NO_OP);
@@ -751,20 +727,18 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.NO_OP);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
0);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
0);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// rebalance is NOOP and no change in assignment caused by new instances
rebalanceResult = tableRebalancer.rebalance(tableConfig, new
RebalanceConfig(), null);
@@ -781,20 +755,19 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
30);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
3);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServersGettingNewSegments(),
6);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getTierInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
// rebalance should change assignment
rebalanceResult = tableRebalancer.rebalance(tableConfig, new
RebalanceConfig(), null);
@@ -822,19 +795,18 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
// Try dry-run summary mode
rebalanceConfig = new RebalanceConfig();
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
rebalanceSummaryResult = rebalanceResult.getRebalanceSummaryResult();
assertNotNull(rebalanceSummaryResult);
assertNotNull(rebalanceSummaryResult.getServerInfo());
assertNotNull(rebalanceSummaryResult.getSegmentInfo());
- assertNull(rebalanceResult.getInstanceAssignment());
- assertNull(rebalanceResult.getTierInstanceAssignment());
- assertNull(rebalanceResult.getSegmentAssignment());
assertEquals(rebalanceSummaryResult.getSegmentInfo().getTotalSegmentsToBeMoved(),
13);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getValueBeforeRebalance(),
6);
assertEquals(rebalanceSummaryResult.getServerInfo().getNumServers().getExpectedValueAfterRebalance(),
6);
+ assertNotNull(rebalanceResult.getInstanceAssignment());
+ assertNotNull(rebalanceResult.getTierInstanceAssignment());
+ assertNotNull(rebalanceResult.getSegmentAssignment());
rebalanceResult = tableRebalancer.rebalance(tableConfig, new
RebalanceConfig(), null);
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index 7e2e0f9ea1..cfd7d8220f 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -4056,13 +4056,9 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
TableConfig tableConfig = getOfflineTableConfig();
- // Ensure summary status is null if not enabled
+ // Ensure summary status is non-null always
RebalanceResult rebalanceResult = _tableRebalancer.rebalance(tableConfig,
rebalanceConfig, null);
- assertNull(rebalanceResult.getRebalanceSummaryResult());
-
- // Enable summary, nothing is set
- rebalanceConfig.setSummary(true);
- rebalanceResult = _tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
+ assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalanceDryRunSummary(rebalanceResult, RebalanceResult.Status.NO_OP,
false, getNumServers(), getNumServers(),
tableConfig.getReplication());
@@ -4073,19 +4069,15 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
checkRebalanceDryRunSummary(rebalanceResult, RebalanceResult.Status.DONE,
true, getNumServers(),
getNumServers() + 1, tableConfig.getReplication());
- // Disable dry-run, summary still enabled
+ // Disable dry-run to do a real rebalance
rebalanceConfig.setDryRun(false);
- rebalanceResult = _tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
- assertNull(rebalanceResult.getRebalanceSummaryResult());
- assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.FAILED);
-
- // Disable summary along with dry-run to do a real rebalance
- rebalanceConfig.setSummary(false);
rebalanceConfig.setDowntime(true);
rebalanceConfig.setReassignInstances(true);
rebalanceResult = _tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
- assertNull(rebalanceResult.getRebalanceSummaryResult());
+ assertNotNull(rebalanceResult.getRebalanceSummaryResult());
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
+ checkRebalanceDryRunSummary(rebalanceResult, RebalanceResult.Status.DONE,
true, getNumServers(),
+ getNumServers() + 1, tableConfig.getReplication());
waitForRebalanceToComplete(rebalanceResult, 600_000L);
@@ -4094,18 +4086,16 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
// Re-enable dry-run
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = _tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
checkRebalanceDryRunSummary(rebalanceResult, RebalanceResult.Status.DONE,
true, getNumServers() + 1,
getNumServers(), tableConfig.getReplication());
- // Disable dry-run and summary to do a real rebalance
+ // Disable dry-run to do a real rebalance
rebalanceConfig.setDryRun(false);
- rebalanceConfig.setSummary(false);
rebalanceConfig.setDowntime(true);
rebalanceConfig.setReassignInstances(true);
rebalanceResult = _tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
- assertNull(rebalanceResult.getRebalanceSummaryResult());
+ assertNotNull(rebalanceResult.getRebalanceSummaryResult());
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
waitForRebalanceToComplete(rebalanceResult, 600_000L);
@@ -4115,14 +4105,13 @@ public class OfflineClusterIntegrationTest extends
BaseClusterIntegrationTestSet
TestUtils.waitForCondition(aVoid ->
_resourceManager.dropInstance(serverStarter1.getInstanceId()).isSuccessful(),
60_000L, "Failed to drop added server");
- // Try dry-run with summary again
+ // Try dry-run again
rebalanceConfig.setDryRun(true);
- rebalanceConfig.setSummary(true);
rebalanceResult = _tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
checkRebalanceDryRunSummary(rebalanceResult, RebalanceResult.Status.NO_OP,
false, getNumServers(), getNumServers(),
tableConfig.getReplication());
- // Try dry-run with summary and pre-checks
+ // Try dry-run with pre-checks
rebalanceConfig.setPreChecks(true);
rebalanceResult = _tableRebalancer.rebalance(tableConfig, rebalanceConfig,
null);
checkRebalanceDryRunSummary(rebalanceResult, RebalanceResult.Status.NO_OP,
false, getNumServers(), getNumServers(),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
