This is an automated email from the ASF dual-hosted git repository.
somandal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5c2dfea9ba Add new parameter in Table Rebalance API: Disk Utilization
Pre-check Threshold Override (#16144)
5c2dfea9ba is described below
commit 5c2dfea9ba28fc41b32f61be961700b8e149be39
Author: Jhow <[email protected]>
AuthorDate: Wed Jun 25 14:54:24 2025 -0700
Add new parameter in Table Rebalance API: Disk Utilization Pre-check
Threshold Override (#16144)
---
.../api/resources/PinotTableRestletResource.java | 6 ++
.../core/rebalance/DefaultRebalancePreChecker.java | 20 +++++--
.../helix/core/rebalance/RebalanceConfig.java | 23 +++++++-
.../RebalanceServerConfigurationOption.tsx | 5 ++
.../RebalanceServerConfigurationOptionDouble.tsx | 66 ++++++++++++++++++++++
.../RebalanceServer/RebalanceServerOptions.ts | 18 +++++-
6 files changed, 128 insertions(+), 10 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index e991674efc..bd97c37d1e 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -673,6 +673,11 @@ public class PinotTableRestletResource {
@QueryParam("retryInitialDelayInMs") long retryInitialDelayInMs,
@ApiParam(value = "Whether to update segment target tier as part of the
rebalance") @DefaultValue("false")
@QueryParam("updateTargetTier") boolean updateTargetTier,
+ @ApiParam(value = "Disk utilization threshold override used in pre-check
(0.0 to 1.0, e.g., 0.85 for 85%). "
+ + "If not provided, uses " +
ControllerConf.REBALANCE_DISK_UTILIZATION_THRESHOLD
+ + " in the controller config")
+ @DefaultValue("-1.0")
+ @QueryParam("diskUtilizationThreshold") double diskUtilizationThreshold,
@ApiParam(value = "Whether to force commit consuming segments for a
REALTIME table before they are rebalanced.")
@DefaultValue("false")
@QueryParam("forceCommit") boolean forceCommit,
@@ -715,6 +720,7 @@ public class PinotTableRestletResource {
rebalanceConfig.setMaxAttempts(maxAttempts);
rebalanceConfig.setRetryInitialDelayInMs(retryInitialDelayInMs);
rebalanceConfig.setUpdateTargetTier(updateTargetTier);
+ rebalanceConfig.setDiskUtilizationThreshold(diskUtilizationThreshold);
String rebalanceJobId =
TableRebalancer.createUniqueRebalanceJobIdentifier();
try {
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
index 37a2f29970..8bc9e8e730 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
@@ -62,7 +62,7 @@ public class DefaultRebalancePreChecker implements
RebalancePreChecker {
public static final int SEGMENT_ADD_THRESHOLD = 200;
public static final int RECOMMENDED_BATCH_SIZE = 200;
- private static double _diskUtilizationThreshold;
+ private static double _defaultDiskUtilizationThreshold;
protected PinotHelixResourceManager _pinotHelixResourceManager;
protected ExecutorService _executorService;
@@ -72,7 +72,7 @@ public class DefaultRebalancePreChecker implements
RebalancePreChecker {
double diskUtilizationThreshold) {
_pinotHelixResourceManager = pinotHelixResourceManager;
_executorService = executorService;
- _diskUtilizationThreshold = diskUtilizationThreshold;
+ _defaultDiskUtilizationThreshold = diskUtilizationThreshold;
}
@Override
@@ -92,17 +92,25 @@ public class DefaultRebalancePreChecker implements
RebalancePreChecker {
// Check whether minimizeDataMovement is set in TableConfig
preCheckResult.put(IS_MINIMIZE_DATA_MOVEMENT,
checkIsMinimizeDataMovement(tableConfig, rebalanceConfig,
tableRebalanceLogger));
+ // Determine the disk utilization threshold to use - either from rebalance
config override or default
+ double diskUtilizationThreshold =
rebalanceConfig.getDiskUtilizationThreshold() >= 0.0
+ ? rebalanceConfig.getDiskUtilizationThreshold() :
_defaultDiskUtilizationThreshold;
+ // clip the disk utilization threshold to [0.0, 1.0]
+ if (diskUtilizationThreshold > 1.0) {
+ tableRebalanceLogger.warn("Provided disk utilization threshold {} is
greater than 1.0, clipping to 1.0",
+ diskUtilizationThreshold);
+ diskUtilizationThreshold = 1.0;
+ }
+
// Check if all servers involved in the rebalance have enough disk space
for rebalance operation.
// Notice this check could have false positives (disk utilization is
subject to change by other operations anytime)
preCheckResult.put(DISK_UTILIZATION_DURING_REBALANCE,
checkDiskUtilization(preCheckContext.getCurrentAssignment(),
preCheckContext.getTargetAssignment(),
- preCheckContext.getTableSubTypeSizeDetails(),
_diskUtilizationThreshold, true));
+ preCheckContext.getTableSubTypeSizeDetails(),
diskUtilizationThreshold, true));
// Check if all servers involved in the rebalance will have enough disk
space after the rebalance.
- // TODO: Add the option to take disk utilization threshold as a
RebalanceConfig option and use that to override
- // the default config
preCheckResult.put(DISK_UTILIZATION_AFTER_REBALANCE,
checkDiskUtilization(preCheckContext.getCurrentAssignment(),
preCheckContext.getTargetAssignment(),
- preCheckContext.getTableSubTypeSizeDetails(),
_diskUtilizationThreshold, false));
+ preCheckContext.getTableSubTypeSizeDetails(),
diskUtilizationThreshold, false));
preCheckResult.put(REBALANCE_CONFIG_OPTIONS,
checkRebalanceConfig(rebalanceConfig, tableConfig,
preCheckContext.getCurrentAssignment(),
preCheckContext.getTargetAssignment(),
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
index 5d97eb18a1..38f1fff33d 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/RebalanceConfig.java
@@ -137,6 +137,14 @@ public class RebalanceConfig {
@ApiModelProperty(example = "300000")
private long _retryInitialDelayInMs = 300000L;
+ // Disk utilization threshold override. If set, this will override the
default disk utilization threshold
+ // configured at the controller level. Value should be between 0.0 and 1.0
(e.g., 0.85 for 85%) or -1.0, which means
+ // no override. In the latter case the pre-checker will use the default disk
utilization threshold from the controller
+ // config.
+ @JsonProperty("diskUtilizationThreshold")
+ @ApiModelProperty(example = "0.85")
+ private double _diskUtilizationThreshold = -1.0;
+
@JsonProperty("forceCommit")
@ApiModelProperty(example = "false")
private boolean _forceCommit = false;
@@ -331,6 +339,14 @@ public class RebalanceConfig {
_minimizeDataMovement = minimizeDataMovement;
}
+ public double getDiskUtilizationThreshold() {
+ return _diskUtilizationThreshold;
+ }
+
+ public void setDiskUtilizationThreshold(double diskUtilizationThreshold) {
+ _diskUtilizationThreshold = diskUtilizationThreshold;
+ }
+
@Override
public String toString() {
return "RebalanceConfig{" + "_dryRun=" + _dryRun + ", preChecks=" +
_preChecks + ", _reassignInstances="
@@ -341,9 +357,9 @@ public class RebalanceConfig {
+ ", _externalViewStabilizationTimeoutInMs=" +
_externalViewStabilizationTimeoutInMs
+ ", _updateTargetTier=" + _updateTargetTier + ",
_heartbeatIntervalInMs=" + _heartbeatIntervalInMs
+ ", _heartbeatTimeoutInMs=" + _heartbeatTimeoutInMs + ",
_maxAttempts=" + _maxAttempts
- + ", _retryInitialDelayInMs=" + _retryInitialDelayInMs
- + ", _forceCommit=" + _forceCommit + ", _forceCommitBatchSize=" +
_forceCommitBatchSize
- + ", _forceCommitBatchStatusCheckIntervalMs=" +
_forceCommitBatchStatusCheckIntervalMs
+ + ", _retryInitialDelayInMs=" + _retryInitialDelayInMs + ",
_diskUtilizationThreshold="
+ + _diskUtilizationThreshold + ", _forceCommit=" + _forceCommit + ",
_forceCommitBatchSize="
+ + _forceCommitBatchSize + ", _forceCommitBatchStatusCheckIntervalMs="
+ _forceCommitBatchStatusCheckIntervalMs
+ ", _forceCommitBatchStatusCheckTimeoutMs=" +
_forceCommitBatchStatusCheckTimeoutMs + '}';
}
@@ -382,6 +398,7 @@ public class RebalanceConfig {
rc._heartbeatTimeoutInMs = cfg._heartbeatTimeoutInMs;
rc._maxAttempts = cfg._maxAttempts;
rc._retryInitialDelayInMs = cfg._retryInitialDelayInMs;
+ rc._diskUtilizationThreshold = cfg._diskUtilizationThreshold;
rc._forceCommit = cfg._forceCommit;
rc._forceCommitBatchSize = cfg._forceCommitBatchSize;
rc._forceCommitBatchStatusCheckIntervalMs =
cfg._forceCommitBatchStatusCheckIntervalMs;
diff --git
a/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerConfigurationOption.tsx
b/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerConfigurationOption.tsx
index 7a20f8e9ac..a5749f5b44 100644
---
a/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerConfigurationOption.tsx
+++
b/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerConfigurationOption.tsx
@@ -27,6 +27,9 @@ import {
import {
RebalanceServerConfigurationOptionSelect
} from
"./RebalanceServerConfigurationOptions/RebalanceServerConfigurationOptionSelect";
+import {
+ RebalanceServerConfigurationOptionDouble
+} from
"./RebalanceServerConfigurationOptions/RebalanceServerConfigurationOptionDouble";
export const RebalanceServerConfigurationOption = (
{
@@ -45,6 +48,8 @@ export const RebalanceServerConfigurationOption = (
return <RebalanceServerConfigurationOptionInteger
rebalanceConfig={rebalanceConfig} option={option}
handleConfigChange={handleConfigChange} />;
case "SELECT":
return <RebalanceServerConfigurationOptionSelect
rebalanceConfig={rebalanceConfig} option={option}
handleConfigChange={handleConfigChange} />;
+ case "DOUBLE":
+ return <RebalanceServerConfigurationOptionDouble
rebalanceConfig={rebalanceConfig} option={option}
handleConfigChange={handleConfigChange} />;
default:
return null;
}
diff --git
a/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerConfigurationOptions/RebalanceServerConfigurationOptionDouble.tsx
b/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerConfigurationOptions/RebalanceServerConfigurationOptionDouble.tsx
new file mode 100644
index 0000000000..cd64710dea
--- /dev/null
+++
b/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerConfigurationOptions/RebalanceServerConfigurationOptionDouble.tsx
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import {Box, FormControl, TextField, Typography} from "@material-ui/core";
+import React, {useState} from "react";
+import {RebalanceServerOption} from "../RebalanceServerOptions";
+import {
+ RebalanceServerConfigurationOptionLabel
+} from
"./RebalanceServerConfigurationOptionLabel/RebalanceServerConfigurationOptionLabel";
+import Utils from "../../../../../utils/Utils";
+
+type RebalanceServerConfigurationOptionDoubleProps = {
+ option: RebalanceServerOption;
+ handleConfigChange: (config: { [key: string]: string | number | boolean })
=> void;
+ rebalanceConfig: { [optionName: string]: string | boolean | number };
+}
+export const RebalanceServerConfigurationOptionDouble = (
+ { option, handleConfigChange, rebalanceConfig }:
RebalanceServerConfigurationOptionDoubleProps
+) => {
+ const [value, setValue] = useState<number>(
+ Utils.getRebalanceConfigValue(rebalanceConfig, option) as number
+ );
+ return (
+ <Box display='flex' flexDirection='column'>
+ <FormControl fullWidth>
+ <RebalanceServerConfigurationOptionLabel option={option} />
+ <TextField
+ variant='outlined'
+ fullWidth
+ style={{ width: '100%' }}
+ size='small'
+ id={`rebalance-server-double-input-${option.name}`}
+ type='number'
+ inputProps={{
+ step: option.valueStep,
+ min: option.valueMin,
+ max: option.valueMax
+ }}
+ value={value}
+ onChange={(e) => {
+ handleConfigChange(
+ {
+ [option.name]: parseFloat(e.target.value)
+ });
+ setValue(parseFloat(e.target.value));
+ }}/>
+ <Typography variant='caption'>{option.description}</Typography>
+ </FormControl>
+ </Box>
+ );
+}
\ No newline at end of file
diff --git
a/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerOptions.ts
b/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerOptions.ts
index 612d338dc5..04b1636746 100644
---
a/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerOptions.ts
+++
b/pinot-controller/src/main/resources/app/components/Homepage/Operations/RebalanceServer/RebalanceServerOptions.ts
@@ -19,7 +19,7 @@
export type RebalanceServerOption = {
name: string;
label: string;
- type: "BOOL" | "INTEGER" | "SELECT";
+ type: "BOOL" | "INTEGER" | "SELECT" | "DOUBLE";
description: string;
defaultValue: string | boolean | number;
isAdvancedConfig: boolean;
@@ -27,6 +27,9 @@ export type RebalanceServerOption = {
markWithWarningIcon: boolean;
allowedValues?: string[];
toolTip?: string;
+ valueStep?: number;
+ valueMin?: number;
+ valueMax?: number;
}
export const rebalanceServerOptions: RebalanceServerOption[] = [
@@ -175,6 +178,19 @@ export const rebalanceServerOptions:
RebalanceServerOption[] = [
"isStatsGatheringConfig": false,
"markWithWarningIcon": false
},
+ {
+ "name": "diskUtilizationThreshold",
+ "defaultValue": -1.0,
+ "type": "DOUBLE",
+ "label": "Disk Utilization Threshold",
+ "description": "Override disk utilization threshold used in pre-check
(0.0 to 1.0, e.g., 0.85 for 85%). If not provided (or any negative value), uses
the controller's default threshold",
+ "isAdvancedConfig": true,
+ "isStatsGatheringConfig": false,
+ "markWithWarningIcon": false,
+ "valueStep": 0.05,
+ "valueMin": -1.0,
+ "valueMax": 1.0
+ },
{
"name": "forceCommit",
"defaultValue": false,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
