This is an automated email from the ASF dual-hosted git repository.
sarvekshayr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8546bb0b637 HDDS-14619. Add option in Container Balancer CLI for
excluding containers (#9785)
8546bb0b637 is described below
commit 8546bb0b63705d8b03a5046a97f7cff3a3438502
Author: sravani <[email protected]>
AuthorDate: Tue Feb 24 14:01:12 2026 +0530
HDDS-14619. Add option in Container Balancer CLI for excluding containers
(#9785)
---
.../apache/hadoop/hdds/scm/client/ScmClient.java | 3 ++-
.../protocol/StorageContainerLocationProtocol.java | 3 ++-
...inerLocationProtocolClientSideTranslatorPB.java | 8 +++++-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
...inerLocationProtocolServerSideTranslatorPB.java | 7 +++++-
.../hdds/scm/server/SCMClientProtocolServer.java | 9 ++++++-
.../scm/cli/ContainerBalancerStartSubcommand.java | 9 ++++++-
.../hdds/scm/cli/ContainerOperationClient.java | 5 ++--
.../datanode/TestContainerBalancerSubCommand.java | 4 +--
.../src/main/smoketest/balancer/testBalancer.robot | 29 +++++++++++++++++++++-
.../ozone/TestContainerBalancerOperations.java | 26 ++++++++++++++++---
11 files changed, 89 insertions(+), 15 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index 892dd4de1ff..5ef0515b92e 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -375,7 +375,8 @@ StartContainerBalancerResponseProto startContainerBalancer(
Optional<Integer> moveReplicationTimeout,
Optional<Boolean> networkTopologyEnable,
Optional<String> includeNodes,
- Optional<String> excludeNodes) throws IOException;
+ Optional<String> excludeNodes,
+ Optional<String> excludeContainers) throws IOException;
/**
* Stop ContainerBalancer.
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 92ddfa7eb8d..f0fb3378c6f 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -433,7 +433,8 @@ StartContainerBalancerResponseProto startContainerBalancer(
Optional<Integer> moveReplicationTimeout,
Optional<Boolean> networkTopologyEnable,
Optional<String> includeNodes,
- Optional<String> excludeNodes) throws IOException;
+ Optional<String> excludeNodes,
+ Optional<String> excludeContainers) throws IOException;
/**
* Stop ContainerBalancer.
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 94b2230e68b..678b40ec788 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -929,7 +929,8 @@ public StartContainerBalancerResponseProto
startContainerBalancer(
Optional<Integer> moveReplicationTimeout,
Optional<Boolean> networkTopologyEnable,
Optional<String> includeNodes,
- Optional<String> excludeNodes) throws IOException {
+ Optional<String> excludeNodes,
+ Optional<String> excludeContainers) throws IOException {
StartContainerBalancerRequestProto.Builder builder =
StartContainerBalancerRequestProto.newBuilder();
builder.setTraceID(TracingUtil.exportCurrentSpan());
@@ -1015,6 +1016,11 @@ public StartContainerBalancerResponseProto
startContainerBalancer(
builder.setExcludeNodes(ex);
}
+ if (excludeContainers.isPresent()) {
+ String ec = excludeContainers.get();
+ builder.setExcludeContainers(ec);
+ }
+
StartContainerBalancerRequestProto request = builder.build();
return submitRequest(Type.StartContainerBalancer,
builder1 -> builder1.setStartContainerBalancerRequest(request))
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index f80a50a3be9..d1aa4798bb6 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -603,6 +603,7 @@ message StartContainerBalancerRequestProto {
optional bool networkTopologyEnable = 13;
optional string includeNodes = 14;
optional string excludeNodes = 15;
+ optional string excludeContainers = 16;
}
message StartContainerBalancerResponseProto {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 62f06079bf0..6693f7be025 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -1141,6 +1141,7 @@ public StartContainerBalancerResponseProto
startContainerBalancer(
Optional<Boolean> networkTopologyEnable = Optional.empty();
Optional<String> includeNodes = Optional.empty();
Optional<String> excludeNodes = Optional.empty();
+ Optional<String> excludeContainers = Optional.empty();
if (request.hasThreshold()) {
threshold = Optional.of(request.getThreshold());
@@ -1201,12 +1202,16 @@ public StartContainerBalancerResponseProto
startContainerBalancer(
excludeNodes = Optional.of(request.getExcludeNodes());
}
+ if (request.hasExcludeContainers()) {
+ excludeContainers = Optional.of(request.getExcludeContainers());
+ }
+
return impl.startContainerBalancer(threshold, iterations,
maxDatanodesPercentageToInvolvePerIteration,
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB, balancingInterval, moveTimeout,
moveReplicationTimeout, networkTopologyEnable, includeNodes,
- excludeNodes);
+ excludeNodes, excludeContainers);
}
public StopContainerBalancerResponseProto stopContainerBalancer(
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 10749615274..80fbb062e90 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -1160,7 +1160,8 @@ public StartContainerBalancerResponseProto
startContainerBalancer(
Optional<Integer> moveReplicationTimeout,
Optional<Boolean> networkTopologyEnable,
Optional<String> includeNodes,
- Optional<String> excludeNodes) throws IOException {
+ Optional<String> excludeNodes,
+ Optional<String> excludeContainers) throws IOException {
Map<String, String> auditMap = Maps.newHashMap();
try {
getScm().checkAdminAccess(getRemoteUser(), false);
@@ -1270,6 +1271,12 @@ public StartContainerBalancerResponseProto
startContainerBalancer(
cbc.setExcludeNodes(ex);
}
+ if (excludeContainers.isPresent()) {
+ String ec = excludeContainers.get();
+ auditMap.put("excludeContainers", (ec));
+ cbc.setExcludeContainers(ec);
+ }
+
ContainerBalancer containerBalancer = scm.getContainerBalancer();
containerBalancer.startBalancer(cbc);
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
index 3db8de84811..2a9925c3c26 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java
@@ -119,6 +119,13 @@ public class ContainerBalancerStartSubcommand extends
ScmSubcommand {
"by default (specify \"hostname1,hostname2,hostname3\").")
private Optional<String> excludeNodes;
+ @Option(names = {"--exclude-containers"},
+ description = "A list of container IDs separated by commas. " +
+ "The containers specified in this list are excluded from balancing.
" +
+ "This configuration is empty by default " +
+ "(specify \"1,2,3\" for container IDs).")
+ private Optional<String> excludeContainers;
+
@Override
public void execute(ScmClient scmClient) throws IOException {
StartContainerBalancerResponseProto response = scmClient.
@@ -127,7 +134,7 @@ public void execute(ScmClient scmClient) throws IOException
{
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB, balancingInterval, moveTimeout,
moveReplicationTimeout, networkTopologyEnable, includeNodes,
- excludeNodes);
+ excludeNodes, excludeContainers);
if (response.getStart()) {
System.out.println("Container Balancer started successfully.");
} else {
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 61c0f4150c3..3ca49be2443 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -510,13 +510,14 @@ public StartContainerBalancerResponseProto
startContainerBalancer(
Optional<Integer> moveReplicationTimeout,
Optional<Boolean> networkTopologyEnable,
Optional<String> includeNodes,
- Optional<String> excludeNodes) throws IOException {
+ Optional<String> excludeNodes,
+ Optional<String> excludeContainers) throws IOException {
return storageContainerLocationClient.startContainerBalancer(threshold,
iterations, maxDatanodesPercentageToInvolvePerIteration,
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB, balancingInterval, moveTimeout,
moveReplicationTimeout, networkTopologyEnable, includeNodes,
- excludeNodes);
+ excludeNodes, excludeContainers);
}
@Override
diff --git
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
index 18d3cc74a7f..d71b8d00be2 100644
---
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
+++
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
@@ -449,7 +449,7 @@ public void
testContainerBalancerStartSubcommandWhenBalancerIsNotRunning()
throws IOException {
ScmClient scmClient = mock(ScmClient.class);
when(scmClient.startContainerBalancer(
- null, null, null, null, null, null, null, null, null, null, null,
null))
+ null, null, null, null, null, null, null, null, null, null, null,
null, null))
.thenReturn(
StorageContainerLocationProtocolProtos
.StartContainerBalancerResponseProto.newBuilder()
@@ -465,7 +465,7 @@ public void
testContainerBalancerStartSubcommandWhenBalancerIsRunning()
throws IOException {
ScmClient scmClient = mock(ScmClient.class);
when(scmClient.startContainerBalancer(
- null, null, null, null, null, null, null, null, null, null, null,
null))
+ null, null, null, null, null, null, null, null, null, null, null,
null, null))
.thenReturn(StorageContainerLocationProtocolProtos
.StartContainerBalancerResponseProto.newBuilder()
.setStart(false)
diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
index 0436348239b..7642719d9d3 100644
--- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
@@ -66,6 +66,11 @@ Run Container Balancer
${result} = Execute ozone admin
containerbalancer start -t 0.1 -d 100 -i 3
Should Contain ${result}
Container Balancer started successfully.
+Run Container Balancer With Exclude Containers
+ [arguments] ${exclude_containers}
+ ${result} = Execute ozone admin
containerbalancer start --exclude-containers "${exclude_containers}" -t 0.1 -d
100 -i 3
+ Should Contain ${result}
Container Balancer started successfully.
+
Wait Finish Of Balancing
${result} = Execute ozone admin
containerbalancer status
Wait Until Keyword Succeeds 4min 10sec
ContainerBalancer is Not Running
@@ -148,6 +153,11 @@ All container is closed
${output} = Execute ozone admin container list --state
OPEN
Should Be Equal ${output} [ ]
+Get All Container IDs
+ [Documentation] Fetches all container IDs from standard text output
using awk
+ ${result} = Execute ozone admin container list | grep
'"containerID"' | awk '{print $3}' | tr -d ',' | xargs | tr ' ' ','
+ [return] ${result}
+
Get Datanode Ozone Used Bytes Info
[arguments] ${uuid}
${output} = Execute export DATANODES=$(ozone admin datanode list
--json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone
admin datanode usageinfo --uuid=$\{datanode\} --json | jq
'{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add
@@ -155,7 +165,7 @@ Get Datanode Ozone Used Bytes Info
[return] ${result}
** Test Cases ***
-Verify Container Balancer for RATIS/EC containers
+Verify exclude command CLI for Container Balancer
Prepare For Tests
Datanode In Maintenance Mode
@@ -172,6 +182,23 @@ Verify Container Balancer for RATIS/EC containers
Datanode Recommission
+ ${all_containers} = Get All Container IDs
+
+ Run Container Balancer With Exclude Containers ${all_containers}
+
+ Wait Finish Of Balancing
+
+ ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} = Get Datanode
Ozone Used Bytes Info ${uuid}
+ Should Be Equal As Integers ${datanodeOzoneUsedBytesInfo}
${datanodeOzoneUsedBytesInfoAfterContainerBalancing}
+
+Verify Container Balancer for RATIS/EC containers
+
+ ${uuid} = Get Uuid
+ Datanode Usageinfo ${uuid}
+
+ ${datanodeOzoneUsedBytesInfo} = Get Datanode Ozone Used Bytes Info
${uuid}
+ Should Be True ${datanodeOzoneUsedBytesInfo} < ${SIZE}
+
Run Container Balancer
Run Balancer Status
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
index b759e7e8cbe..34b990278ab 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java
@@ -25,12 +25,16 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
import
org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration;
import
org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.ozone.test.GenericTestUtils;
@@ -89,12 +93,13 @@ public void testContainerBalancerCLIOperations() throws
Exception {
Optional<Boolean> networkTopologyEnable = Optional.of(false);
Optional<String> includeNodes = Optional.of("");
Optional<String> excludeNodes = Optional.of("");
+ Optional<String> excludeContainers = Optional.of("");
containerBalancerClient.startContainerBalancer(threshold, iterations,
maxDatanodesPercentageToInvolvePerIteration,
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB, balancingInterval, moveTimeout,
moveReplicationTimeout, networkTopologyEnable, includeNodes,
- excludeNodes);
+ excludeNodes, excludeContainers);
running = containerBalancerClient.getContainerBalancerStatus();
assertTrue(running);
@@ -116,7 +121,7 @@ public void testContainerBalancerCLIOperations() throws
Exception {
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB, balancingInterval, moveTimeout,
moveReplicationTimeout, networkTopologyEnable, includeNodes,
- excludeNodes);
+ excludeNodes, excludeContainers);
running = containerBalancerClient.getContainerBalancerStatus();
assertTrue(running);
@@ -143,6 +148,7 @@ public void testIfCBCLIOverridesConfigs() throws Exception {
//CLI option for iterations and balancing interval is not passed
Optional<Integer> iterations = Optional.empty();
Optional<Integer> balancingInterval = Optional.empty();
+ String excludedContainersList = "1,2,3";
//CLI options are passed
Optional<Double> threshold = Optional.of(0.1);
@@ -156,12 +162,13 @@ public void testIfCBCLIOverridesConfigs() throws
Exception {
Optional<Boolean> networkTopologyEnable = Optional.of(true);
Optional<String> includeNodes = Optional.of("");
Optional<String> excludeNodes = Optional.of("");
+ Optional<String> excludeContainers = Optional.of(excludedContainersList);
containerBalancerClient.startContainerBalancer(threshold, iterations,
maxDatanodesPercentageToInvolvePerIteration,
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB, balancingInterval, moveTimeout,
moveReplicationTimeout, networkTopologyEnable, includeNodes,
- excludeNodes);
+ excludeNodes, excludeContainers);
running = containerBalancerClient.getContainerBalancerStatus();
assertTrue(running);
@@ -179,6 +186,9 @@ public void testIfCBCLIOverridesConfigs() throws Exception {
//then it takes the CLI option.
assertEquals(100, config.getMaxDatanodesPercentageToInvolvePerIteration());
+ //Verifies that the 'excludeContainers' passed via CLI overrides the
default empty set
+ assertEquals(parseContainerIDs(excludedContainersList),
config.getExcludeContainers());
+
containerBalancerClient.stopContainerBalancer();
running = containerBalancerClient.getContainerBalancerStatus();
assertFalse(running);
@@ -207,12 +217,13 @@ public void testStopBalancerIdempotent() throws
IOException {
Optional<Boolean> networkTopologyEnable = Optional.of(false);
Optional<String> includeNodes = Optional.of("");
Optional<String> excludeNodes = Optional.of("");
+ Optional<String> excludeContainers = Optional.of("");
containerBalancerClient.startContainerBalancer(threshold, iterations,
maxDatanodesPercentageToInvolvePerIteration,
maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
maxSizeLeavingSourceInGB, balancingInterval, moveTimeout,
moveReplicationTimeout, networkTopologyEnable, includeNodes,
- excludeNodes);
+ excludeNodes, excludeContainers);
running = containerBalancerClient.getContainerBalancerStatus();
assertTrue(running);
@@ -223,4 +234,11 @@ public void testStopBalancerIdempotent() throws
IOException {
// Calling stop balancer again should not throw an exception
assertDoesNotThrow(() -> containerBalancerClient.stopContainerBalancer());
}
+
+ private Set<ContainerID> parseContainerIDs(String containerList) {
+ return Arrays.stream(containerList.split(","))
+ .map(Long::parseLong)
+ .map(ContainerID::valueOf)
+ .collect(Collectors.toSet());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
