This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a3f987ffe1 HDDS-10917. Refactor more tests from
TestContainerBalancerTask (#6734)
a3f987ffe1 is described below
commit a3f987ffe12c50ca60887cabbb9af8db85f7969d
Author: Andrei Mikhalev <[email protected]>
AuthorDate: Wed Jul 31 22:21:25 2024 -0700
HDDS-10917. Refactor more tests from TestContainerBalancerTask (#6734)
---
.../TestContainerBalancerDatanodeNodeLimit.java | 275 ++++++++++++++++++++-
.../balancer/TestContainerBalancerTask.java | 259 -------------------
2 files changed, 269 insertions(+), 265 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
index fc8eaf2ff5..7a8f655f06 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java
@@ -21,8 +21,16 @@ package org.apache.hadoop.hdds.scm.container.balancer;
import jakarta.annotation.Nonnull;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ContainerReplicaNotFoundException;
import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ozone.test.GenericTestUtils;
@@ -37,9 +45,13 @@ import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import static
org.apache.hadoop.hdds.scm.container.balancer.TestableCluster.RANDOM;
@@ -47,6 +59,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.atLeastOnce;
@@ -256,14 +269,11 @@ public class TestContainerBalancerDatanodeNodeLimit {
mockedSCM.disableLegacyReplicationManager();
mockedSCM.startBalancerTask(config);
- verify(mockedSCM.getMoveManager(), atLeastOnce())
- .move(any(ContainerID.class),
- any(DatanodeDetails.class),
- any(DatanodeDetails.class));
+ verify(mockedSCM.getMoveManager(), atLeastOnce()).
+ move(any(ContainerID.class), any(DatanodeDetails.class),
any(DatanodeDetails.class));
verify(mockedSCM.getReplicationManager(), times(0))
- .move(any(ContainerID.class), any(
- DatanodeDetails.class), any(DatanodeDetails.class));
+ .move(any(ContainerID.class), any(DatanodeDetails.class),
any(DatanodeDetails.class));
}
@ParameterizedTest(name = "MockedSCM #{index}: {0}")
@@ -325,6 +335,259 @@ public class TestContainerBalancerDatanodeNodeLimit {
assertEquals(1, metrics.getNumContainerMovesFailed());
}
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void containerBalancerShouldSelectOnlyClosedContainers(@Nonnull
MockedSCM mockedSCM) {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(10);
+ config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
+
+ Map<ContainerID, ContainerInfo> cidToInfoMap =
mockedSCM.getCluster().getCidToInfoMap();
+ // Make all containers open, balancer should not select any of them
+ for (ContainerInfo containerInfo : cidToInfoMap.values()) {
+ containerInfo.setState(HddsProtos.LifeCycleState.OPEN);
+ }
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+
+ // Balancer should have identified unbalanced nodes
+
assertFalse(TestContainerBalancerDatanodeNodeLimit.getUnBalancedNodes(task).isEmpty());
+ // No container should have been selected
+ assertTrue(task.getContainerToSourceMap().isEmpty());
+
+ // Iteration result should be CAN_NOT_BALANCE_ANY_MORE because no
container move is generated
+
assertEquals(ContainerBalancerTask.IterationResult.CAN_NOT_BALANCE_ANY_MORE,
task.getIterationResult());
+
+ // Now, close all containers
+ for (ContainerInfo containerInfo : cidToInfoMap.values()) {
+ containerInfo.setState(HddsProtos.LifeCycleState.CLOSED);
+ }
+ ContainerBalancerTask nextTask = mockedSCM.startBalancerTask(config);
+
+ // Check whether all selected containers are closed
+ for (ContainerID cid: nextTask.getContainerToSourceMap().keySet()) {
+ assertSame(cidToInfoMap.get(cid).getState(),
HddsProtos.LifeCycleState.CLOSED);
+ }
+ }
+
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void balancerShouldNotSelectNonClosedContainerReplicas(@Nonnull
MockedSCM mockedSCM)
+ throws ContainerNotFoundException {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(10);
+ config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
+
+ // Let's mock such that all replicas have CLOSING state
+ Map<ContainerID, Set<ContainerReplica>> cidToReplicasMap =
mockedSCM.getCluster().getCidToReplicasMap();
+
when(mockedSCM.getContainerManager().getContainerReplicas(any(ContainerID.class)))
+ .thenAnswer(invocationOnMock -> {
+ ContainerID cid = (ContainerID) invocationOnMock.getArguments()[0];
+ Set<ContainerReplica> replicas = cidToReplicasMap.get(cid);
+ Set<ContainerReplica> replicasToReturn = new
HashSet<>(replicas.size());
+ for (ContainerReplica replica : replicas) {
+ ContainerReplica newReplica = replica.toBuilder()
+
.setContainerState(StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSING)
+ .build();
+ replicasToReturn.add(newReplica);
+ }
+
+ return replicasToReturn;
+ });
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+
+ // Balancer should have identified unbalanced nodes
+
assertFalse(TestContainerBalancerDatanodeNodeLimit.getUnBalancedNodes(task).isEmpty());
+ // No container should have moved because all replicas are CLOSING
+ assertTrue(task.getContainerToSourceMap().isEmpty());
+ }
+
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void containerBalancerShouldObeyMaxSizeToMoveLimit(@Nonnull MockedSCM
mockedSCM) {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(1);
+ config.setMaxSizeToMovePerIteration(10 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(10 * STORAGE_UNIT);
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+
+ // Balancer should not have moved more size than the limit
+
assertThat(task.getSizeScheduledForMoveInLatestIteration()).isLessThanOrEqualTo(10
* STORAGE_UNIT);
+
+ long size = task.getMetrics().getDataSizeMovedGBInLatestIteration();
+ assertThat(size).isGreaterThan(0);
+ assertThat(size).isLessThanOrEqualTo(10);
+ }
+
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void targetDatanodeShouldNotAlreadyContainSelectedContainer(@Nonnull
MockedSCM mockedSCM) {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(10);
+ config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+
+ Map<ContainerID, DatanodeDetails> map = task.getContainerToTargetMap();
+ Map<ContainerID, Set<ContainerReplica>> cidToReplicasMap =
mockedSCM.getCluster().getCidToReplicasMap();
+ for (Map.Entry<ContainerID, DatanodeDetails> entry : map.entrySet()) {
+ ContainerID container = entry.getKey();
+ DatanodeDetails target = entry.getValue();
+ assertTrue(cidToReplicasMap.get(container)
+ .stream()
+ .map(ContainerReplica::getDatanodeDetails)
+ .noneMatch(target::equals));
+ }
+ }
+
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void containerMoveSelectionShouldFollowPlacementPolicy(@Nonnull
MockedSCM mockedSCM) {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(10);
+ config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+
+ Map<ContainerID, DatanodeDetails> containerFromSourceMap =
task.getContainerToSourceMap();
+ Map<ContainerID, DatanodeDetails> containerToTargetMap =
task.getContainerToTargetMap();
+
+ // For each move selection, check if {replicas - source + target}
satisfies placement policy
+ for (Map.Entry<ContainerID, DatanodeDetails> entry :
containerFromSourceMap.entrySet()) {
+ ContainerID container = entry.getKey();
+ DatanodeDetails source = entry.getValue();
+
+ List<DatanodeDetails> replicas =
mockedSCM.getCluster().getCidToReplicasMap().get(container)
+ .stream()
+ .map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+ // Remove source and add target
+ replicas.remove(source);
+ replicas.add(containerToTargetMap.get(container));
+
+ ContainerInfo containerInfo =
mockedSCM.getCluster().getCidToInfoMap().get(container);
+ ContainerPlacementStatus placementStatus;
+ int requiredNodes =
containerInfo.getReplicationConfig().getRequiredNodes();
+ if (containerInfo.getReplicationType() ==
HddsProtos.ReplicationType.RATIS) {
+ placementStatus =
mockedSCM.getPlacementPolicy().validateContainerPlacement(replicas,
requiredNodes);
+ } else {
+ placementStatus =
mockedSCM.getEcPlacementPolicy().validateContainerPlacement(replicas,
requiredNodes);
+ }
+ assertTrue(placementStatus.isPolicySatisfied());
+ }
+ }
+
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void targetDatanodeShouldBeInServiceHealthy(@Nonnull MockedSCM
mockedSCM) throws NodeNotFoundException {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(10);
+ config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+
+ for (DatanodeDetails target : task.getSelectedTargets()) {
+ NodeStatus status = mockedSCM.getNodeManager().getNodeStatus(target);
+ assertSame(HddsProtos.NodeOperationalState.IN_SERVICE,
status.getOperationalState());
+ assertTrue(status.isHealthy());
+ }
+ }
+
+
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void selectedContainerShouldNotAlreadyHaveBeenSelected(@Nonnull
MockedSCM mockedSCM)
+ throws NodeNotFoundException, ContainerNotFoundException,
TimeoutException, ContainerReplicaNotFoundException {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(10);
+ config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
+
+ mockedSCM.enableLegacyReplicationManager();
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+ int numContainers = task.getContainerToTargetMap().size();
+
+ /* Assuming move is called exactly once for each unique container, number
of calls to move should equal number of
+ unique containers. If number of calls to move is more than number of
unique containers, at least one container
+ has been re-selected. It's expected that number of calls to move should
equal number of unique, selected containers
+ (from containerToTargetMap).
+ */
+ verify(mockedSCM.getReplicationManager(), times(numContainers))
+ .move(any(ContainerID.class), any(DatanodeDetails.class),
any(DatanodeDetails.class));
+
+ // Try the same test by disabling LegacyReplicationManager so that
MoveManager is used.
+ mockedSCM.disableLegacyReplicationManager();
+ ContainerBalancerTask nextTask = mockedSCM.startBalancerTask(config);
+
+ numContainers = nextTask.getContainerToTargetMap().size();
+ verify(mockedSCM.getMoveManager(), times(numContainers))
+ .move(any(ContainerID.class), any(DatanodeDetails.class),
any(DatanodeDetails.class));
+ }
+
+ @ParameterizedTest(name = "MockedSCM #{index}: {0}")
+ @MethodSource("createMockedSCMs")
+ public void balancerShouldNotSelectConfiguredExcludeContainers(@Nonnull
MockedSCM mockedSCM) {
+ ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(new
OzoneConfiguration());
+ int nodeCount = mockedSCM.getCluster().getNodeCount();
+ if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) {
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ }
+ config.setIterations(1);
+ config.setThreshold(10);
+ config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
+ config.setExcludeContainers("1, 4, 5");
+
+ ContainerBalancerTask task = mockedSCM.startBalancerTask(config);
+
+ Set<ContainerID> excludeContainers = config.getExcludeContainers();
+ for (ContainerID container : task.getContainerToSourceMap().keySet()) {
+ assertThat(excludeContainers).doesNotContain(container);
+ }
+ }
public static List<DatanodeUsageInfo> getUnBalancedNodes(@Nonnull
ContainerBalancerTask task) {
ArrayList<DatanodeUsageInfo> result = new ArrayList<>();
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java
index 0f4551b45c..d0e9cd53fe 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
-import org.apache.hadoop.hdds.scm.ContainerPlacementStatus;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.PlacementPolicyValidateProxy;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -47,7 +46,6 @@ import
org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManager;
import org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManagerImpl;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo;
-import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -74,11 +72,9 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame;
-import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static
org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration;
@@ -86,10 +82,8 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.anyString;
-import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
import static org.mockito.Mockito.when;
/**
@@ -248,259 +242,6 @@ public class TestContainerBalancerTask {
sb.getMetrics(), balancerConfiguration, false);
}
- @Test
- public void containerBalancerShouldSelectOnlyClosedContainers()
- throws IllegalContainerBalancerStateException, IOException,
- InvalidContainerBalancerConfigurationException, TimeoutException {
- // make all containers open, balancer should not select any of them
- for (ContainerInfo containerInfo : cidToInfoMap.values()) {
- containerInfo.setState(HddsProtos.LifeCycleState.OPEN);
- }
- balancerConfiguration.setThreshold(10);
- startBalancer(balancerConfiguration);
- stopBalancer();
-
- // balancer should have identified unbalanced nodes
-
assertFalse(TestContainerBalancerDatanodeNodeLimit.getUnBalancedNodes(containerBalancerTask).isEmpty());
- // no container should have been selected
- assertTrue(containerBalancerTask.getContainerToSourceMap()
- .isEmpty());
- /*
- Iteration result should be CAN_NOT_BALANCE_ANY_MORE because no container
- move is generated
- */
- assertEquals(
- ContainerBalancerTask.IterationResult.CAN_NOT_BALANCE_ANY_MORE,
- containerBalancerTask.getIterationResult());
-
- // now, close all containers
- for (ContainerInfo containerInfo : cidToInfoMap.values()) {
- containerInfo.setState(HddsProtos.LifeCycleState.CLOSED);
- }
- startBalancer(balancerConfiguration);
- stopBalancer();
-
- // check whether all selected containers are closed
- for (ContainerID cid:
- containerBalancerTask.getContainerToSourceMap().keySet()) {
- assertSame(
- cidToInfoMap.get(cid).getState(), HddsProtos.LifeCycleState.CLOSED);
- }
- }
-
- /**
- * Container Balancer should not select a non-CLOSED replica for moving.
- */
- @Test
- public void balancerShouldNotSelectNonClosedContainerReplicas()
- throws IOException, IllegalContainerBalancerStateException,
- InvalidContainerBalancerConfigurationException, TimeoutException {
-
- // let's mock such that all replicas have CLOSING state
- when(containerManager.getContainerReplicas(any(ContainerID.class)))
- .thenAnswer(invocationOnMock -> {
- ContainerID cid = (ContainerID) invocationOnMock.getArguments()[0];
- Set<ContainerReplica> replicas = cidToReplicasMap.get(cid);
- Set<ContainerReplica> replicasToReturn =
- new HashSet<>(replicas.size());
- for (ContainerReplica replica : replicas) {
- ContainerReplica newReplica =
- replica.toBuilder().setContainerState(
- ContainerReplicaProto.State.CLOSING).build();
- replicasToReturn.add(newReplica);
- }
-
- return replicasToReturn;
- });
-
- balancerConfiguration.setThreshold(10);
- balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
- balancerConfiguration.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
- balancerConfiguration.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
-
- startBalancer(balancerConfiguration);
- stopBalancer();
-
- // balancer should have identified unbalanced nodes
-
assertFalse(TestContainerBalancerDatanodeNodeLimit.getUnBalancedNodes(containerBalancerTask).isEmpty());
- // no container should have moved because all replicas are CLOSING
- assertTrue(
- containerBalancerTask.getContainerToSourceMap().isEmpty());
- }
-
- @Test
- public void containerBalancerShouldObeyMaxSizeToMoveLimit()
- throws IllegalContainerBalancerStateException, IOException,
- InvalidContainerBalancerConfigurationException, TimeoutException {
- balancerConfiguration.setThreshold(1);
- balancerConfiguration.setMaxSizeToMovePerIteration(10 * STORAGE_UNIT);
- balancerConfiguration.setIterations(1);
- startBalancer(balancerConfiguration);
-
- // balancer should not have moved more size than the limit
-
assertThat(containerBalancerTask.getSizeScheduledForMoveInLatestIteration())
- .isLessThanOrEqualTo(10 * STORAGE_UNIT);
-
- long size = containerBalancerTask.getMetrics()
- .getDataSizeMovedGBInLatestIteration();
- assertThat(size).isGreaterThan(0);
- assertThat(size).isLessThanOrEqualTo(10);
- stopBalancer();
- }
-
- @Test
- public void targetDatanodeShouldNotAlreadyContainSelectedContainer()
- throws IllegalContainerBalancerStateException, IOException,
- InvalidContainerBalancerConfigurationException, TimeoutException {
- balancerConfiguration.setThreshold(10);
- balancerConfiguration.setMaxSizeToMovePerIteration(100 * STORAGE_UNIT);
- balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
- startBalancer(balancerConfiguration);
-
- stopBalancer();
- Map<ContainerID, DatanodeDetails> map =
- containerBalancerTask.getContainerToTargetMap();
- for (Map.Entry<ContainerID, DatanodeDetails> entry : map.entrySet()) {
- ContainerID container = entry.getKey();
- DatanodeDetails target = entry.getValue();
- assertTrue(cidToReplicasMap.get(container)
- .stream()
- .map(ContainerReplica::getDatanodeDetails)
- .noneMatch(target::equals));
- }
- }
-
- @Test
- public void containerMoveSelectionShouldFollowPlacementPolicy()
- throws IllegalContainerBalancerStateException, IOException,
- InvalidContainerBalancerConfigurationException, TimeoutException {
- balancerConfiguration.setThreshold(10);
- balancerConfiguration.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
- balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
- balancerConfiguration.setIterations(1);
- startBalancer(balancerConfiguration);
-
- stopBalancer();
- Map<ContainerID, DatanodeDetails> containerFromSourceMap =
- containerBalancerTask.getContainerToSourceMap();
- Map<ContainerID, DatanodeDetails> containerToTargetMap =
- containerBalancerTask.getContainerToTargetMap();
-
- // for each move selection, check if {replicas - source + target}
- // satisfies placement policy
- for (Map.Entry<ContainerID, DatanodeDetails> entry :
- containerFromSourceMap.entrySet()) {
- ContainerID container = entry.getKey();
- DatanodeDetails source = entry.getValue();
-
- List<DatanodeDetails> replicas = cidToReplicasMap.get(container)
- .stream()
- .map(ContainerReplica::getDatanodeDetails)
- .collect(Collectors.toList());
- // remove source and add target
- replicas.remove(source);
- replicas.add(containerToTargetMap.get(container));
-
- ContainerInfo containerInfo = cidToInfoMap.get(container);
- ContainerPlacementStatus placementStatus;
- if (containerInfo.getReplicationType() ==
- HddsProtos.ReplicationType.RATIS) {
- placementStatus = placementPolicy.validateContainerPlacement(replicas,
- containerInfo.getReplicationConfig().getRequiredNodes());
- } else {
- placementStatus =
- ecPlacementPolicy.validateContainerPlacement(replicas,
- containerInfo.getReplicationConfig().getRequiredNodes());
- }
- assertTrue(placementStatus.isPolicySatisfied());
- }
- }
-
- @Test
- public void targetDatanodeShouldBeInServiceHealthy()
- throws NodeNotFoundException, IllegalContainerBalancerStateException,
- IOException, InvalidContainerBalancerConfigurationException,
- TimeoutException {
- balancerConfiguration.setThreshold(10);
- balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
- balancerConfiguration.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
- balancerConfiguration.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
- balancerConfiguration.setIterations(1);
- startBalancer(balancerConfiguration);
-
- stopBalancer();
- for (DatanodeDetails target : containerBalancerTask.getSelectedTargets()) {
- NodeStatus status = mockNodeManager.getNodeStatus(target);
- assertSame(HddsProtos.NodeOperationalState.IN_SERVICE,
- status.getOperationalState());
- assertTrue(status.isHealthy());
- }
- }
-
- @Test
- public void selectedContainerShouldNotAlreadyHaveBeenSelected()
- throws IllegalContainerBalancerStateException, IOException,
- InvalidContainerBalancerConfigurationException, NodeNotFoundException,
- TimeoutException {
- balancerConfiguration.setThreshold(10);
- balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
- balancerConfiguration.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
- balancerConfiguration.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
- balancerConfiguration.setIterations(1);
- rmConf.setEnableLegacy(true);
-
- startBalancer(balancerConfiguration);
-
- stopBalancer();
-
- int numContainers = containerBalancerTask.getContainerToTargetMap().size();
-
- /*
- Assuming move is called exactly once for each unique container, number of
- calls to move should equal number of unique containers. If number of
- calls to move is more than number of unique containers, at least one
- container has been re-selected. It's expected that number of calls to
- move should equal number of unique, selected containers (from
- containerToTargetMap).
- */
- verify(replicationManager, times(numContainers))
- .move(any(ContainerID.class), any(DatanodeDetails.class),
- any(DatanodeDetails.class));
-
- /*
- Try the same test by disabling LegacyReplicationManager so that
- MoveManager is used.
- */
- rmConf.setEnableLegacy(false);
- startBalancer(balancerConfiguration);
- stopBalancer();
- numContainers = containerBalancerTask.getContainerToTargetMap().size();
- verify(moveManager, times(numContainers))
- .move(any(ContainerID.class), any(DatanodeDetails.class),
- any(DatanodeDetails.class));
- }
-
- @Test
- public void balancerShouldNotSelectConfiguredExcludeContainers()
- throws IllegalContainerBalancerStateException, IOException,
- InvalidContainerBalancerConfigurationException, TimeoutException {
- balancerConfiguration.setThreshold(10);
- balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100);
- balancerConfiguration.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT);
- balancerConfiguration.setMaxSizeEnteringTarget(50 * STORAGE_UNIT);
- balancerConfiguration.setExcludeContainers("1, 4, 5");
-
- startBalancer(balancerConfiguration);
-
- stopBalancer();
- Set<ContainerID> excludeContainers =
- balancerConfiguration.getExcludeContainers();
- for (ContainerID container :
- containerBalancerTask.getContainerToSourceMap().keySet()) {
- assertThat(excludeContainers).doesNotContain(container);
- }
- }
-
/**
* Tests if {@link ContainerBalancer} follows the includeNodes and
* excludeNodes configurations in {@link ContainerBalancerConfiguration}.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
