This is an automated email from the ASF dual-hosted git repository.
sarvekshayr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e6dfa38dbdf HDDS-15889. Show failed MoveResult breakdown in ozone
admin containerbalancer status -v (#10938)
e6dfa38dbdf is described below
commit e6dfa38dbdf0657a6a0d1ef701fa53c366275dce
Author: sreejasahithi <[email protected]>
AuthorDate: Tue Aug 25 13:52:53 2026 +0530
HDDS-15889. Show failed MoveResult breakdown in ozone admin
containerbalancer status -v (#10938)
---
.../docs/content/feature/ContainerBalancer.md | 4 +
.../src/main/proto/ScmAdminProtocol.proto | 14 ++
.../interface-admin/src/main/resources/proto.lock | 58 +++++
.../container/balancer/ContainerBalancerTask.java | 60 ++++-
.../ContainerBalancerTaskIterationStatusInfo.java | 42 ++++
.../balancer/ContainerMoveFailureDetail.java | 62 +++++
.../balancer/ContainerMoveFailureTracker.java | 93 +++++++
.../scm/container/balancer/ContainerMoveInfo.java | 13 +-
.../TestContainerBalancerMoveFailureBreakdown.java | 278 +++++++++++++++++++++
.../scm/cli/ContainerBalancerStatusSubcommand.java | 43 ++++
.../datanode/TestContainerBalancerSubCommand.java | 109 ++++++++
11 files changed, 764 insertions(+), 12 deletions(-)
diff --git a/hadoop-hdds/docs/content/feature/ContainerBalancer.md
b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
index 848e1b998ff..94689d53223 100644
--- a/hadoop-hdds/docs/content/feature/ContainerBalancer.md
+++ b/hadoop-hdds/docs/content/feature/ContainerBalancer.md
@@ -78,6 +78,10 @@ To get a more detailed status, including the history of
iterations:
ozone admin containerbalancer status -v --history
```
+With `-v`/`--verbose`, when failures occur, a **Failed container moves**
section lists each failure reason
+(e.g. `REPLICATION_FAIL_TIME_OUT`, `ITERATION_MOVE_TIMEOUT`,
`PRE_MOVE_CONTAINER_NOT_FOUND` and more)
+with the total count and a per-datanode breakdown of which source and target
nodes were involved.
+
### Stop
To stop the Container Balancer:
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index 6c39fc22cc4..9e7244ecf82 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -675,6 +675,20 @@ message ContainerBalancerTaskIterationStatusInfoProto {
repeated NodeTransferInfoProto sizeEnteringNodes = 9;
repeated NodeTransferInfoProto sizeLeavingNodes = 10;
optional int64 iterationDuration = 11;
+ repeated ContainerMoveFailureDetailProto containerMoveFailures = 12;
+}
+
+message ContainerMoveFailureDetailProto {
+ optional string reason = 1;
+ optional int64 count = 2;
+ repeated NodeFailureCountProto sourceFailureCounts = 3;
+ repeated NodeFailureCountProto targetFailureCounts = 4;
+}
+
+message NodeFailureCountProto {
+ optional string datanodeUuid = 1;
+ optional int64 count = 2;
+ optional string hostname = 3;
}
message NodeTransferInfoProto {
diff --git a/hadoop-hdds/interface-admin/src/main/resources/proto.lock
b/hadoop-hdds/interface-admin/src/main/resources/proto.lock
index 02184011b69..74954907009 100644
--- a/hadoop-hdds/interface-admin/src/main/resources/proto.lock
+++ b/hadoop-hdds/interface-admin/src/main/resources/proto.lock
@@ -2339,6 +2339,64 @@
"name": "iterationDuration",
"type": "int64",
"optional": true
+ },
+ {
+ "id": 12,
+ "name": "containerMoveFailures",
+ "type": "ContainerMoveFailureDetailProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "ContainerMoveFailureDetailProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "reason",
+ "type": "string",
+ "optional": true
+ },
+ {
+ "id": 2,
+ "name": "count",
+ "type": "int64",
+ "optional": true
+ },
+ {
+ "id": 3,
+ "name": "sourceFailureCounts",
+ "type": "NodeFailureCountProto",
+ "is_repeated": true
+ },
+ {
+ "id": 4,
+ "name": "targetFailureCounts",
+ "type": "NodeFailureCountProto",
+ "is_repeated": true
+ }
+ ]
+ },
+ {
+ "name": "NodeFailureCountProto",
+ "fields": [
+ {
+ "id": 1,
+ "name": "datanodeUuid",
+ "type": "string",
+ "optional": true
+ },
+ {
+ "id": 2,
+ "name": "count",
+ "type": "int64",
+ "optional": true
+ },
+ {
+ "id": 3,
+ "name": "hostname",
+ "type": "string",
+ "optional": true
}
]
},
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
index c409c5eff7a..358d696d5b8 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
@@ -37,6 +37,7 @@
import java.util.Objects;
import java.util.Queue;
import java.util.Set;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -94,6 +95,7 @@ public class ContainerBalancerTask implements Runnable {
private Set<String> includeNodes;
private ContainerBalancerConfiguration config;
private ContainerBalancerMetrics metrics;
+ private final ContainerMoveFailureTracker moveFailureTracker;
private double upperLimit;
private double lowerLimit;
private ContainerBalancerSelectionCriteria selectionCriteria;
@@ -152,6 +154,7 @@ public ContainerBalancerTask(StorageContainerManager scm,
this.containerBalancer = containerBalancer;
this.config = config;
this.metrics = metrics;
+ this.moveFailureTracker = new ContainerMoveFailureTracker();
this.scmContext = scm.getScmContext();
this.overUtilizedNodes = new ArrayList<>();
this.underUtilizedNodes = new ArrayList<>();
@@ -353,7 +356,7 @@ private ContainerBalancerTaskIterationStatusInfo
getIterationStatistic(Integer i
currentIterationResultName,
iterationDuration
);
- ContainerMoveInfo containerMoveInfo = new ContainerMoveInfo(metrics);
+ ContainerMoveInfo containerMoveInfo = new ContainerMoveInfo(metrics,
moveFailureTracker);
DataMoveInfo dataMoveInfo =
getDataMoveInfo(sizeEnteringDataToNodes, sizeLeavingDataFromNodes);
@@ -858,11 +861,13 @@ private long cancelMovesThatExceedTimeoutDuration() {
CompletableFuture<MoveManager.MoveResult>>
entry = iterator.next();
if (!entry.getValue().isDone()) {
+ ContainerMoveSelection moveSelection = entry.getKey();
+ ContainerID containerID = moveSelection.getContainerID();
+ DatanodeDetails source = containerToSourceMap.get(containerID);
+ DatanodeDetails target = moveSelection.getTargetNode();
LOG.warn("Container move timed out for container {} from source {}" +
- " to target {}.", entry.getKey().getContainerID(),
- containerToSourceMap.get(entry.getKey().getContainerID()),
- entry.getKey().getTargetNode());
-
+ " to target {}.", containerID, source, target);
+
moveFailureTracker.recordFailure(ContainerMoveFailureReason.ITERATION_MOVE_TIMEOUT,
source, target);
entry.getValue().cancel(true);
numCancelled += 1;
}
@@ -1011,11 +1016,15 @@ private boolean moveContainer(DatanodeDetails source,
metrics.incrementCurrentIterationContainerMoveMetric(result, 1);
moveSelectionToFutureMap.remove(moveSelection);
if (ex != null) {
- LOG.info("Container move for container {} from source {} to " +
- "target {} failed with exceptions.",
- containerID, source,
- moveSelection.getTargetNode(), ex);
- metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+ if (!isCancellationCause(ex)) {
+ LOG.info("Container move for container {} from source {} to " +
+ "target {} failed with exceptions.",
+ containerID, source,
+ moveSelection.getTargetNode(), ex);
+ metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+
moveFailureTracker.recordFailure(MoveManager.MoveResult.FAIL_UNEXPECTED_ERROR,
+ source, moveSelection.getTargetNode());
+ }
} else {
if (result == MoveManager.MoveResult.COMPLETED) {
metrics.incrementDataSizeMovedInLatestIteration(containerInfo.getUsedBytes());
@@ -1028,6 +1037,7 @@ private boolean moveContainer(DatanodeDetails source,
" {} failed: {}",
moveSelection.getContainerID(), source,
moveSelection.getTargetNode(), result);
+ moveFailureTracker.recordFailure(result, source,
moveSelection.getTargetNode());
}
}
});
@@ -1039,14 +1049,20 @@ private boolean moveContainer(DatanodeDetails source,
// exclude the permanently missing container across balancer iterations.
selectionCriteria.addToExcludeNotFoundContainers(moveSelection.getContainerID());
metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+
moveFailureTracker.recordFailure(ContainerMoveFailureReason.PRE_MOVE_CONTAINER_NOT_FOUND,
+ source, moveSelection.getTargetNode());
return false;
} catch (NodeNotFoundException e) {
LOG.warn("Container move failed for container {}", containerID, e);
metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+
moveFailureTracker.recordFailure(ContainerMoveFailureReason.PRE_MOVE_NODE_NOT_FOUND,
+ source, moveSelection.getTargetNode());
return false;
} catch (ContainerReplicaNotFoundException e) {
LOG.warn("Container move failed for container {}", containerID, e);
metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+
moveFailureTracker.recordFailure(ContainerMoveFailureReason.PRE_MOVE_REPLICA_NOT_FOUND,
+ source, moveSelection.getTargetNode());
// add source back to queue for replica not found only
// the container is not excluded as it is a replica related failure
findSourceStrategy.addBackSourceDataNode(source);
@@ -1233,6 +1249,11 @@ private void resetState() {
metrics.resetDataSizeUnbalancedGB();
metrics.resetNumDatanodesUnbalanced();
metrics.resetNumContainerMovesFailedInLatestIteration();
+ moveFailureTracker.reset();
+ }
+
+ private static boolean isCancellationCause(Throwable ex) {
+ return ex instanceof CancellationException;
}
/**
@@ -1350,4 +1371,23 @@ enum Status {
STOPPING,
STOPPED
}
+
+ /**
+ * Failure reasons recorded by {@link ContainerBalancerTask} that are not
represented by
+ * {@link MoveManager.MoveResult}. Other move failures use {@code
MoveResult} names in the
+ * failure breakdown.
+ * <p>
+ * {@code PRE_MOVE_*} values are recorded when {@link MoveManager#move}
throws before returning
+ * a completed future. {@link #ITERATION_MOVE_TIMEOUT} is recorded when an
in-flight move does
+ * not finish before the iteration move wait timeout expires.
+ */
+ enum ContainerMoveFailureReason {
+ PRE_MOVE_CONTAINER_NOT_FOUND,
+ PRE_MOVE_NODE_NOT_FOUND,
+ PRE_MOVE_REPLICA_NOT_FOUND,
+ /**
+ * Move did not complete before the iteration move wait timeout expired.
+ */
+ ITERATION_MOVE_TIMEOUT
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTaskIterationStatusInfo.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTaskIterationStatusInfo.java
index f16f3cfe7e5..7791b6db12b 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTaskIterationStatusInfo.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTaskIterationStatusInfo.java
@@ -106,6 +106,14 @@ public long getContainerMovesTimeout() {
return containerMoveInfo.getContainerMovesTimeout();
}
+ /**
+ * Get per-reason failure summaries with per-datanode counts for this
iteration.
+ * @return list of failure details, one entry per failure reason
+ */
+ public List<ContainerMoveFailureDetail> getFailures() {
+ return containerMoveInfo.getFailures();
+ }
+
/**
* Get a map of the node IDs and the corresponding data sizes moved to each
node.
* @return nodeId to size entering from node map
@@ -151,9 +159,43 @@ public
StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStat
.addAllSizeLeavingNodes(
mapToProtoNodeTransferInfo(getSizeLeavingNodes())
)
+ .addAllContainerMoveFailures(mapToProtoFailures(getFailures()))
.build();
}
+ private
List<StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto>
mapToProtoFailures(
+ List<ContainerMoveFailureDetail> failures) {
+ return failures.stream()
+ .map(failure -> {
+ Map<String, String> datanodeHostnames =
failure.getDatanodeHostnames();
+
StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto.Builder
builder =
+
StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto.newBuilder()
+ .setReason(failure.getReason())
+ .setCount(failure.getCount());
+ failure.getSourceFailureCounts().forEach((uuid, count) ->
+ builder.addSourceFailureCounts(
+ toNodeFailureCountProto(uuid, count, datanodeHostnames)));
+ failure.getTargetFailureCounts().forEach((uuid, count) ->
+ builder.addTargetFailureCounts(
+ toNodeFailureCountProto(uuid, count, datanodeHostnames)));
+ return builder.build();
+ })
+ .collect(Collectors.toList());
+ }
+
+ private StorageContainerLocationProtocolProtos.NodeFailureCountProto
toNodeFailureCountProto(
+ String uuid, long count, Map<String, String> datanodeHostnames) {
+ StorageContainerLocationProtocolProtos.NodeFailureCountProto.Builder
builder =
+
StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+ .setDatanodeUuid(uuid)
+ .setCount(count);
+ String hostname = datanodeHostnames.get(uuid);
+ if (hostname != null && !hostname.isEmpty()) {
+ builder.setHostname(hostname);
+ }
+ return builder.build();
+ }
+
/**
* Converts an instance into the protobuf compatible object.
* @param nodes node id to node traffic size
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureDetail.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureDetail.java
new file mode 100644
index 00000000000..a25f44a4f76
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureDetail.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Per-reason container move failure summary with per-datanode failure counts.
+ */
+public final class ContainerMoveFailureDetail {
+ private final String reason;
+ private final long count;
+ private final Map<String, Long> sourceFailureCounts;
+ private final Map<String, Long> targetFailureCounts;
+ private final Map<String, String> datanodeHostnames;
+
+ public ContainerMoveFailureDetail(String reason, long count,
+ Map<String, Long> sourceFailureCounts, Map<String, Long>
targetFailureCounts,
+ Map<String, String> datanodeHostnames) {
+ this.reason = reason;
+ this.count = count;
+ this.sourceFailureCounts =
Collections.unmodifiableMap(sourceFailureCounts);
+ this.targetFailureCounts =
Collections.unmodifiableMap(targetFailureCounts);
+ this.datanodeHostnames = Collections.unmodifiableMap(datanodeHostnames);
+ }
+
+ public String getReason() {
+ return reason;
+ }
+
+ public long getCount() {
+ return count;
+ }
+
+ public Map<String, Long> getSourceFailureCounts() {
+ return sourceFailureCounts;
+ }
+
+ public Map<String, Long> getTargetFailureCounts() {
+ return targetFailureCounts;
+ }
+
+ public Map<String, String> getDatanodeHostnames() {
+ return datanodeHostnames;
+ }
+}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureTracker.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureTracker.java
new file mode 100644
index 00000000000..ac8d712ba4b
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureTracker.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+/**
+ * Tracks per-iteration container move failures by reason and per-datanode
counts.
+ */
+final class ContainerMoveFailureTracker {
+ private final Map<String, Long> failuresByReason = new HashMap<>();
+ private final Map<String, Map<String, Long>> sourceFailureCountsByReason =
new HashMap<>();
+ private final Map<String, Map<String, Long>> targetFailureCountsByReason =
new HashMap<>();
+ private final Map<String, String> datanodeHostnames = new HashMap<>();
+
+ synchronized void recordFailure(MoveManager.MoveResult result,
DatanodeDetails source,
+ DatanodeDetails target) {
+ recordFailure(result.name(), source, target);
+ }
+
+ synchronized void
recordFailure(ContainerBalancerTask.ContainerMoveFailureReason result,
+ DatanodeDetails source,
DatanodeDetails target) {
+ recordFailure(result.name(), source, target);
+ }
+
+ synchronized void recordFailure(String reason, DatanodeDetails source,
DatanodeDetails target) {
+ failuresByReason.merge(reason, 1L, Long::sum);
+ if (source != null) {
+ datanodeHostnames.putIfAbsent(source.getUuidString(),
source.getHostName());
+ sourceFailureCountsByReason.computeIfAbsent(reason, k -> new HashMap<>())
+ .merge(source.getUuidString(), 1L, Long::sum);
+ }
+ if (target != null) {
+ datanodeHostnames.putIfAbsent(target.getUuidString(),
target.getHostName());
+ targetFailureCountsByReason.computeIfAbsent(reason, k -> new HashMap<>())
+ .merge(target.getUuidString(), 1L, Long::sum);
+ }
+ }
+
+ synchronized void reset() {
+ failuresByReason.clear();
+ sourceFailureCountsByReason.clear();
+ targetFailureCountsByReason.clear();
+ datanodeHostnames.clear();
+ }
+
+ synchronized List<ContainerMoveFailureDetail> getFailures() {
+ List<ContainerMoveFailureDetail> result = new ArrayList<>();
+ for (Map.Entry<String, Long> entry : failuresByReason.entrySet()) {
+ String reason = entry.getKey();
+ long count = entry.getValue();
+ Map<String, Long> srcCounts =
sourceFailureCountsByReason.getOrDefault(reason, Collections.emptyMap());
+ Map<String, Long> tgtCounts =
targetFailureCountsByReason.getOrDefault(reason, Collections.emptyMap());
+ result.add(new ContainerMoveFailureDetail(reason, count, new
HashMap<>(srcCounts), new HashMap<>(tgtCounts),
+ copyHostnamesForDetail(srcCounts, tgtCounts)));
+ }
+ return result;
+ }
+
+ private Map<String, String> copyHostnamesForDetail(Map<String, Long>
srcCounts, Map<String, Long> tgtCounts) {
+ Map<String, String> hostnames = new HashMap<>();
+ srcCounts.keySet().forEach(uuid -> copyHostnameIfPresent(uuid, hostnames));
+ tgtCounts.keySet().forEach(uuid -> copyHostnameIfPresent(uuid, hostnames));
+ return hostnames;
+ }
+
+ private void copyHostnameIfPresent(String uuid, Map<String, String>
hostnames) {
+ String hostname = datanodeHostnames.get(uuid);
+ if (hostname != null && !hostname.isEmpty()) {
+ hostnames.put(uuid, hostname);
+ }
+ }
+}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveInfo.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveInfo.java
index 3b3c7f91933..2dab77120dd 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveInfo.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveInfo.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm.container.balancer;
+import java.util.List;
+
/**
* Information about moving containers.
*/
@@ -25,20 +27,23 @@ public class ContainerMoveInfo {
private final long containerMovesCompleted;
private final long containerMovesFailed;
private final long containerMovesTimeout;
+ private final List<ContainerMoveFailureDetail> failures;
public ContainerMoveInfo(long containerMovesScheduled, long
containerMovesCompleted, long containerMovesFailed,
- long containerMovesTimeout) {
+ long containerMovesTimeout,
List<ContainerMoveFailureDetail> failures) {
this.containerMovesScheduled = containerMovesScheduled;
this.containerMovesCompleted = containerMovesCompleted;
this.containerMovesFailed = containerMovesFailed;
this.containerMovesTimeout = containerMovesTimeout;
+ this.failures = failures;
}
- public ContainerMoveInfo(ContainerBalancerMetrics metrics) {
+ public ContainerMoveInfo(ContainerBalancerMetrics metrics,
ContainerMoveFailureTracker failureTracker) {
this.containerMovesScheduled =
metrics.getNumContainerMovesScheduledInLatestIteration();
this.containerMovesCompleted =
metrics.getNumContainerMovesCompletedInLatestIteration();
this.containerMovesFailed =
metrics.getNumContainerMovesFailedInLatestIteration();
this.containerMovesTimeout =
metrics.getNumContainerMovesTimeoutInLatestIteration();
+ this.failures = failureTracker.getFailures();
}
public long getContainerMovesScheduled() {
@@ -56,4 +61,8 @@ public long getContainerMovesFailed() {
public long getContainerMovesTimeout() {
return containerMovesTimeout;
}
+
+ public List<ContainerMoveFailureDetail> getFailures() {
+ return failures;
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerMoveFailureBreakdown.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerMoveFailureBreakdown.java
new file mode 100644
index 00000000000..edc159f07a4
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerMoveFailureBreakdown.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.container.ContainerReplicaNotFoundException;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+/**
+ * Tests that {@link ContainerBalancerTask} records failure breakdown and
details
+ * in iteration statistics for common move failure reasons.
+ */
+class TestContainerBalancerMoveFailureBreakdown {
+
+ private static final int NODE_COUNT = 5;
+ private static final long STORAGE_UNIT = OzoneConsts.GB;
+
+ @Test
+ void testReplicationTimeoutRecordedInFailureBreakdown()
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ mockMoveFailureOnce(mockedScm,
MoveManager.MoveResult.REPLICATION_FAIL_TIME_OUT);
+
+ ContainerBalancerTask task =
mockedScm.startBalancerTask(buildConfig(mockedScm));
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+
+ assertThat(iteration.getContainerMovesTimeout()).isEqualTo(1);
+ assertFailureBreakdown(mockedScm, iteration,
MoveManager.MoveResult.REPLICATION_FAIL_TIME_OUT.name(), 0);
+ }
+
+ @Test
+ void testReplicationNotHealthyAfterMoveRecordedInFailureBreakdown()
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ mockMoveFailureOnce(mockedScm,
MoveManager.MoveResult.REPLICATION_NOT_HEALTHY_AFTER_MOVE);
+
+ ContainerBalancerTask task =
mockedScm.startBalancerTask(buildConfig(mockedScm));
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+
+ assertThat(iteration.getContainerMovesFailed()).isEqualTo(1);
+ assertFailureBreakdown(mockedScm, iteration,
+ MoveManager.MoveResult.REPLICATION_NOT_HEALTHY_AFTER_MOVE.name(),
0);
+ }
+
+ @Test
+ void testDeletionTimeoutRecordedInFailureBreakdown()
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ mockMoveFailureOnce(mockedScm,
MoveManager.MoveResult.DELETION_FAIL_TIME_OUT);
+
+ ContainerBalancerTask task =
mockedScm.startBalancerTask(buildConfig(mockedScm));
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+
+ assertThat(iteration.getContainerMovesTimeout()).isEqualTo(1);
+ assertFailureBreakdown(mockedScm, iteration,
MoveManager.MoveResult.DELETION_FAIL_TIME_OUT.name(), 0);
+ }
+
+ @Test
+ void testIterationMoveTimeoutRecordedInFailureBreakdown()
+ throws NodeNotFoundException, ContainerNotFoundException,
ContainerReplicaNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ ContainerBalancerConfiguration config = buildConfig(mockedScm);
+ config.setMoveTimeout(Duration.ofMillis(50));
+ mockFirstMoveNeverCompletes(mockedScm);
+
+ ContainerBalancerTask task = mockedScm.startBalancerTask(config);
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+
+ assertThat(iteration.getContainerMovesTimeout()).isEqualTo(1);
+ assertFailureBreakdown(mockedScm, iteration,
+
ContainerBalancerTask.ContainerMoveFailureReason.ITERATION_MOVE_TIMEOUT.name(),
0);
+ }
+
+ @Test
+ void testFailureBreakdownTotalsMatchHeadlineCountersInControlledScenario()
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ when(mockedScm.getMoveManager().move(any(ContainerID.class),
+ any(DatanodeDetails.class), any(DatanodeDetails.class)))
+ .thenReturn(CompletableFuture.completedFuture(
+ MoveManager.MoveResult.REPLICATION_FAIL_TIME_OUT))
+ .thenReturn(CompletableFuture.completedFuture(
+ MoveManager.MoveResult.REPLICATION_NOT_HEALTHY_AFTER_MOVE))
+
.thenReturn(CompletableFuture.completedFuture(MoveManager.MoveResult.COMPLETED));
+
+ ContainerBalancerTask task =
mockedScm.startBalancerTask(buildConfig(mockedScm));
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+
+ assertThat(iteration.getContainerMovesTimeout()).isEqualTo(1);
+ assertThat(iteration.getContainerMovesFailed()).isEqualTo(1);
+ verify(mockedScm.getMoveManager(), atLeast(2)).move(
+ any(ContainerID.class), any(DatanodeDetails.class),
any(DatanodeDetails.class));
+ assertBreakdownTotalsMatchHeadlineCounters(iteration);
+ }
+
+ @Test
+ void testPreMoveContainerNotFoundRecordedInFailureBreakdown()
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ when(mockedScm.getMoveManager().move(any(ContainerID.class),
+ any(DatanodeDetails.class), any(DatanodeDetails.class)))
+ .thenThrow(ContainerNotFoundException.newInstanceForTesting())
+
.thenReturn(CompletableFuture.completedFuture(MoveManager.MoveResult.COMPLETED));
+
+ ContainerBalancerTask task =
mockedScm.startBalancerTask(buildConfig(mockedScm));
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+
+ assertThat(iteration.getContainerMovesFailed()).isEqualTo(1);
+ assertFailureBreakdown(mockedScm, iteration,
+
ContainerBalancerTask.ContainerMoveFailureReason.PRE_MOVE_CONTAINER_NOT_FOUND.name(),
0);
+ }
+
+ @Test
+ void testNodeNotFoundRecordedInFailureBreakdown()
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ when(mockedScm.getMoveManager().move(any(ContainerID.class),
+ any(DatanodeDetails.class), any(DatanodeDetails.class)))
+ .thenThrow(new NodeNotFoundException())
+
.thenReturn(CompletableFuture.completedFuture(MoveManager.MoveResult.COMPLETED));
+ ContainerBalancerTask task =
mockedScm.startBalancerTask(buildConfig(mockedScm));
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+ assertThat(iteration.getContainerMovesFailed()).isEqualTo(1);
+ assertFailureBreakdown(mockedScm, iteration,
+
ContainerBalancerTask.ContainerMoveFailureReason.PRE_MOVE_NODE_NOT_FOUND.name(),
0);
+ }
+
+ @Test
+ void testReplicaNotFoundRecordedInFailureBreakdown()
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ MockedSCM mockedScm = createMockedScm();
+ when(mockedScm.getMoveManager().move(any(ContainerID.class),
+ any(DatanodeDetails.class), any(DatanodeDetails.class)))
+ .thenThrow(new ContainerReplicaNotFoundException("test"))
+
.thenReturn(CompletableFuture.completedFuture(MoveManager.MoveResult.COMPLETED));
+ ContainerBalancerTask task =
mockedScm.startBalancerTask(buildConfig(mockedScm));
+ ContainerBalancerTaskIterationStatusInfo iteration =
getCompletedIteration(task);
+ assertThat(iteration.getContainerMovesFailed()).isEqualTo(1);
+ assertFailureBreakdown(mockedScm, iteration,
+
ContainerBalancerTask.ContainerMoveFailureReason.PRE_MOVE_REPLICA_NOT_FOUND.name(),
0);
+ }
+
+ @Test
+ void testSameReasonAggregatesSourceFailureCountsFromMultipleSources() {
+ ContainerMoveFailureTracker tracker = new ContainerMoveFailureTracker();
+ DatanodeDetails source1 =
MockDatanodeDetails.createDatanodeDetails("1.1.1.1", "/r1");
+ DatanodeDetails source2 =
MockDatanodeDetails.createDatanodeDetails("2.2.2.2", "/r1");
+ DatanodeDetails target =
MockDatanodeDetails.createDatanodeDetails("3.3.3.3", "/r2");
+
+ String reason = MoveManager.MoveResult.REPLICATION_FAIL_TIME_OUT.name();
+ tracker.recordFailure(reason, source1, target);
+ tracker.recordFailure(reason, source2, target);
+
+ ContainerMoveFailureDetail detail = tracker.getFailures().stream()
+ .filter(f -> reason.equals(f.getReason()))
+ .findFirst()
+ .orElse(null);
+ assertThat(detail).as("failure detail for reason " + reason).isNotNull();
+ assertThat(detail.getCount()).isEqualTo(2L);
+ assertThat(detail.getSourceFailureCounts())
+ .hasSize(2)
+ .containsEntry(source1.getUuidString(), 1L)
+ .containsEntry(source2.getUuidString(), 1L);
+ assertThat(detail.getTargetFailureCounts())
+ .hasSize(1)
+ .containsEntry(target.getUuidString(), 2L);
+ }
+
+ private static MockedSCM createMockedScm() {
+ return new MockedSCM(new MockCluster(NODE_COUNT, STORAGE_UNIT));
+ }
+
+ private static void mockMoveFailureOnce(MockedSCM mockedScm,
MoveManager.MoveResult failureResult)
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ when(mockedScm.getMoveManager().move(any(ContainerID.class),
+ any(DatanodeDetails.class), any(DatanodeDetails.class)))
+ .thenReturn(CompletableFuture.completedFuture(failureResult))
+
.thenReturn(CompletableFuture.completedFuture(MoveManager.MoveResult.COMPLETED));
+ }
+
+ private static void mockFirstMoveNeverCompletes(MockedSCM mockedScm)
+ throws NodeNotFoundException, ContainerReplicaNotFoundException,
ContainerNotFoundException {
+ AtomicInteger moveInvocations = new AtomicInteger(0);
+ when(mockedScm.getMoveManager().move(any(ContainerID.class),
+ any(DatanodeDetails.class), any(DatanodeDetails.class)))
+ .thenAnswer(invocation -> {
+ if (moveInvocations.getAndIncrement() == 0) {
+ return new CompletableFuture<>();
+ }
+ return
CompletableFuture.completedFuture(MoveManager.MoveResult.COMPLETED);
+ });
+ }
+
+ private static ContainerBalancerConfiguration buildConfig(MockedSCM
mockedScm) {
+ ContainerBalancerConfiguration config = new
ContainerBalancerConfigBuilder(mockedScm.getNodeCount()).build();
+ config.setMaxSizeToMovePerIteration(5 * STORAGE_UNIT);
+ config.setMaxSizeEnteringTarget(5 * STORAGE_UNIT);
+ config.setMaxDatanodesPercentageToInvolvePerIteration(100);
+ return config;
+ }
+
+ private static ContainerBalancerTaskIterationStatusInfo
getCompletedIteration(
+ ContainerBalancerTask task) {
+ List<ContainerBalancerTaskIterationStatusInfo> iterations =
+ task.getCurrentIterationsStatistic();
+ assertEquals(1, iterations.size());
+ ContainerBalancerTaskIterationStatusInfo iteration = iterations.get(0);
+ assertEquals("ITERATION_COMPLETED", iteration.getIterationResult());
+ return iteration;
+ }
+
+ private static void assertBreakdownTotalsMatchHeadlineCounters(
+ ContainerBalancerTaskIterationStatusInfo iteration) {
+ long breakdownTotal = iteration.getFailures().stream()
+ .mapToLong(ContainerMoveFailureDetail::getCount)
+ .sum();
+ long headlineTotal = iteration.getContainerMovesFailed() +
iteration.getContainerMovesTimeout();
+ assertEquals(headlineTotal, breakdownTotal,
+ "sum(failure breakdown) should equal failed + timeout counters");
+ }
+
+ private static void assertFailureBreakdown(MockedSCM mockedScm,
ContainerBalancerTaskIterationStatusInfo iteration,
+ String expectedReason, int moveIndex) throws NodeNotFoundException,
ContainerReplicaNotFoundException,
+ ContainerNotFoundException {
+ ArgumentCaptor<DatanodeDetails> sourceCaptor =
ArgumentCaptor.forClass(DatanodeDetails.class);
+ ArgumentCaptor<DatanodeDetails> targetCaptor =
ArgumentCaptor.forClass(DatanodeDetails.class);
+ verify(mockedScm.getMoveManager(), atLeastOnce()).move(
+ any(ContainerID.class), sourceCaptor.capture(),
targetCaptor.capture());
+ assertThat(sourceCaptor.getAllValues().size()).isGreaterThan(moveIndex);
+ assertThat(targetCaptor.getAllValues().size()).isGreaterThan(moveIndex);
+ String sourceUuid =
sourceCaptor.getAllValues().get(moveIndex).getUuidString();
+ String targetUuid =
targetCaptor.getAllValues().get(moveIndex).getUuidString();
+
+ List<ContainerMoveFailureDetail> failures = iteration.getFailures();
+ assertThat(failures).as("failure details").isNotEmpty();
+ ContainerMoveFailureDetail detail = failures.stream()
+ .filter(f -> expectedReason.equals(f.getReason()))
+ .findFirst()
+ .orElse(null);
+ assertThat(detail).as("failure detail for reason " +
expectedReason).isNotNull();
+ assertThat(detail.getCount()).isEqualTo(1L);
+
assertThat(detail.getSourceFailureCounts()).hasSize(1).containsEntry(sourceUuid,
1L);
+
assertThat(detail.getTargetFailureCounts()).hasSize(1).containsEntry(targetUuid,
1L);
+ }
+}
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
index a55fc7906db..1839c3915fb 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
@@ -27,6 +27,7 @@
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
+import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
@@ -34,6 +35,8 @@
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoResponseProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStatusInfoProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.NodeFailureCountProto;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.ozone.OzoneConsts;
import picocli.CommandLine;
@@ -232,6 +235,7 @@ private String
getPrettyIterationStatusInfo(ContainerBalancerTaskIterationStatus
if (leavingDataNodeList.isEmpty()) {
leavingDataNodeList = " -" + System.lineSeparator();
}
+ String failures = formatFailures(containerMovesFailed,
iterationStatusInfo.getContainerMoveFailuresList());
return String.format(
"%-50s %s%n" +
"%-50s %s%n" +
@@ -243,6 +247,7 @@ private String
getPrettyIterationStatusInfo(ContainerBalancerTaskIterationStatus
"%-50s %s%n" +
"%-50s %s%n" +
"%-50s %s%n" +
+ "%s" +
"%-50s %n%s" +
"%-50s %n%s",
"Key", "Value",
@@ -256,9 +261,47 @@ private String
getPrettyIterationStatusInfo(ContainerBalancerTaskIterationStatus
"Already moved containers", containerMovesCompleted,
"Failed to move containers", containerMovesFailed,
"Failed to move containers by timeout", containerMovesTimeout,
+ failures,
"Entered data to nodes", enteringDataNodeList,
"Exited data from nodes", leavingDataNodeList);
}
+ private String formatFailures(long containerMovesFailed,
List<ContainerMoveFailureDetailProto> failures) {
+ if (containerMovesFailed > 0 && failures.isEmpty()) {
+ return String.format("%-50s %s%n", "Failed container moves", "(no
breakdown available)");
+ }
+ if (failures.isEmpty()) {
+ return "";
+ }
+ List<ContainerMoveFailureDetailProto> sorted = failures.stream()
+
.sorted(Comparator.comparingLong(ContainerMoveFailureDetailProto::getCount).reversed()
+ .thenComparing(ContainerMoveFailureDetailProto::getReason))
+ .collect(Collectors.toList());
+ StringBuilder builder = new StringBuilder();
+ builder.append(String.format("%-50s %n", "Failed container moves"));
+ for (ContainerMoveFailureDetailProto failure : sorted) {
+ builder.append(String.format(" %-48s %d%n", failure.getReason(),
failure.getCount()));
+ if (!failure.getSourceFailureCountsList().isEmpty()) {
+ builder.append(String.format(" %-46s %n", "Source datanodes"));
+ for (NodeFailureCountProto src : failure.getSourceFailureCountsList())
{
+ builder.append(String.format(" %-44s %d%n",
formatDatanodeLabel(src), src.getCount()));
+ }
+ }
+ if (!failure.getTargetFailureCountsList().isEmpty()) {
+ builder.append(String.format(" %-46s %n", "Target datanodes"));
+ for (NodeFailureCountProto tgt : failure.getTargetFailureCountsList())
{
+ builder.append(String.format(" %-44s %d%n",
formatDatanodeLabel(tgt), tgt.getCount()));
+ }
+ }
+ }
+ return builder.toString();
+ }
+
+ private static String formatDatanodeLabel(NodeFailureCountProto node) {
+ return node.hasHostname()
+ ? node.getHostname() + " (" + node.getDatanodeUuid() + ")"
+ : node.getDatanodeUuid();
+ }
+
}
diff --git
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
index 407785d48a3..5eca86a8a6c 100644
---
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
+++
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
@@ -712,4 +712,113 @@ void
testContainerBalancerStatusSubcommandStoppedAfterAllIterationsCompleteVerbo
.contains(ITERATION_2_COMPLETED_OUTPUT)
.doesNotContain(ITERATION_3_COMPLETED_OUTPUT);
}
+
+ @Test
+ void testContainerBalancerStatusVerboseShowsFailureBreakdown() throws
IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ ContainerBalancerConfiguration config =
getContainerBalancerConfiguration();
+
StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStatusInfoProto
iteration =
+
StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStatusInfoProto.newBuilder()
+ .setIterationNumber(1)
+ .setIterationResult("ITERATION_COMPLETED")
+ .setIterationDuration(120L)
+ .setContainerMovesScheduled(5)
+ .setContainerMovesCompleted(3)
+ .setContainerMovesFailed(1)
+ .setContainerMovesTimeout(2)
+ .addContainerMoveFailures(
+
StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto.newBuilder()
+ .setReason("REPLICATION_FAIL_TIME_OUT")
+ .setCount(2)
+ .addSourceFailureCounts(
+
StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+
.setDatanodeUuid("source-uuid-1").setHostname("datanode1.example.com").setCount(1).build())
+ .addSourceFailureCounts(
+
StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+
.setDatanodeUuid("source-uuid-2").setCount(1).build())
+ .addTargetFailureCounts(
+
StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+
.setDatanodeUuid("target-uuid-1").setHostname("datanode3.example.com").setCount(1).build())
+ .addTargetFailureCounts(
+
StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+
.setDatanodeUuid("target-uuid-2").setHostname("datanode4.example.com").setCount(1).build())
+ .build())
+ .addContainerMoveFailures(
+
StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto.newBuilder()
+ .setReason("PRE_MOVE_CONTAINER_NOT_FOUND")
+ .setCount(1)
+ .addSourceFailureCounts(
+
StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+
.setDatanodeUuid("source-uuid-3").setHostname("datanode5.example.com").setCount(1).build())
+ .addTargetFailureCounts(
+
StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+
.setDatanodeUuid("target-uuid-3").setHostname("datanode6.example.com").setCount(1).build())
+ .build())
+ .build();
+
+ long stoppedAt = OffsetDateTime.now().toEpochSecond();
+ ContainerBalancerStatusInfoProto statusInfo =
ContainerBalancerStatusInfoProto.newBuilder()
+ .setStartedAt(stoppedAt - 600)
+ .setStoppedAt(stoppedAt)
+ .setStopReason("USER_REQUESTED")
+ .setStopMessage("Stopped by user request.")
+ .setConfiguration(config.toProtobufBuilder().setShouldRun(false))
+ .addIterationsStatusInfo(iteration)
+ .build();
+
+ when(scmClient.getContainerBalancerStatusInfo())
+ .thenReturn(ContainerBalancerStatusInfoResponseProto.newBuilder()
+ .setIsRunning(false)
+ .setContainerBalancerStatusInfo(statusInfo)
+ .build());
+
+ verbose.set(true);
+ statusCmd.execute(scmClient);
+
+ assertThat(out.get())
+ .contains("Failed container moves")
+ .contains("REPLICATION_FAIL_TIME_OUT")
+ .contains("PRE_MOVE_CONTAINER_NOT_FOUND")
+ .contains("datanode1.example.com (source-uuid-1)")
+ .contains("source-uuid-2")
+ .doesNotContain("(source-uuid-2)")
+ .contains("datanode3.example.com (target-uuid-1)")
+ .contains("datanode5.example.com (source-uuid-3)")
+ .contains("datanode6.example.com (target-uuid-3)");
+ }
+
+ @Test
+ void testContainerBalancerStatusVerboseShowsNoBreakdownWhenFailuresMissing()
throws IOException {
+ ScmClient scmClient = mock(ScmClient.class);
+ ContainerBalancerConfiguration config =
getContainerBalancerConfiguration();
+
StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStatusInfoProto
iteration =
+
StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStatusInfoProto.newBuilder()
+ .setIterationNumber(1)
+ .setIterationResult("ITERATION_COMPLETED")
+ .setIterationDuration(120L)
+ .setContainerMovesFailed(3)
+ .build();
+
+ long stoppedAt = OffsetDateTime.now().toEpochSecond();
+ ContainerBalancerStatusInfoProto statusInfo =
ContainerBalancerStatusInfoProto.newBuilder()
+ .setStartedAt(stoppedAt - 600)
+ .setStoppedAt(stoppedAt)
+ .setStopReason("USER_REQUESTED")
+ .setConfiguration(config.toProtobufBuilder().setShouldRun(false))
+ .addIterationsStatusInfo(iteration)
+ .build();
+
+ when(scmClient.getContainerBalancerStatusInfo())
+ .thenReturn(ContainerBalancerStatusInfoResponseProto.newBuilder()
+ .setIsRunning(false)
+ .setContainerBalancerStatusInfo(statusInfo)
+ .build());
+
+ verbose.set(true);
+ statusCmd.execute(scmClient);
+
+ assertThat(out.get())
+ .contains("Failed to move containers 3")
+ .contains("Failed container moves (no
breakdown available)");
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
