ashishkumar50 commented on code in PR #10938:
URL: https://github.com/apache/ozone/pull/10938#discussion_r3723656640
##########
hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto:
##########
@@ -675,6 +675,20 @@ message ContainerBalancerTaskIterationStatusInfoProto {
repeated NodeTransferInfoProto sizeEnteringNodes = 9;
repeated NodeTransferInfoProto sizeLeavingNodes = 10;
optional int64 iterationDuration = 11;
+ repeated ContainerMoveFailureSummaryProto containerMoveFailuresByReason = 12;
+ repeated ContainerMoveFailureDetailProto containerMoveFailureDetails = 13;
+}
+
+message ContainerMoveFailureSummaryProto {
+ optional string reason = 1;
+ optional int64 count = 2;
+}
+
+message ContainerMoveFailureDetailProto {
Review Comment:
I think `Failure breakdown `and `Failed move details` should be merged. Also
instead of showing containerids, we should show count of failures for source
and target, otherwise it will be too verbose. Issue generally happens at DN
level(like slow/unhealthy etc), so count is enough to know which dn is
problematic.
##########
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java:
##########
@@ -256,9 +269,75 @@ private String
getPrettyIterationStatusInfo(ContainerBalancerTaskIterationStatus
"Already moved containers", containerMovesCompleted,
"Failed to move containers", containerMovesFailed,
"Failed to move containers by timeout", containerMovesTimeout,
+ failureBreakdown,
+ failureDetails,
"Entered data to nodes", enteringDataNodeList,
"Exited data from nodes", leavingDataNodeList);
}
+ private String formatFailureBreakdown(List<ContainerMoveFailureSummaryProto>
summaries) {
+ List<ContainerMoveFailureSummaryProto> sortedSummaries = summaries.stream()
Review Comment:
The proto already sorts failure summaries, then the CLI sorts them again
which is not required. Remove from server side sort.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureTracker.java:
##########
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+
+/**
+ * Tracks per-iteration container move failures by reason and optional details.
+ */
+public final class ContainerMoveFailureTracker {
+ public static final int DEFAULT_MAX_FAILURE_DETAILS = 100;
+ public static final int DEFAULT_MAX_FAILURE_DETAILS_PER_REASON = 10;
Review Comment:
Please update as per dn count suggestion and remove these limits.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureTracker.java:
##########
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+
+/**
+ * Tracks per-iteration container move failures by reason and optional details.
+ */
+public final class ContainerMoveFailureTracker {
+ public static final int DEFAULT_MAX_FAILURE_DETAILS = 100;
+ public static final int DEFAULT_MAX_FAILURE_DETAILS_PER_REASON = 10;
+
+ private final int maxFailureDetails;
+ private final int maxFailureDetailsPerReason;
+ private final Map<String, Long> failuresByReason = new HashMap<>();
Review Comment:
Use enum FailureReason instead of plain string.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureTracker.java:
##########
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+
+/**
+ * Tracks per-iteration container move failures by reason and optional details.
+ */
+public final class ContainerMoveFailureTracker {
+ public static final int DEFAULT_MAX_FAILURE_DETAILS = 100;
+ public static final int DEFAULT_MAX_FAILURE_DETAILS_PER_REASON = 10;
+
+ private final int maxFailureDetails;
+ private final int maxFailureDetailsPerReason;
+ private final Map<String, Long> failuresByReason = new HashMap<>();
+ private final Map<String, Integer> failureDetailCountByReason = new
HashMap<>();
+ private final List<ContainerMoveFailureDetail> failureDetails = new
ArrayList<>();
+
+ public ContainerMoveFailureTracker() {
+ this(DEFAULT_MAX_FAILURE_DETAILS, DEFAULT_MAX_FAILURE_DETAILS_PER_REASON);
+ }
+
+ public ContainerMoveFailureTracker(int maxFailureDetails, int
maxFailureDetailsPerReason) {
+ this.maxFailureDetails = maxFailureDetails;
+ this.maxFailureDetailsPerReason = Math.min(maxFailureDetailsPerReason,
maxFailureDetails);
+ }
+
+ public synchronized void recordFailure(MoveManager.MoveResult result,
ContainerID containerId,
+ DatanodeDetails source, DatanodeDetails target) {
+ recordFailure(result.name(), containerId, source, target);
+ }
+
+ public synchronized void recordFailure(String reason, ContainerID
containerId,
+ DatanodeDetails source, DatanodeDetails target) {
+ failuresByReason.merge(reason, 1L, Long::sum);
+ if (shouldRecordFailureDetail(reason)) {
+ failureDetails.add(new ContainerMoveFailureDetail(
+ containerId.getId(),
+ source.getUuidString(),
+ target.getUuidString(),
Review Comment:
Source or target can be null.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
