sreejasahithi commented on code in PR #11420: URL: https://github.com/apache/ozone/pull/11420#discussion_r4295410383
########## hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/storagepolicy/UsageInfoSubCommand.java: ########## @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli.storagepolicy; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.client.StorageTypeUtils; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DatanodeStorageTypeUsageInfoProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListStorageTypeUsageInfoRequestProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StorageTypeUsageInfoProto; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.hdds.server.JsonUtils; +import org.apache.hadoop.util.StringUtils; +import picocli.CommandLine; + +/** + * Handler of list storage usage info by StorageType command. + */ [email protected]( + name = "usageinfo", + description = "List usage information in the StorageType dimension", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class UsageInfoSubCommand extends ScmSubcommand { + + @CommandLine.Option( + names = {"-d", "--with-datanode"}, + description = "Print detailed per-datanode storage-type usage", + defaultValue = "false") + private boolean printDatanodeInfo; + + @CommandLine.Option( + names = {"-o", "--operational-state"}, + description = "Show datanodes in a specific operational state " + + "(IN_SERVICE, DECOMMISSIONING, DECOMMISSIONED, " + + "ENTERING_MAINTENANCE, IN_MAINTENANCE), " + + "or ALL to show all operational states", + defaultValue = "IN_SERVICE") + private String nodeOperationalStateStr; + + @CommandLine.Option( + names = {"-n", "--node-state"}, + description = "Show datanodes in a specific health state " + + "(HEALTHY, STALE, DEAD), or ALL to show all health states", + defaultValue = "HEALTHY") + private String nodeStateStr; + + @CommandLine.Option( + names = {"--json"}, + defaultValue = "false", + description = "Format output as JSON") + private boolean json; + + @Override + public void execute(ScmClient scmClient) throws IOException { + NodeOperationalState nodeOpState = null; + NodeState nodeState = null; + if (!nodeOperationalStateStr.equalsIgnoreCase("ALL")) { + nodeOpState = NodeOperationalState.valueOf(nodeOperationalStateStr.toUpperCase()); + } + if (!nodeStateStr.equalsIgnoreCase("ALL")) { + nodeState = NodeState.valueOf(nodeStateStr.toUpperCase()); + } + + ListStorageTypeUsageInfoRequestProto.Builder reqBuilder = ListStorageTypeUsageInfoRequestProto.newBuilder(); + if (nodeOpState != null) { + reqBuilder.setOpState(nodeOpState); + } + if (nodeState != null) { + reqBuilder.setState(nodeState); + } + + List<DatanodeStorageTypeUsageInfoProto> usageInfos = scmClient.listStorageTypeUsageInfo(reqBuilder.build()); + + List<ClusterStorageTypeUsage> summary = aggregateClusterUsage(usageInfos); + List<DatanodeStorageTypeUsage> datanodeUsage = printDatanodeInfo ? buildDatanodeUsageList(usageInfos) : null; + + if (json) { + UsageInfoOutput output = new UsageInfoOutput(summary, datanodeUsage); + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(output)); + return; + } + + printStorageTypeSummaryInfo(summary); + if (datanodeUsage != null) { + printStorageTypeDatanodeInfo(datanodeUsage); + } + } + + /** + * Sums capacity/used/remaining/committed/freeSpaceToSpare across all returned datanodes, per StorageType. + * Uses Math.addExact since this sums values across multiple datanodes and must not silently overflow. + */ + private List<ClusterStorageTypeUsage> aggregateClusterUsage(List<DatanodeStorageTypeUsageInfoProto> dnInfos) { + Map<StorageType, ClusterStorageTypeUsage> totals = new EnumMap<>(StorageType.class); + for (DatanodeStorageTypeUsageInfoProto dn : dnInfos) { + for (StorageTypeUsageInfoProto st : dn.getStorageTypeUsageInfoList()) { + StorageType storageType = StorageTypeUtils.getFromProtobuf(st.getStorageType()); + ClusterStorageTypeUsage usage = totals.computeIfAbsent(storageType, ClusterStorageTypeUsage::new); + usage.datanodeCount++; + usage.capacity = Math.addExact(usage.capacity, st.getCapacity()); + usage.used = Math.addExact(usage.used, st.getUsed()); + usage.remaining = Math.addExact(usage.remaining, st.getRemaining()); + usage.committed = Math.addExact(usage.committed, st.getCommitted()); + usage.freeSpaceToSpare = Math.addExact(usage.freeSpaceToSpare, st.getFreeSpaceToSpare()); + } + } + List<ClusterStorageTypeUsage> summary = new ArrayList<>(totals.values()); + summary.sort((a, b) -> a.getStorageType().compareTo(b.getStorageType())); + return summary; + } + + private List<DatanodeStorageTypeUsage> buildDatanodeUsageList(List<DatanodeStorageTypeUsageInfoProto> dnInfos) { + List<DatanodeStorageTypeUsage> datanodeUsage = new ArrayList<>(); + for (DatanodeStorageTypeUsageInfoProto dn : dnInfos) { + for (StorageTypeUsageInfoProto st : dn.getStorageTypeUsageInfoList()) { + datanodeUsage.add(new DatanodeStorageTypeUsage(dn, st)); + } + } + return datanodeUsage; + } + + private void printStorageTypeSummaryInfo(List<ClusterStorageTypeUsage> summary) { + System.out.println("Cluster StorageType Usage Summary"); + for (ClusterStorageTypeUsage usage : summary) { + StorageType storageType = usage.getStorageType(); + System.out.printf(" %-23s: %s %n", storageType + " Datanode Count", usage.getDatanodeCount()); + System.out.printf(" %-23s: %s (%s) %n", storageType + " Capacity", + usage.getCapacity() + " B", StringUtils.byteDesc(usage.getCapacity())); + System.out.printf(" %-23s: %s (%s) %n", storageType + " Ozone Used", + usage.getUsed() + " B", StringUtils.byteDesc(usage.getUsed())); + System.out.printf(" %-23s: %s (%s) %n", storageType + " Remaining", + usage.getRemaining() + " B", StringUtils.byteDesc(usage.getRemaining())); + System.out.printf(" %-23s: %s (%s) %n", storageType + " Committed", + usage.getCommitted() + " B", StringUtils.byteDesc(usage.getCommitted())); + System.out.printf(" %-23s: %s (%s) %n", storageType + " FreeSpaceToSpare", + usage.getFreeSpaceToSpare() + " B", StringUtils.byteDesc(usage.getFreeSpaceToSpare())); + System.out.println(); + } + } + + private void printStorageTypeDatanodeInfo(List<DatanodeStorageTypeUsage> datanodeUsage) { Review Comment: With --with-datanode, printStorageTypeDatanodeInfo emits a StorageType X: header whenever the type changes from the previous row. buildDatanodeUsageList walks each datanode, then each type, so row order is dn1/DISK, dn1/SSD, dn2/DISK, dn2/SSD, … When two or more datanodes each have multiple types (SCM only includes types with capacity > 0), the type column alternates (DISK -> SSD -> DISK -> SSD), so StorageType DISK: can appear more than once instead of one DISK block for all datanodes, then one SSD block. DISK-only clusters (e.g compose) don’t show this because the type never changes between consecutive rows. Fix: Before printing, sort or group by StorageType, then datanode (sort in buildDatanodeUsageList, or Map<StorageType, List<…>> in printStorageTypeDatanodeInfo). ########## hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/storagepolicy/TestUsageInfoSubCommand.java: ########## Review Comment: In TestUsageInfoSubCommand: mock ScmClient.listStorageTypeUsageInfo with two protos, each with DISK+SSD in storageTypeUsageInfo, run CommandLine with --with-datanode (no --json), assert StorageType DISK: and StorageType SSD: each appear once. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
