errose28 commented on a change in pull request #3042: URL: https://github.com/apache/ozone/pull/3042#discussion_r799889435
########## File path: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java ########## @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.keyvalue; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.utils.MetadataKeyFilters; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; +import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; +import org.apache.hadoop.ozone.container.common.interfaces.BlockIterator; +import org.apache.hadoop.ozone.container.common.interfaces.ContainerInspector; +import org.apache.hadoop.ozone.container.metadata.DatanodeStore; +import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; + +/** + * Container inspector for key value container metadata. It is capable of + * logging metadata information about a container, and repairing the metadata + * database values of #BLOCKCOUNT and #BYTESUSED. + */ +public class KeyValueContainerMetadataInspector implements ContainerInspector { + public static final Logger LOG = + LoggerFactory.getLogger(KeyValueContainerMetadataInspector.class); + + /** + * The mode to run the inspector in. + */ + public enum Mode { + REPAIR("repair"), + INSPECT("inspect"), + OFF("off"); + + private final String name; + + Mode(String name) { + this.name = name; + } + + public String toString() { + return name; + } + } + + public static final String SYSTEM_PROPERTY = "ozone.datanode.container" + + ".metadata.inspector"; + + private Mode mode; + + public KeyValueContainerMetadataInspector() { + mode = Mode.OFF; + } + + /** + * Validate configuration here so that an invalid config value is only + * logged once, and not once per container. + */ + @Override + public boolean load() { + String propertyValue = System.getProperty(SYSTEM_PROPERTY); + boolean propertySet = false; + + if (propertyValue != null && !propertyValue.isEmpty()) { + if (propertyValue.equals(Mode.REPAIR.toString())) { + mode = Mode.REPAIR; + propertySet = true; + } else if (propertyValue.equals(Mode.INSPECT.toString())) { + mode = Mode.INSPECT; + propertySet = true; + } else { + LOG.error("{} system property specified with invalid mode {}. " + + "Valid options are {} and {}. Container metadata inspection " + + "will not be run.", SYSTEM_PROPERTY, propertyValue, + Mode.REPAIR, Mode.INSPECT); + } + } + + if (!propertySet) { + mode = Mode.OFF; + } + + return propertySet; + } + + @Override + public void unload() { + mode = Mode.OFF; + } + + @Override + public boolean isReadOnly() { + return mode != Mode.REPAIR; + } + + @Override + public void process(ContainerData containerData, DatanodeStore store) { + // If the system property to process container metadata was not + // specified, or the inspector is unloaded, this method is a no-op. + if (mode == Mode.OFF) { + return; + } + + StringBuilder messageBuilder = new StringBuilder(); + boolean passed = false; + + try { + messageBuilder.append(String.format("Audit of container %d metadata%n", + containerData.getContainerID())); + + // Read metadata values. + Table<String, Long> metadataTable = store.getMetadataTable(); + Long blockCount = getAndAppend(metadataTable, + OzoneConsts.BLOCK_COUNT, messageBuilder); + Long bytesUsed = getAndAppend(metadataTable, + OzoneConsts.CONTAINER_BYTES_USED, messageBuilder); + // No repair action taken on these values. + // Pending delete blocks could be repaired from the DB values in the + // future if necessary. + getAndAppend(metadataTable, + OzoneConsts.PENDING_DELETE_BLOCK_COUNT, messageBuilder); + getAndAppend(metadataTable, + OzoneConsts.DELETE_TRANSACTION_KEY, messageBuilder); + getAndAppend(metadataTable, + OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID, messageBuilder); + + // Count number of block keys and total bytes used in the DB. + long usedBytesTotal = 0; + long blockCountTotal = 0; + long pendingDeleteBlockCountTotal = 0; + // Count normal blocks. + try (BlockIterator<BlockData> blockIter = + store.getBlockIterator( + MetadataKeyFilters.getUnprefixedKeyFilter())) { + + while (blockIter.hasNext()) { + blockCountTotal++; + usedBytesTotal += getBlockLength(blockIter.nextBlock()); + } + } + + String schemaVersion = + ((KeyValueContainerData) containerData).getSchemaVersion(); + + // Count pending delete blocks. + if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) { + try (BlockIterator<BlockData> blockIter = + store.getBlockIterator( + MetadataKeyFilters.getDeletingKeyFilter())) { + + while (blockIter.hasNext()) { + blockCountTotal++; + pendingDeleteBlockCountTotal++; + usedBytesTotal += getBlockLength(blockIter.nextBlock()); + } + } + } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V2)) { + DatanodeStoreSchemaTwoImpl schemaTwoStore = + (DatanodeStoreSchemaTwoImpl) store; + pendingDeleteBlockCountTotal = + countPendingDeletesSchemaV2(schemaTwoStore); + } else { + messageBuilder.append("Cannot process deleted blocks for unknown " + + "container schema ").append(schemaVersion); + } + + // Count number of files in chunks directory. + countFilesAndAppend(containerData.getChunksPath(), messageBuilder); + + messageBuilder.append("Container state: ") + .append(containerData.getState()).append("\n"); + messageBuilder.append("Schema Version: ") + .append(schemaVersion).append("\n"); + messageBuilder.append("Total block keys in DB: ").append(blockCountTotal) + .append("\n"); + messageBuilder.append("Total used bytes in DB: ").append(usedBytesTotal) + .append("\n"); + messageBuilder.append("Total pending delete block keys in DB: ") + .append(pendingDeleteBlockCountTotal).append("\n"); + + boolean blockCountPassed = + checkMetadataMatchAndAppend(metadataTable, OzoneConsts.BLOCK_COUNT, + blockCount, + blockCountTotal, messageBuilder); + boolean bytesUsedPassed = + checkMetadataMatchAndAppend(metadataTable, + OzoneConsts.CONTAINER_BYTES_USED, + bytesUsed, usedBytesTotal, messageBuilder); + passed = blockCountPassed && bytesUsedPassed; + } catch(IOException ex) { + LOG.error("Inspecting container {} failed", + containerData.getContainerID(), ex); + } + + if (passed) { + LOG.trace(messageBuilder.toString()); + } else { + LOG.error(messageBuilder.toString()); + } Review comment: Yeah I think that's a good idea. What are your thoughts on: 1. Separate file for each container vs. one file for all containers? 2. Json output vs. unstructured output? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org For additional commands, e-mail: issues-h...@ozone.apache.org
