http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java deleted file mode 100644 index 9de84da..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.interfaces; - -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; - -/** - * Chunk Manager allows read, write, delete and listing of chunks in - * a container. - */ -public interface ChunkManager { - - /** - * writes a given chunk. - * @param blockID - ID of the block. - * @param info - ChunkInfo. - * @param stage - Chunk Stage write. - * @throws StorageContainerException - */ - void writeChunk(BlockID blockID, - ChunkInfo info, byte[] data, ContainerProtos.Stage stage) - throws StorageContainerException; - - /** - * reads the data defined by a chunk. - * @param blockID - ID of the block. - * @param info - ChunkInfo. - * @return byte array - * @throws StorageContainerException - * - * TODO: Right now we do not support partial reads and writes of chunks. - * TODO: Explore if we need to do that for ozone. - */ - byte[] readChunk(BlockID blockID, ChunkInfo info) throws - StorageContainerException; - - /** - * Deletes a given chunk. - * @param blockID - ID of the block. - * @param info - Chunk Info - * @throws StorageContainerException - */ - void deleteChunk(BlockID blockID, ChunkInfo info) throws - StorageContainerException; - - // TODO : Support list operations. - - /** - * Shutdown the chunkManager. - */ - void shutdown(); - -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java index a5559aa..f0f1b37 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.container.common.interfaces; - import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerLifeCycleState; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java index 1ed50fb..2538368 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.container.common.interfaces; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; import java.util.List; import java.util.Map; @@ -28,6 +28,7 @@ import java.util.Map; * This interface is used for choosing desired containers for * block deletion. */ +// TODO: Fix ContainerDeletionChoosingPolicy to work with new StorageLayer public interface ContainerDeletionChoosingPolicy { /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java deleted file mode 100644 index 49b68dc..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java +++ /dev/null @@ -1,267 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.interfaces; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.hdfs.server.datanode.StorageLocation; -import org.apache.hadoop.hdfs.util.RwLock; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.NodeReportProto; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; - -import java.io.IOException; -import java.security.NoSuchAlgorithmException; -import java.util.List; - -/** - * Interface for container operations. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public interface ContainerManager extends RwLock { - - /** - * Init call that sets up a container Manager. - * - * @param config - Configuration. - * @param containerDirs - List of Metadata Container locations. - * @param datanodeDetails - DatanodeDetails - * @throws StorageContainerException - */ - void init(Configuration config, List<StorageLocation> containerDirs, - DatanodeDetails datanodeDetails) throws IOException; - - /** - * Creates a container with the given name. - * - * @param containerData - Container Name and metadata. - * @throws StorageContainerException - */ - void createContainer(ContainerData containerData) - throws StorageContainerException; - - /** - * Deletes an existing container. - * - * @param containerID - ID of the container. - * @param forceDelete - whether this container should be deleted forcibly. - * @throws StorageContainerException - */ - void deleteContainer(long containerID, - boolean forceDelete) throws StorageContainerException; - - /** - * Update an existing container. - * - * @param containerID ID of the container - * @param data container data - * @param forceUpdate if true, update container forcibly. - * @throws StorageContainerException - */ - void updateContainer(long containerID, ContainerData data, - boolean forceUpdate) throws StorageContainerException; - - /** - * As simple interface for container Iterations. - * - * @param startContainerID - Return containers with ID >= startContainerID. - * @param count - how many to return - * @param data - Actual containerData - * @throws StorageContainerException - */ - void listContainer(long startContainerID, long count, - List<ContainerData> data) throws StorageContainerException; - - /** - * Choose containers for block deletion. - * - * @param count - how many to return - * @throws StorageContainerException - */ - List<ContainerData> chooseContainerForBlockDeletion(int count) - throws StorageContainerException; - - /** - * Get metadata about a specific container. - * - * @param containerID - ID of the container. - * @return ContainerData - Container Data. - * @throws StorageContainerException - */ - ContainerData readContainer(long containerID) - throws StorageContainerException; - - /** - * Closes a open container, if it is already closed or does not exist a - * StorageContainerException is thrown. - * @param containerID - ID of the container. - * @throws StorageContainerException - */ - void closeContainer(long containerID) - throws StorageContainerException, NoSuchAlgorithmException; - - /** - * Checks if a container exists. - * @param containerID - ID of the container. - * @return true if the container is open false otherwise. - * @throws StorageContainerException - Throws Exception if we are not - * able to find the container. - */ - boolean isOpen(long containerID) throws StorageContainerException; - - /** - * Supports clean shutdown of container. - * - * @throws StorageContainerException - */ - void shutdown() throws IOException; - - /** - * Sets the Chunk Manager. - * - * @param chunkManager - ChunkManager. - */ - void setChunkManager(ChunkManager chunkManager); - - /** - * Gets the Chunk Manager. - * - * @return ChunkManager. - */ - ChunkManager getChunkManager(); - - /** - * Sets the Key Manager. - * - * @param keyManager - Key Manager. - */ - void setKeyManager(KeyManager keyManager); - - /** - * Gets the Key Manager. - * - * @return KeyManager. - */ - KeyManager getKeyManager(); - - /** - * Get the Node Report of container storage usage. - * @return node report. - */ - NodeReportProto getNodeReport() throws IOException; - - /** - * Gets container report. - * @return container report. - * @throws IOException - */ - ContainerReportsProto getContainerReport() throws IOException; - - /** - * Gets container reports. - * @return List of all closed containers. - * @throws IOException - */ - List<ContainerData> getClosedContainerReports() throws IOException; - - /** - * Increase pending deletion blocks count number of specified container. - * - * @param numBlocks - * increment count number - * @param containerId - * container id - */ - void incrPendingDeletionBlocks(int numBlocks, long containerId); - - /** - * Decrease pending deletion blocks count number of specified container. - * - * @param numBlocks - * decrement count number - * @param containerId - * container id - */ - void decrPendingDeletionBlocks(int numBlocks, long containerId); - - /** - * Increase the read count of the container. - * @param containerId - ID of the container. - */ - void incrReadCount(long containerId); - - /** - * Increse the read counter for bytes read from the container. - * @param containerId - ID of the container. - * @param readBytes - bytes read from the container. - */ - void incrReadBytes(long containerId, long readBytes); - - - /** - * Increase the write count of the container. - * @param containerId - ID of the container. - */ - void incrWriteCount(long containerId); - - /** - * Increase the write counter for bytes write into the container. - * @param containerId - ID of the container. - * @param writeBytes - bytes write into the container. - */ - void incrWriteBytes(long containerId, long writeBytes); - - /** - * Increase the bytes used by the container. - * @param containerId - ID of the container. - * @param used - additional bytes used by the container. - * @return the current bytes used. - */ - long incrBytesUsed(long containerId, long used); - - /** - * Decrease the bytes used by the container. - * @param containerId - ID of the container. - * @param used - additional bytes reclaimed by the container. - * @return the current bytes used. - */ - long decrBytesUsed(long containerId, long used); - - /** - * Get the bytes used by the container. - * @param containerId - ID of the container. - * @return the current bytes used by the container. - */ - long getBytesUsed(long containerId); - - /** - * Get the number of keys in the container. - * @param containerId - ID of the container. - * @return the current key count. - */ - long getNumKeys(long containerId); - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java deleted file mode 100644 index 158ce38..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.interfaces; - -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; - -import java.io.IOException; -import java.util.List; - -/** - * KeyManager deals with Key Operations in the container Level. - */ -public interface KeyManager { - /** - * Puts or overwrites a key. - * - * @param data - Key Data. - * @throws IOException - */ - void putKey(KeyData data) throws IOException; - - /** - * Gets an existing key. - * - * @param data - Key Data. - * @return Key Data. - * @throws IOException - */ - KeyData getKey(KeyData data) throws IOException; - - /** - * Deletes an existing Key. - * - * @param blockID - ID of the block. - * @throws StorageContainerException - */ - void deleteKey(BlockID blockID) - throws IOException; - - /** - * List keys in a container. - * - * @param containerID - ID of the container. - * @param startLocalID - Key to start from, 0 to begin. - * @param count - Number of keys to return. - * @return List of Keys that match the criteria. - */ - List<KeyData> listKey(long containerID, long startLocalID, - int count) throws IOException; - - /** - * Shutdown keyManager. - */ - void shutdown(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java deleted file mode 100644 index 52cf2e0..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java +++ /dev/null @@ -1,247 +0,0 @@ - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.container.common.statemachine.background; - -import com.google.common.collect.Lists; -import org.apache.ratis.shaded.com.google.protobuf - .InvalidProtocolBufferException; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.common.helpers.ContainerData; -import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.helpers.KeyUtils; -import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.utils.BackgroundService; -import org.apache.hadoop.utils.BackgroundTask; -import org.apache.hadoop.utils.BackgroundTaskQueue; -import org.apache.hadoop.utils.BackgroundTaskResult; -import org.apache.hadoop.utils.BatchOperation; -import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter; -import org.apache.hadoop.utils.MetadataStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT; - -/** - * A per-datanode container block deleting service takes in charge - * of deleting staled ozone blocks. - */ -public class BlockDeletingService extends BackgroundService{ - - private static final Logger LOG = - LoggerFactory.getLogger(BlockDeletingService.class); - - private final ContainerManager containerManager; - private final Configuration conf; - - // Throttle number of blocks to delete per task, - // set to 1 for testing - private final int blockLimitPerTask; - - // Throttle the number of containers to process concurrently at a time, - private final int containerLimitPerInterval; - - // Task priority is useful when a to-delete block has weight. - private final static int TASK_PRIORITY_DEFAULT = 1; - // Core pool size for container tasks - private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10; - - public BlockDeletingService(ContainerManager containerManager, - long serviceInterval, long serviceTimeout, TimeUnit unit, - Configuration conf) { - super("BlockDeletingService", serviceInterval, unit, - BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout); - this.containerManager = containerManager; - this.conf = conf; - this.blockLimitPerTask = conf.getInt( - OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, - OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT); - this.containerLimitPerInterval = conf.getInt( - OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, - OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT); - } - - - @Override - public BackgroundTaskQueue getTasks() { - BackgroundTaskQueue queue = new BackgroundTaskQueue(); - List<ContainerData> containers = Lists.newArrayList(); - try { - // We at most list a number of containers a time, - // in case there are too many containers and start too many workers. - // We must ensure there is no empty container in this result. - // The chosen result depends on what container deletion policy is - // configured. - containers = containerManager.chooseContainerForBlockDeletion( - containerLimitPerInterval); - LOG.info("Plan to choose {} containers for block deletion, " - + "actually returns {} valid containers.", - containerLimitPerInterval, containers.size()); - - for(ContainerData container : containers) { - BlockDeletingTask containerTask = - new BlockDeletingTask(container, TASK_PRIORITY_DEFAULT); - queue.add(containerTask); - } - } catch (StorageContainerException e) { - LOG.warn("Failed to initiate block deleting tasks, " - + "caused by unable to get containers info. " - + "Retry in next interval. ", e); - } catch (Exception e) { - // In case listContainer call throws any uncaught RuntimeException. - if (LOG.isDebugEnabled()) { - LOG.debug("Unexpected error occurs during deleting blocks.", e); - } - } - return queue; - } - - private static class ContainerBackgroundTaskResult - implements BackgroundTaskResult { - private List<String> deletedBlockIds; - - ContainerBackgroundTaskResult() { - deletedBlockIds = new LinkedList<>(); - } - - public void addBlockId(String blockId) { - deletedBlockIds.add(blockId); - } - - public void addAll(List<String> blockIds) { - deletedBlockIds.addAll(blockIds); - } - - public List<String> getDeletedBlocks() { - return deletedBlockIds; - } - - @Override - public int getSize() { - return deletedBlockIds.size(); - } - } - - private class BlockDeletingTask - implements BackgroundTask<BackgroundTaskResult> { - - private final int priority; - private final ContainerData containerData; - - BlockDeletingTask(ContainerData containerName, int priority) { - this.priority = priority; - this.containerData = containerName; - } - - @Override - public BackgroundTaskResult call() throws Exception { - ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult(); - long startTime = Time.monotonicNow(); - // Scan container's db and get list of under deletion blocks - MetadataStore meta = KeyUtils.getDB(containerData, conf); - // # of blocks to delete is throttled - KeyPrefixFilter filter = - new KeyPrefixFilter().addFilter(OzoneConsts.DELETING_KEY_PREFIX); - List<Map.Entry<byte[], byte[]>> toDeleteBlocks = - meta.getSequentialRangeKVs(null, blockLimitPerTask, filter); - if (toDeleteBlocks.isEmpty()) { - LOG.debug("No under deletion block found in container : {}", - containerData.getContainerID()); - } - - List<String> succeedBlocks = new LinkedList<>(); - LOG.debug("Container : {}, To-Delete blocks : {}", - containerData.getContainerID(), toDeleteBlocks.size()); - File dataDir = ContainerUtils.getDataDirectory(containerData).toFile(); - if (!dataDir.exists() || !dataDir.isDirectory()) { - LOG.error("Invalid container data dir {} : " - + "not exist or not a directory", dataDir.getAbsolutePath()); - return crr; - } - - toDeleteBlocks.forEach(entry -> { - String blockName = DFSUtil.bytes2String(entry.getKey()); - LOG.debug("Deleting block {}", blockName); - try { - ContainerProtos.KeyData data = - ContainerProtos.KeyData.parseFrom(entry.getValue()); - for (ContainerProtos.ChunkInfo chunkInfo : data.getChunksList()) { - File chunkFile = dataDir.toPath() - .resolve(chunkInfo.getChunkName()).toFile(); - if (FileUtils.deleteQuietly(chunkFile)) { - LOG.debug("block {} chunk {} deleted", blockName, - chunkFile.getAbsolutePath()); - } - } - succeedBlocks.add(blockName); - } catch (InvalidProtocolBufferException e) { - LOG.error("Failed to parse block info for block {}", blockName, e); - } - }); - - // Once files are deleted... replace deleting entries with deleted entries - BatchOperation batch = new BatchOperation(); - succeedBlocks.forEach(entry -> { - String blockId = - entry.substring(OzoneConsts.DELETING_KEY_PREFIX.length()); - String deletedEntry = OzoneConsts.DELETED_KEY_PREFIX + blockId; - batch.put(DFSUtil.string2Bytes(deletedEntry), - DFSUtil.string2Bytes(blockId)); - batch.delete(DFSUtil.string2Bytes(entry)); - }); - meta.writeBatch(batch); - // update count of pending deletion blocks in in-memory container status - containerManager.decrPendingDeletionBlocks(succeedBlocks.size(), - containerData.getContainerID()); - - if (!succeedBlocks.isEmpty()) { - LOG.info("Container: {}, deleted blocks: {}, task elapsed time: {}ms", - containerData.getContainerID(), succeedBlocks.size(), - Time.monotonicNow() - startTime); - } - crr.addAll(succeedBlocks); - return crr; - } - - @Override - public int getPriority() { - return priority; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java deleted file mode 100644 index a9e202e..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.statemachine.background; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java index e35becd..692a9d1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java @@ -86,20 +86,17 @@ public class VolumeSet { */ private final AutoCloseableLock volumeSetLock; - private final DatanodeDetails dnDetails; - private String datanodeUuid; + private final String datanodeUuid; private String clusterID; - public VolumeSet(DatanodeDetails datanodeDetails, Configuration conf) + public VolumeSet(String dnUuid, Configuration conf) throws DiskOutOfSpaceException { - this(datanodeDetails, null, conf); + this(dnUuid, null, conf); } - public VolumeSet(DatanodeDetails datanodeDetails, String clusterID, - Configuration conf) + public VolumeSet(String dnUuid, String clusterID, Configuration conf) throws DiskOutOfSpaceException { - this.dnDetails = datanodeDetails; - this.datanodeUuid = datanodeDetails.getUuidString(); + this.datanodeUuid = dnUuid; this.clusterID = clusterID; this.conf = conf; this.volumeSetLock = new AutoCloseableLock( http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java index 474c625..95621e5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java @@ -31,6 +31,8 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; @@ -112,7 +114,7 @@ public class KeyValueContainer implements Container { .getVolumesList(), maxSize); String containerBasePath = containerVolume.getHddsRootDir().toString(); - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); String containerName = Long.toString(containerId); containerMetaDataPath = KeyValueContainerLocationUtil @@ -127,7 +129,7 @@ public class KeyValueContainer implements Container { containerMetaDataPath, containerName); // Check if it is new Container. - KeyValueContainerUtil.verifyIsNewContainer(containerMetaDataPath); + ContainerUtils.verifyIsNewContainer(containerMetaDataPath); //Create Metadata path chunks path and metadata db KeyValueContainerUtil.createContainerMetaData(containerMetaDataPath, @@ -184,7 +186,7 @@ public class KeyValueContainer implements Container { File tempCheckSumFile = null; FileOutputStream containerCheckSumStream = null; Writer writer = null; - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); try { tempContainerFile = createTempFile(containerFile); tempCheckSumFile = createTempFile(containerCheckSumFile); @@ -238,7 +240,7 @@ public class KeyValueContainer implements Container { File containerBkpFile = null; File checkSumBkpFile = null; - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); try { if (containerFile.exists() && containerCheckSumFile.exists()) { @@ -251,8 +253,8 @@ public class KeyValueContainer implements Container { } else { containerData.setState(ContainerProtos.ContainerLifeCycleState.INVALID); throw new StorageContainerException("Container is an Inconsistent " + - "state, missing required files(.container, .chksm)", - INVALID_CONTAINER_STATE); + "state, missing required files(.container, .chksm). ContainerID: " + + containerId, INVALID_CONTAINER_STATE); } } catch (StorageContainerException ex) { throw ex; @@ -303,7 +305,7 @@ public class KeyValueContainer implements Container { @Override public void delete(boolean forceDelete) throws StorageContainerException { - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); try { KeyValueContainerUtil.removeContainer(containerData, config, forceDelete); } catch (StorageContainerException ex) { @@ -326,11 +328,11 @@ public class KeyValueContainer implements Container { // complete this action try { writeLock(); - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); if(!containerData.isValid()) { LOG.debug("Invalid container data. Container Id: {}", containerId); - throw new StorageContainerException("Invalid container data. Name : " + - containerId, INVALID_CONTAINER_STATE); + throw new StorageContainerException("Invalid container data. " + + "ContainerID: " + containerId, INVALID_CONTAINER_STATE); } containerData.closeContainer(); File containerFile = getContainerFile(); @@ -380,16 +382,16 @@ public class KeyValueContainer implements Container { // TODO: Now, when writing the updated data to .container file, we are // holding lock and writing data to disk. We can have async implementation // to flush the update container data to disk. - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); if(!containerData.isValid()) { - LOG.debug("Invalid container data. ID: {}", containerId); + LOG.debug("Invalid container data. ContainerID: {}", containerId); throw new StorageContainerException("Invalid container data. " + - "Container Name : " + containerId, INVALID_CONTAINER_STATE); + "ContainerID: " + containerId, INVALID_CONTAINER_STATE); } if (!forceUpdate && !containerData.isOpen()) { throw new StorageContainerException( - "Updating a closed container is not allowed. ID: " + containerId, - UNSUPPORTED_REQUEST); + "Updating a closed container without force option is not allowed. " + + "ContainerID: " + containerId, UNSUPPORTED_REQUEST); } try { for (Map.Entry<String, String> entry : metadata.entrySet()) { @@ -482,7 +484,7 @@ public class KeyValueContainer implements Container { */ private File getContainerFile() { return new File(containerData.getMetadataPath(), containerData - .getContainerId() + OzoneConsts.CONTAINER_EXTENSION); + .getContainerID() + OzoneConsts.CONTAINER_EXTENSION); } /** @@ -491,7 +493,7 @@ public class KeyValueContainer implements Container { */ private File getContainerCheckSumFile() { return new File(containerData.getMetadataPath(), containerData - .getContainerId() + OzoneConsts.CONTAINER_FILE_CHECKSUM_EXTENSION); + .getContainerID() + OzoneConsts.CONTAINER_FILE_CHECKSUM_EXTENSION); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java index ed2c6af..d9ae38a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java @@ -18,16 +18,29 @@ package org.apache.hadoop.ozone.container.keyvalue; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.yaml.snakeyaml.nodes.Tag; import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Map; +import static org.apache.hadoop.ozone.OzoneConsts.CHUNKS_PATH; +import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE; +import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ID; +import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_TYPE; +import static org.apache.hadoop.ozone.OzoneConsts.LAYOUTVERSION; +import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE_GB; +import static org.apache.hadoop.ozone.OzoneConsts.METADATA; +import static org.apache.hadoop.ozone.OzoneConsts.METADATA_PATH; +import static org.apache.hadoop.ozone.OzoneConsts.STATE; + /** * This class represents the KeyValueContainer metadata, which is the * in-memory representation of container metadata and is represented on disk @@ -36,12 +49,20 @@ import java.util.Map; public class KeyValueContainerData extends ContainerData { // Yaml Tag used for KeyValueContainerData. - public static final Tag YAML_TAG = new Tag("KeyValueContainerData"); + public static final Tag KEYVALUE_YAML_TAG = new Tag("KeyValueContainerData"); // Fields need to be stored in .container file. - private static final List<String> YAML_FIELDS = Lists.newArrayList( - "containerType", "containerId", "layOutVersion", "state", "metadata", - "metadataPath", "chunksPath", "containerDBType", "maxSizeGB"); + private static final List<String> YAML_FIELDS = + Lists.newArrayList( + CONTAINER_TYPE, + CONTAINER_ID, + LAYOUTVERSION, + STATE, + METADATA, + METADATA_PATH, + CHUNKS_PATH, + CONTAINER_DB_TYPE, + MAX_SIZE_GB); // Path to Container metadata Level DB/RocksDB Store and .container file. private String metadataPath; @@ -96,11 +117,11 @@ public class KeyValueContainerData extends ContainerData { public File getDbFile() { return dbFile; } + /** * Returns container metadata path. - * - * @return - path */ + @Override public String getMetadataPath() { return metadataPath; } @@ -123,6 +144,14 @@ public class KeyValueContainerData extends ContainerData { } /** + * Returns container chunks path. + */ + @Override + public String getDataPath() { + return chunksPath; + } + + /** * Set chunks Path. * @param chunkPath - File path. */ @@ -181,7 +210,7 @@ public class KeyValueContainerData extends ContainerData { public ContainerProtos.ContainerData getProtoBufMessage() { ContainerProtos.ContainerData.Builder builder = ContainerProtos .ContainerData.newBuilder(); - builder.setContainerID(this.getContainerId()); + builder.setContainerID(this.getContainerID()); builder.setDbPath(this.getDbFile().getPath()); builder.setContainerPath(this.getMetadataPath()); builder.setState(this.getState()); @@ -211,4 +240,41 @@ public class KeyValueContainerData extends ContainerData { public static List<String> getYamlFields() { return YAML_FIELDS; } + + /** + * Constructs a KeyValueContainerData object from ProtoBuf classes. + * + * @param protoData - ProtoBuf Message + * @throws IOException + */ + @VisibleForTesting + public static KeyValueContainerData getFromProtoBuf( + ContainerProtos.ContainerData protoData) throws IOException { + // TODO: Add containerMaxSize to ContainerProtos.ContainerData + KeyValueContainerData data = new KeyValueContainerData( + protoData.getContainerID(), + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT); + for (int x = 0; x < protoData.getMetadataCount(); x++) { + data.addMetadata(protoData.getMetadata(x).getKey(), + protoData.getMetadata(x).getValue()); + } + + if (protoData.hasContainerPath()) { + data.setContainerPath(protoData.getContainerPath()); + } + + if (protoData.hasState()) { + data.setState(protoData.getState()); + } + + if (protoData.hasBytesUsed()) { + data.setBytesUsed(protoData.getBytesUsed()); + } + + if(protoData.hasContainerDBType()) { + data.setContainerDBType(protoData.getContainerDBType()); + } + + return data; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 8166a83..b2c82f0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -288,7 +288,7 @@ public class KeyValueHandler extends Handler { DELETE_ON_OPEN_CONTAINER); } else { containerSet.removeContainer( - kvContainer.getContainerData().getContainerId()); + kvContainer.getContainerData().getContainerID()); // Release the lock first. // Avoid holding write locks for disk operations kvContainer.writeUnlock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java index 3529af8..62e328e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java @@ -33,8 +33,8 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl; import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats; import org.apache.hadoop.util.Time; @@ -229,17 +229,16 @@ public final class ChunkUtils { * Validates chunk data and returns a file object to Chunk File that we are * expected to write data to. * - * @param data - container data. + * @param chunkFile - chunkFile to write data into. * @param info - chunk info. - * @return File + * @return boolean isOverwrite * @throws StorageContainerException */ - public static File validateChunk(KeyValueContainerData data, ChunkInfo info) - throws StorageContainerException { + public static boolean validateChunkForOverwrite(File chunkFile, + ChunkInfo info) throws StorageContainerException { Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class); - File chunkFile = getChunkFile(data, info); if (isOverWriteRequested(chunkFile, info)) { if (!isOverWritePermitted(info)) { log.error("Rejecting write chunk request. Chunk overwrite " + @@ -248,8 +247,9 @@ public final class ChunkUtils { "OverWrite flag required." + info.toString(), OVERWRITE_FLAG_REQUIRED); } + return true; } - return chunkFile; + return false; } /** @@ -340,8 +340,8 @@ public final class ChunkUtils { public static ContainerCommandResponseProto getReadChunkResponse( ContainerCommandRequestProto msg, byte[] data, ChunkInfo info) { Preconditions.checkNotNull(msg); - Preconditions.checkNotNull("Chunk data is null", data); - Preconditions.checkNotNull("Chunk Info is null", info); + Preconditions.checkNotNull(data, "Chunk data is null"); + Preconditions.checkNotNull(info, "Chunk Info is null"); ReadChunkResponseProto.Builder response = ReadChunkResponseProto.newBuilder(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java index 714f445..5845fae 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java @@ -57,24 +57,25 @@ public final class KeyUtils { * add into cache. This function is called with containerManager * ReadLock held. * - * @param container container. + * @param containerData containerData. * @param conf configuration. * @return MetadataStore handle. * @throws StorageContainerException */ - public static MetadataStore getDB(KeyValueContainerData container, + public static MetadataStore getDB(KeyValueContainerData containerData, Configuration conf) throws StorageContainerException { - Preconditions.checkNotNull(container); + Preconditions.checkNotNull(containerData); ContainerCache cache = ContainerCache.getInstance(conf); Preconditions.checkNotNull(cache); - Preconditions.checkNotNull(container.getDbFile()); + Preconditions.checkNotNull(containerData.getDbFile()); try { - return cache.getDB(container.getContainerId(), container - .getContainerDBType(), container.getDbFile().getAbsolutePath()); + return cache.getDB(containerData.getContainerID(), containerData + .getContainerDBType(), containerData.getDbFile().getAbsolutePath()); } catch (IOException ex) { - String message = String.format("Unable to open DB Path: " + - "%s. ex: %s", container.getDbFile(), ex.getMessage()); + String message = String.format("Error opening DB. Container:%s " + + "ContainerPath:%s", containerData.getContainerID(), containerData + .getDbFile().getPath()); throw new StorageContainerException(message, UNABLE_TO_READ_METADATA_DB); } } @@ -89,7 +90,7 @@ public final class KeyUtils { Preconditions.checkNotNull(container); ContainerCache cache = ContainerCache.getInstance(conf); Preconditions.checkNotNull(cache); - cache.removeDB(container.getContainerId()); + cache.removeDB(container.getContainerID()); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java index 4c17dce..3c3c9cb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java @@ -34,6 +34,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.helpers.KeyData; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.utils.MetadataKeyFilters; import org.apache.hadoop.utils.MetadataStore; @@ -68,19 +69,6 @@ public final class KeyValueContainerUtil { private static final Logger LOG = LoggerFactory.getLogger( KeyValueContainerUtil.class); - - public static void verifyIsNewContainer(File containerFile) throws - FileAlreadyExistsException { - Preconditions.checkNotNull(containerFile, "containerFile Should not be " + - "null"); - if (containerFile.getParentFile().exists()) { - LOG.error("container already exists on disk. File: {}", containerFile - .toPath()); - throw new FileAlreadyExistsException("container already exists on " + - "disk."); - } - } - /** * creates metadata path, chunks path and metadata DB for the specified * container. @@ -271,7 +259,7 @@ public final class KeyValueContainerUtil { Preconditions.checkNotNull(dbFile, "dbFile cannot be null"); Preconditions.checkNotNull(config, "ozone config cannot be null"); - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); String containerName = String.valueOf(containerId); File metadataPath = new File(containerData.getMetadataPath()); @@ -282,7 +270,7 @@ public final class KeyValueContainerUtil { // Verify Checksum String checksum = KeyValueContainerUtil.computeCheckSum( - containerData.getContainerId(), containerFile); + containerData.getContainerID(), containerFile); KeyValueContainerUtil.verifyCheckSum(containerId, checksumFile, checksum); containerData.setDbFile(dbFile); @@ -305,4 +293,34 @@ public final class KeyValueContainerUtil { containerData.setKeyCount(liveKeys.size()); } + /** + * Returns the path where data or chunks live for a given container. + * + * @param kvContainerData - KeyValueContainerData + * @return - Path to the chunks directory + */ + public static Path getDataDirectory(KeyValueContainerData kvContainerData) { + + String chunksPath = kvContainerData.getChunksPath(); + Preconditions.checkNotNull(chunksPath); + + return Paths.get(chunksPath); + } + + /** + * Container metadata directory -- here is where the level DB and + * .container file lives. + * + * @param kvContainerData - KeyValueContainerData + * @return Path to the metadata directory + */ + public static Path getMetadataDirectory( + KeyValueContainerData kvContainerData) { + + String metadataPath = kvContainerData.getMetadataPath(); + Preconditions.checkNotNull(metadataPath); + + return Paths.get(metadataPath); + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java index c3160a8..ce317bd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java @@ -76,7 +76,10 @@ public class ChunkManagerImpl implements ChunkManager { HddsVolume volume = containerData.getVolume(); VolumeIOStats volumeIOStats = volume.getVolumeIOStats(); - File chunkFile = ChunkUtils.validateChunk(containerData, info); + File chunkFile = ChunkUtils.getChunkFile(containerData, info); + + boolean isOverwrite = ChunkUtils.validateChunkForOverwrite( + chunkFile, info); File tmpChunkFile = getTmpChunkFile(chunkFile, info); LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file", @@ -101,8 +104,9 @@ public class ChunkManagerImpl implements ChunkManager { case COMBINED: // directly write to the chunk file ChunkUtils.writeData(chunkFile, info, data, volumeIOStats); - // Increment container stats here, as we directly write to chunk file. - containerData.incrBytesUsed(info.getLen()); + if (!isOverwrite) { + containerData.incrBytesUsed(info.getLen()); + } containerData.incrWriteCount(); containerData.incrWriteBytes(info.getLen()); break; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java new file mode 100644 index 0000000..6aa54d1 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java @@ -0,0 +1,248 @@ + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.keyvalue.statemachine.background; + +import com.google.common.collect.Lists; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; +import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.ratis.shaded.com.google.protobuf + .InvalidProtocolBufferException; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.scm.container.common.helpers + .StorageContainerException; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.utils.BackgroundService; +import org.apache.hadoop.utils.BackgroundTask; +import org.apache.hadoop.utils.BackgroundTaskQueue; +import org.apache.hadoop.utils.BackgroundTaskResult; +import org.apache.hadoop.utils.BatchOperation; +import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter; +import org.apache.hadoop.utils.MetadataStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.hadoop.ozone.OzoneConfigKeys + .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys + .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys + .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER; +import static org.apache.hadoop.ozone.OzoneConfigKeys + .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT; + +/** + * A per-datanode container block deleting service takes in charge + * of deleting staled ozone blocks. + */ +// TODO: Fix BlockDeletingService to work with new StorageLayer +public class BlockDeletingService extends BackgroundService{ + + private static final Logger LOG = + LoggerFactory.getLogger(BlockDeletingService.class); + + ContainerSet containerSet; + private final Configuration conf; + + // Throttle number of blocks to delete per task, + // set to 1 for testing + private final int blockLimitPerTask; + + // Throttle the number of containers to process concurrently at a time, + private final int containerLimitPerInterval; + + // Task priority is useful when a to-delete block has weight. + private final static int TASK_PRIORITY_DEFAULT = 1; + // Core pool size for container tasks + private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10; + + public BlockDeletingService(ContainerSet containerSet, + long serviceInterval, long serviceTimeout, Configuration conf) { + super("BlockDeletingService", serviceInterval, + TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, + serviceTimeout); + this.containerSet = containerSet; + this.conf = conf; + this.blockLimitPerTask = conf.getInt( + OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, + OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT); + this.containerLimitPerInterval = conf.getInt( + OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, + OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT); + } + + + @Override + public BackgroundTaskQueue getTasks() { + BackgroundTaskQueue queue = new BackgroundTaskQueue(); + List<ContainerData> containers = Lists.newArrayList(); + try { + // We at most list a number of containers a time, + // in case there are too many containers and start too many workers. + // We must ensure there is no empty container in this result. + // The chosen result depends on what container deletion policy is + // configured. + containers = containerSet.chooseContainerForBlockDeletion( + containerLimitPerInterval); + LOG.info("Plan to choose {} containers for block deletion, " + + "actually returns {} valid containers.", + containerLimitPerInterval, containers.size()); + + for(ContainerData container : containers) { + BlockDeletingTask containerTask = + new BlockDeletingTask(container, TASK_PRIORITY_DEFAULT); + queue.add(containerTask); + } + } catch (StorageContainerException e) { + LOG.warn("Failed to initiate block deleting tasks, " + + "caused by unable to get containers info. " + + "Retry in next interval. ", e); + } catch (Exception e) { + // In case listContainer call throws any uncaught RuntimeException. + if (LOG.isDebugEnabled()) { + LOG.debug("Unexpected error occurs during deleting blocks.", e); + } + } + return queue; + } + + private static class ContainerBackgroundTaskResult + implements BackgroundTaskResult { + private List<String> deletedBlockIds; + + ContainerBackgroundTaskResult() { + deletedBlockIds = new LinkedList<>(); + } + + public void addBlockId(String blockId) { + deletedBlockIds.add(blockId); + } + + public void addAll(List<String> blockIds) { + deletedBlockIds.addAll(blockIds); + } + + public List<String> getDeletedBlocks() { + return deletedBlockIds; + } + + @Override + public int getSize() { + return deletedBlockIds.size(); + } + } + + private class BlockDeletingTask + implements BackgroundTask<BackgroundTaskResult> { + + private final int priority; + private final ContainerData containerData; + + BlockDeletingTask(ContainerData containerName, int priority) { + this.priority = priority; + this.containerData = containerName; + } + + @Override + public BackgroundTaskResult call() throws Exception { + ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult(); + long startTime = Time.monotonicNow(); + // Scan container's db and get list of under deletion blocks + MetadataStore meta = KeyUtils.getDB( + (KeyValueContainerData) containerData, conf); + // # of blocks to delete is throttled + KeyPrefixFilter filter = + new KeyPrefixFilter().addFilter(OzoneConsts.DELETING_KEY_PREFIX); + List<Map.Entry<byte[], byte[]>> toDeleteBlocks = + meta.getSequentialRangeKVs(null, blockLimitPerTask, filter); + if (toDeleteBlocks.isEmpty()) { + LOG.debug("No under deletion block found in container : {}", + containerData.getContainerID()); + } + + List<String> succeedBlocks = new LinkedList<>(); + LOG.debug("Container : {}, To-Delete blocks : {}", + containerData.getContainerID(), toDeleteBlocks.size()); + File dataDir = new File(containerData.getDataPath()); + if (!dataDir.exists() || !dataDir.isDirectory()) { + LOG.error("Invalid container data dir {} : " + + "not exist or not a directory", dataDir.getAbsolutePath()); + return crr; + } + + toDeleteBlocks.forEach(entry -> { + String blockName = DFSUtil.bytes2String(entry.getKey()); + LOG.debug("Deleting block {}", blockName); + try { + ContainerProtos.KeyData data = + ContainerProtos.KeyData.parseFrom(entry.getValue()); + for (ContainerProtos.ChunkInfo chunkInfo : data.getChunksList()) { + File chunkFile = dataDir.toPath() + .resolve(chunkInfo.getChunkName()).toFile(); + if (FileUtils.deleteQuietly(chunkFile)) { + LOG.debug("block {} chunk {} deleted", blockName, + chunkFile.getAbsolutePath()); + } + } + succeedBlocks.add(blockName); + } catch (InvalidProtocolBufferException e) { + LOG.error("Failed to parse block info for block {}", blockName, e); + } + }); + + // Once files are deleted... replace deleting entries with deleted entries + BatchOperation batch = new BatchOperation(); + succeedBlocks.forEach(entry -> { + String blockId = + entry.substring(OzoneConsts.DELETING_KEY_PREFIX.length()); + String deletedEntry = OzoneConsts.DELETED_KEY_PREFIX + blockId; + batch.put(DFSUtil.string2Bytes(deletedEntry), + DFSUtil.string2Bytes(blockId)); + batch.delete(DFSUtil.string2Bytes(entry)); + }); + meta.writeBatch(batch); + // update count of pending deletion blocks in in-memory container status + containerData.decrPendingDeletionBlocks(succeedBlocks.size()); + + if (!succeedBlocks.isEmpty()) { + LOG.info("Container: {}, deleted blocks: {}, task elapsed time: {}ms", + containerData.getContainerID(), succeedBlocks.size(), + Time.monotonicNow() - startTime); + } + crr.addAll(succeedBlocks); + return crr; + } + + @Override + public int getPriority() { + return priority; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/package-info.java new file mode 100644 index 0000000..69d8042 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/package-info.java @@ -0,0 +1,18 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.keyvalue.statemachine.background; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 9e25c59..667ea5c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -73,7 +73,7 @@ public class OzoneContainer { conf) throws IOException { this.dnDetails = datanodeDetails; this.config = conf; - this.volumeSet = new VolumeSet(datanodeDetails, conf); + this.volumeSet = new VolumeSet(datanodeDetails.getUuidString(), conf); this.containerSet = new ContainerSet(); boolean useGrpc = this.config.getBoolean( ScmConfigKeys.DFS_CONTAINER_GRPC_ENABLED_KEY, http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java index 16c4c2a..42db66d 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java @@ -47,7 +47,7 @@ public class TestKeyValueContainerData { MAXSIZE); assertEquals(containerType, kvData.getContainerType()); - assertEquals(containerId, kvData.getContainerId()); + assertEquals(containerId, kvData.getContainerID()); assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN, kvData .getState()); assertEquals(0, kvData.getMetadata().size()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java index 41d8315..eed5606 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java @@ -64,7 +64,7 @@ public class TestContainerDataYaml { // Read from .container file, and verify data. KeyValueContainerData kvData = (KeyValueContainerData) ContainerDataYaml .readContainerFile(containerFile); - assertEquals(Long.MAX_VALUE, kvData.getContainerId()); + assertEquals(Long.MAX_VALUE, kvData.getContainerID()); assertEquals(ContainerProtos.ContainerType.KeyValueContainer, kvData .getContainerType()); assertEquals("RocksDB", kvData.getContainerDBType()); @@ -92,7 +92,7 @@ public class TestContainerDataYaml { containerFile); // verify data. - assertEquals(Long.MAX_VALUE, kvData.getContainerId()); + assertEquals(Long.MAX_VALUE, kvData.getContainerID()); assertEquals(ContainerProtos.ContainerType.KeyValueContainer, kvData .getContainerType()); assertEquals("RocksDB", kvData.getContainerDBType()); @@ -150,7 +150,7 @@ public class TestContainerDataYaml { assertEquals("RocksDB", kvData.getContainerDBType()); assertEquals(ContainerProtos.ContainerType.KeyValueContainer, kvData .getContainerType()); - assertEquals(9223372036854775807L, kvData.getContainerId()); + assertEquals(9223372036854775807L, kvData.getContainerID()); assertEquals("/hdds/current/aed-fg4-hji-jkl/containerdir0/1", kvData .getChunksPath()); assertEquals("/hdds/current/aed-fg4-hji-jkl/containerdir0/1", kvData http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java index 6ec1fe4..ae670e0 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java @@ -74,7 +74,7 @@ public class TestContainerSet { .getContainer(containerId); KeyValueContainerData keyValueContainerData = (KeyValueContainerData) container.getContainerData(); - assertEquals(containerId, keyValueContainerData.getContainerId()); + assertEquals(containerId, keyValueContainerData.getContainerID()); assertEquals(state, keyValueContainerData.getState()); assertNull(containerSet.getContainer(1000L)); @@ -97,7 +97,7 @@ public class TestContainerSet { while(containerIterator.hasNext()) { Container kv = containerIterator.next(); ContainerData containerData = kv.getContainerData(); - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); if (containerId%2 == 0) { assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED, containerData.getState()); @@ -117,7 +117,7 @@ public class TestContainerSet { while (containerMapIterator.hasNext()) { Container kv = containerMapIterator.next().getValue(); ContainerData containerData = kv.getContainerData(); - long containerId = containerData.getContainerId(); + long containerId = containerData.getContainerID(); if (containerId%2 == 0) { assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED, containerData.getState()); @@ -155,8 +155,8 @@ public class TestContainerSet { assertEquals(5, result.size()); for(ContainerData containerData : result) { - assertTrue(containerData.getContainerId() >=2 && containerData - .getContainerId()<=6); + assertTrue(containerData.getContainerID() >=2 && containerData + .getContainerID()<=6); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java index 41610af..a45a639 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java @@ -54,11 +54,7 @@ public class TestRoundRobinVolumeChoosingPolicy { conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDirKey); policy = ReflectionUtils.newInstance( RoundRobinVolumeChoosingPolicy.class, null); - DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder() - .setUuid(UUID.randomUUID().toString()) - .setIpAddress(DUMMY_IP_ADDR) - .build(); - VolumeSet volumeSet = new VolumeSet(datanodeDetails, conf); + VolumeSet volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf); volumes = volumeSet.getVolumesList(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java index 61383de..41f75bd 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java @@ -53,11 +53,7 @@ public class TestVolumeSet { private static final String DUMMY_IP_ADDR = "0.0.0.0"; private void initializeVolumeSet() throws Exception { - DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder() - .setUuid(UUID.randomUUID().toString()) - .setIpAddress(DUMMY_IP_ADDR) - .build(); - volumeSet = new VolumeSet(datanodeDetails, conf); + volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf); } @Rule http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index e55ea57..4f00507 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -245,24 +245,6 @@ public class TestKeyValueContainer { } @Test - public void testUpdateContainerInvalidMetadata() throws IOException { - try { - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - Map<String, String> metadata = new HashMap<>(); - metadata.put("VOLUME", "ozone"); - keyValueContainer.update(metadata, true); - //Trying to update again with same metadata - keyValueContainer.update(metadata, true); - fail("testUpdateContainerInvalidMetadata failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("Container Metadata update " + - "error", ex); - assertEquals(ContainerProtos.Result.CONTAINER_METADATA_ERROR, ex - .getResult()); - } - } - - @Test public void testUpdateContainerUnsupportedRequest() throws Exception { try { keyValueContainerData.setState(ContainerProtos.ContainerLifeCycleState @@ -275,7 +257,7 @@ public class TestKeyValueContainer { fail("testUpdateContainerUnsupportedRequest failed"); } catch (StorageContainerException ex) { GenericTestUtils.assertExceptionContains("Updating a closed container " + - "is not allowed", ex); + "without force option is not allowed", ex); assertEquals(ContainerProtos.Result.UNSUPPORTED_REQUEST, ex .getResult()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java index 947ad51..a997145 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.keyvalue; +import com.google.common.base.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -30,6 +31,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -71,12 +73,7 @@ public class TestKeyValueHandler { conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, volume); this.containerSet = new ContainerSet(); - DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder() - .setUuid(DATANODE_UUID) - .setHostName("localhost") - .setIpAddress("127.0.0.1") - .build(); - this.volumeSet = new VolumeSet(datanodeDetails, conf); + this.volumeSet = new VolumeSet(DATANODE_UUID, conf); this.dispatcher = new HddsDispatcher(conf, containerSet, volumeSet); this.handler = (KeyValueHandler) dispatcher.getHandler( @@ -246,7 +243,7 @@ public class TestKeyValueHandler { // Verify that new container is added to containerSet. Container container = containerSet.getContainer(contId); - Assert.assertEquals(contId, container.getContainerData().getContainerId()); + Assert.assertEquals(contId, container.getContainerData().getContainerID()); Assert.assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN, container.getContainerState()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 26e1c77..27c6528 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -62,7 +62,7 @@ public class TestOzoneContainer { conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, folder.getRoot() .getAbsolutePath() + "," + folder.newFolder().getAbsolutePath()); conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().getAbsolutePath()); - volumeSet = new VolumeSet(datanodeDetails, conf); + volumeSet = new VolumeSet(datanodeDetails.getUuidString(), conf); volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy(); for (int i=0; i<10; i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java index 7c12945..a87f655 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java @@ -19,8 +19,8 @@ package org.apache.hadoop.ozone.container.testutils; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager; -import org.apache.hadoop.ozone.container.common.statemachine.background +import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.keyvalue.statemachine.background .BlockDeletingService; import java.util.concurrent.CountDownLatch; @@ -42,10 +42,9 @@ public class BlockDeletingServiceTestImpl private Thread testingThread; private AtomicInteger numOfProcessed = new AtomicInteger(0); - public BlockDeletingServiceTestImpl(ContainerManager containerManager, + public BlockDeletingServiceTestImpl(ContainerSet containerSet, int serviceInterval, Configuration conf) { - super(containerManager, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS, - TimeUnit.MILLISECONDS, conf); + super(containerSet, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS, conf); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/resources/additionalfields.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/additionalfields.container b/hadoop-hdds/container-service/src/test/resources/additionalfields.container index 9027538..38c104a 100644 --- a/hadoop-hdds/container-service/src/test/resources/additionalfields.container +++ b/hadoop-hdds/container-service/src/test/resources/additionalfields.container @@ -1,7 +1,7 @@ !<KeyValueContainerData> containerDBType: RocksDB chunksPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1 -containerId: 9223372036854775807 +containerID: 9223372036854775807 containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1 layOutVersion: 1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-hdds/container-service/src/test/resources/incorrect.container ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/resources/incorrect.container b/hadoop-hdds/container-service/src/test/resources/incorrect.container index 6848484..abbb6aa 100644 --- a/hadoop-hdds/container-service/src/test/resources/incorrect.container +++ b/hadoop-hdds/container-service/src/test/resources/incorrect.container @@ -1,7 +1,7 @@ !<KeyValueContainerData> containerDBType: RocksDB chunksPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1 -containerId: 9223372036854775807 +containerID: 9223372036854775807 containerType: KeyValueContainer metadataPath: /hdds/current/aed-fg4-hji-jkl/containerdir0/1 layOutVersion: 1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2351e8/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java index b720549..d25b73e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java @@ -58,6 +58,8 @@ public final class ContainerTestHelper { ContainerTestHelper.class); private static Random r = new Random(); + public static final int CONTAINER_MAX_SIZE_GB = 1; + /** * Never constructed. */ --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org