http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 66136f1..4eedf1f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -160,7 +160,7 @@ public class SCMClientProtocolServer implements String remoteUser = getRpcRemoteUsername(); getScm().checkAdminAccess(remoteUser); - return scm.getScmContainerManager() + return scm.getContainerManager() .allocateContainer(replicationType, factor, owner); } @@ -168,7 +168,7 @@ public class SCMClientProtocolServer implements public ContainerInfo getContainer(long containerID) throws IOException { String remoteUser = getRpcRemoteUsername(); getScm().checkAdminAccess(remoteUser); - return scm.getScmContainerManager() + return scm.getContainerManager() .getContainer(containerID); } @@ -176,7 +176,7 @@ public class SCMClientProtocolServer implements public ContainerWithPipeline getContainerWithPipeline(long containerID) throws IOException { if (chillModePrecheck.isInChillMode()) { - ContainerInfo contInfo = scm.getScmContainerManager() + ContainerInfo contInfo = scm.getContainerManager() .getContainer(containerID); if (contInfo.isContainerOpen()) { if (!hasRequiredReplicas(contInfo)) { @@ -188,7 +188,7 @@ public class SCMClientProtocolServer implements } String remoteUser = getRpcRemoteUsername(); getScm().checkAdminAccess(remoteUser); - return scm.getScmContainerManager() + return scm.getContainerManager() .getContainerWithPipeline(containerID); } @@ -198,7 +198,7 @@ public class SCMClientProtocolServer implements */ private boolean hasRequiredReplicas(ContainerInfo contInfo) { try{ - return getScm().getScmContainerManager().getStateManager() + return getScm().getContainerManager().getStateManager() .getContainerReplicas(contInfo.containerID()) .size() >= contInfo.getReplicationFactor().getNumber(); } catch (SCMException ex) { @@ -211,7 +211,7 @@ public class SCMClientProtocolServer implements @Override public List<ContainerInfo> listContainer(long startContainerID, int count) throws IOException { - return scm.getScmContainerManager(). + return scm.getContainerManager(). listContainer(startContainerID, count); } @@ -219,7 +219,7 @@ public class SCMClientProtocolServer implements public void deleteContainer(long containerID) throws IOException { String remoteUser = getRpcRemoteUsername(); getScm().checkAdminAccess(remoteUser); - scm.getScmContainerManager().deleteContainer(containerID); + scm.getContainerManager().deleteContainer(containerID); } @@ -257,10 +257,10 @@ public class SCMClientProtocolServer implements .ObjectStageChangeRequestProto.Op.create) { if (stage == StorageContainerLocationProtocolProtos .ObjectStageChangeRequestProto.Stage.begin) { - scm.getScmContainerManager().updateContainerState(id, HddsProtos + scm.getContainerManager().updateContainerState(id, HddsProtos .LifeCycleEvent.CREATE); } else { - scm.getScmContainerManager().updateContainerState(id, HddsProtos + scm.getContainerManager().updateContainerState(id, HddsProtos .LifeCycleEvent.CREATED); } } else { @@ -268,10 +268,10 @@ public class SCMClientProtocolServer implements .ObjectStageChangeRequestProto.Op.close) { if (stage == StorageContainerLocationProtocolProtos .ObjectStageChangeRequestProto.Stage.begin) { - scm.getScmContainerManager().updateContainerState(id, HddsProtos + scm.getContainerManager().updateContainerState(id, HddsProtos .LifeCycleEvent.FINALIZE); } else { - scm.getScmContainerManager().updateContainerState(id, HddsProtos + scm.getContainerManager().updateContainerState(id, HddsProtos .LifeCycleEvent.CLOSE); } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java index 9c6fa88..2c96856 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java @@ -196,7 +196,7 @@ public class SCMDatanodeProtocolServer implements .register(datanodeDetails, nodeReport, pipelineReportsProto); if (registeredCommand.getError() == SCMRegisteredResponseProto.ErrorCode.success) { - scm.getScmContainerManager().processContainerReports(datanodeDetails, + scm.getContainerManager().processContainerReports(datanodeDetails, containerReportsProto, true); eventPublisher.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT, new NodeRegistrationContainerReport(datanodeDetails, http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index bdafe0c..efd5fc5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -40,9 +40,9 @@ import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler; import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler; import org.apache.hadoop.hdds.scm.container.CloseContainerWatcher; import org.apache.hadoop.hdds.scm.container.ContainerActionsHandler; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerReportHandler; -import org.apache.hadoop.hdds.scm.container.Mapping; import org.apache.hadoop.hdds.scm.container.replication .ReplicationActivityStatus; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; @@ -151,7 +151,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl * State Managers of SCM. */ private final NodeManager scmNodeManager; - private final Mapping scmContainerManager; + private final ContainerManager containerManager; private final BlockManager scmBlockManager; private final SCMStorage scmStorage; @@ -206,43 +206,43 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl scmNodeManager = new SCMNodeManager( conf, scmStorage.getClusterID(), this, eventQueue); - scmContainerManager = new ContainerMapping( + containerManager = new SCMContainerManager( conf, getScmNodeManager(), cacheSize, eventQueue); scmBlockManager = new BlockManagerImpl( - conf, getScmNodeManager(), scmContainerManager, eventQueue); + conf, getScmNodeManager(), containerManager, eventQueue); replicationStatus = new ReplicationActivityStatus(); CloseContainerEventHandler closeContainerHandler = - new CloseContainerEventHandler(scmContainerManager); + new CloseContainerEventHandler(containerManager); NodeReportHandler nodeReportHandler = new NodeReportHandler(scmNodeManager); PipelineReportHandler pipelineReportHandler = new PipelineReportHandler( - scmContainerManager.getPipelineSelector()); + containerManager.getPipelineSelector()); CommandStatusReportHandler cmdStatusReportHandler = new CommandStatusReportHandler(); NewNodeHandler newNodeHandler = new NewNodeHandler(scmNodeManager); StaleNodeHandler staleNodeHandler = - new StaleNodeHandler(scmContainerManager.getPipelineSelector()); + new StaleNodeHandler(containerManager.getPipelineSelector()); DeadNodeHandler deadNodeHandler = new DeadNodeHandler(scmNodeManager, - getScmContainerManager().getStateManager()); + getContainerManager().getStateManager()); ContainerActionsHandler actionsHandler = new ContainerActionsHandler(); PendingDeleteHandler pendingDeleteHandler = new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService()); ContainerReportHandler containerReportHandler = - new ContainerReportHandler(scmContainerManager, scmNodeManager, + new ContainerReportHandler(containerManager, scmNodeManager, replicationStatus); scmChillModeManager = new SCMChillModeManager(conf, - getScmContainerManager().getStateManager().getAllContainers(), + getContainerManager().getStateManager().getAllContainers(), eventQueue); PipelineActionEventHandler pipelineActionEventHandler = new PipelineActionEventHandler(); PipelineCloseHandler pipelineCloseHandler = - new PipelineCloseHandler(scmContainerManager.getPipelineSelector()); + new PipelineCloseHandler(containerManager.getPipelineSelector()); long watcherTimeout = conf.getTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT, @@ -263,14 +263,14 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl new SCMContainerPlacementCapacity(scmNodeManager, conf); replicationManager = new ReplicationManager(containerPlacementPolicy, - scmContainerManager.getStateManager(), eventQueue, + containerManager.getStateManager(), eventQueue, commandWatcherLeaseManager); // setup CloseContainer watcher CloseContainerWatcher closeContainerWatcher = new CloseContainerWatcher(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ, SCMEvents.CLOSE_CONTAINER_STATUS, commandWatcherLeaseManager, - scmContainerManager); + containerManager); closeContainerWatcher.start(eventQueue); scmAdminUsernames = conf.getTrimmedStringCollection(OzoneConfigKeys @@ -632,7 +632,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl @VisibleForTesting public ContainerInfo getContainerInfo(long containerID) throws IOException { - return scmContainerManager.getContainer(containerID); + return containerManager.getContainer(containerID); } /** @@ -774,7 +774,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl } catch (Exception ex) { LOG.error("SCM Event Queue stop failed", ex); } - IOUtils.cleanupWithLogger(LOG, scmContainerManager); + IOUtils.cleanupWithLogger(LOG, containerManager); } /** @@ -805,8 +805,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl * Returns SCM container manager. */ @VisibleForTesting - public Mapping getScmContainerManager() { - return scmContainerManager; + public ContainerManager getContainerManager() { + return containerManager; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index e70e444..25f6ae3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -56,7 +56,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.MB; * Tests for SCM Block Manager. */ public class TestBlockManager implements EventHandler<Boolean> { - private static ContainerMapping mapping; + private static SCMContainerManager mapping; private static MockNodeManager nodeManager; private static BlockManagerImpl blockManager; private static File testDir; @@ -83,7 +83,7 @@ public class TestBlockManager implements EventHandler<Boolean> { throw new IOException("Unable to create test directory path"); } nodeManager = new MockNodeManager(true, 10); - mapping = new ContainerMapping(conf, nodeManager, 128, eventQueue); + mapping = new SCMContainerManager(conf, nodeManager, 128, eventQueue); blockManager = new BlockManagerImpl(conf, nodeManager, mapping, eventQueue); eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java index 9f0e336..0812027 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java @@ -19,8 +19,8 @@ package org.apache.hadoop.hdds.scm.block; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; -import org.apache.hadoop.hdds.scm.container.Mapping; +import org.apache.hadoop.hdds.scm.container.SCMContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; @@ -72,7 +72,7 @@ public class TestDeletedBlockLog { private static DeletedBlockLogImpl deletedBlockLog; private OzoneConfiguration conf; private File testDir; - private Mapping containerManager; + private ContainerManager containerManager; private List<DatanodeDetails> dnList; @Before @@ -82,7 +82,7 @@ public class TestDeletedBlockLog { conf = new OzoneConfiguration(); conf.setInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 20); conf.set(OZONE_METADATA_DIRS, testDir.getAbsolutePath()); - containerManager = Mockito.mock(ContainerMapping.class); + containerManager = Mockito.mock(SCMContainerManager.class); deletedBlockLog = new DeletedBlockLogImpl(conf, containerManager); dnList = new ArrayList<>(3); setupContainerManager(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index 38050c9..3917d39 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -50,7 +50,7 @@ public class TestCloseContainerEventHandler { private static Configuration configuration; private static MockNodeManager nodeManager; - private static ContainerMapping mapping; + private static SCMContainerManager mapping; private static long size; private static File testDir; private static EventQueue eventQueue; @@ -65,7 +65,7 @@ public class TestCloseContainerEventHandler { configuration .set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath()); nodeManager = new MockNodeManager(true, 10); - mapping = new ContainerMapping(configuration, nodeManager, 128, + mapping = new SCMContainerManager(configuration, nodeManager, 128, new EventQueue()); eventQueue = new EventQueue(); eventQueue.addHandler(CLOSE_CONTAINER, http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java deleted file mode 100644 index f9a881e..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java +++ /dev/null @@ -1,380 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.hdds.scm.container; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.container.common.SCMTestUtils; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.test.LambdaTestUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.NavigableSet; -import java.util.Random; -import java.util.Set; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -/** - * Tests for Container Mapping. - */ -public class TestContainerMapping { - private static ContainerMapping mapping; - private static MockNodeManager nodeManager; - private static File testDir; - private static XceiverClientManager xceiverClientManager; - private static String containerOwner = "OZONE"; - private static Random random; - - private static final long TIMEOUT = 10000; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - @BeforeClass - public static void setUp() throws Exception { - Configuration conf = SCMTestUtils.getConf(); - - testDir = GenericTestUtils - .getTestDir(TestContainerMapping.class.getSimpleName()); - conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, - testDir.getAbsolutePath()); - conf.setTimeDuration( - ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT, - TIMEOUT, - TimeUnit.MILLISECONDS); - boolean folderExisted = testDir.exists() || testDir.mkdirs(); - if (!folderExisted) { - throw new IOException("Unable to create test directory path"); - } - nodeManager = new MockNodeManager(true, 10); - mapping = new ContainerMapping(conf, nodeManager, 128, - new EventQueue()); - xceiverClientManager = new XceiverClientManager(conf); - random = new Random(); - } - - @AfterClass - public static void cleanup() throws IOException { - if(mapping != null) { - mapping.close(); - } - FileUtil.fullyDelete(testDir); - } - - @Before - public void clearChillMode() { - nodeManager.setChillmode(false); - } - - @Test - public void testallocateContainer() throws Exception { - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - Assert.assertNotNull(containerInfo); - } - - @Test - public void testallocateContainerDistributesAllocation() throws Exception { - /* This is a lame test, we should really be testing something like - z-score or make sure that we don't have 3sigma kind of events. Too lazy - to write all that code. This test very lamely tests if we have more than - 5 separate nodes from the list of 10 datanodes that got allocated a - container. - */ - Set<UUID> pipelineList = new TreeSet<>(); - for (int x = 0; x < 30; x++) { - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - - Assert.assertNotNull(containerInfo); - Assert.assertNotNull(containerInfo.getPipeline()); - pipelineList.add(containerInfo.getPipeline().getLeader() - .getUuid()); - } - Assert.assertTrue(pipelineList.size() > 5); - } - - @Test - public void testGetContainer() throws IOException { - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - Pipeline pipeline = containerInfo.getPipeline(); - Assert.assertNotNull(pipeline); - Pipeline newPipeline = containerInfo.getPipeline(); - Assert.assertEquals(pipeline.getLeader().getUuid(), - newPipeline.getLeader().getUuid()); - } - - @Test - public void testGetContainerWithPipeline() throws Exception { - ContainerWithPipeline containerWithPipeline = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - ContainerInfo contInfo = containerWithPipeline.getContainerInfo(); - // Add dummy replicas for container. - DatanodeDetails dn1 = DatanodeDetails.newBuilder() - .setHostName("host1") - .setIpAddress("1.1.1.1") - .setUuid(UUID.randomUUID().toString()).build(); - DatanodeDetails dn2 = DatanodeDetails.newBuilder() - .setHostName("host2") - .setIpAddress("2.2.2.2") - .setUuid(UUID.randomUUID().toString()).build(); - mapping - .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CREATE); - mapping.updateContainerState(contInfo.getContainerID(), - LifeCycleEvent.CREATED); - mapping.updateContainerState(contInfo.getContainerID(), - LifeCycleEvent.FINALIZE); - mapping - .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CLOSE); - ContainerInfo finalContInfo = contInfo; - LambdaTestUtils.intercept(SCMException.class, "No entry exist for " - + "containerId:", () -> mapping.getContainerWithPipeline( - finalContInfo.getContainerID())); - - mapping.getStateManager().getContainerStateMap() - .addContainerReplica(contInfo.containerID(), dn1, dn2); - - contInfo = mapping.getContainer(contInfo.getContainerID()); - Assert.assertEquals(contInfo.getState(), LifeCycleState.CLOSED); - Pipeline pipeline = containerWithPipeline.getPipeline(); - mapping.getPipelineSelector().finalizePipeline(pipeline); - - ContainerWithPipeline containerWithPipeline2 = mapping - .getContainerWithPipeline(contInfo.getContainerID()); - pipeline = containerWithPipeline2.getPipeline(); - Assert.assertNotEquals(containerWithPipeline, containerWithPipeline2); - Assert.assertNotNull("Pipeline should not be null", pipeline); - Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn1.getHostName())); - Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn2.getHostName())); - } - - @Test - public void testgetNoneExistentContainer() throws IOException { - thrown.expectMessage("Specified key does not exist."); - mapping.getContainer(random.nextLong()); - } - - @Test - public void testContainerCreationLeaseTimeout() throws IOException, - InterruptedException { - nodeManager.setChillmode(false); - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - mapping.updateContainerState(containerInfo.getContainerInfo() - .getContainerID(), HddsProtos.LifeCycleEvent.CREATE); - Thread.sleep(TIMEOUT + 1000); - - NavigableSet<ContainerID> deleteContainers = mapping.getStateManager() - .getMatchingContainerIDs( - "OZONE", - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - HddsProtos.LifeCycleState.DELETING); - Assert.assertTrue(deleteContainers - .contains(containerInfo.getContainerInfo().containerID())); - - thrown.expect(IOException.class); - thrown.expectMessage("Lease Exception"); - mapping - .updateContainerState(containerInfo.getContainerInfo().getContainerID(), - HddsProtos.LifeCycleEvent.CREATED); - } - - @Test - public void testFullContainerReport() throws Exception { - ContainerInfo info = createContainer(); - DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); - List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports = - new ArrayList<>(); - StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = - StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); - ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2") - .setSize(5368709120L) - .setUsed(2000000000L) - .setKeyCount(100000000L) - .setReadCount(100000000L) - .setWriteCount(100000000L) - .setReadBytes(2000000000L) - .setWriteBytes(2000000000L) - .setContainerID(info.getContainerID()) - .setDeleteTransactionId(0); - - reports.add(ciBuilder.build()); - - ContainerReportsProto.Builder crBuilder = ContainerReportsProto - .newBuilder(); - crBuilder.addAllReports(reports); - - mapping.processContainerReports(datanodeDetails, crBuilder.build(), false); - - ContainerInfo updatedContainer = - mapping.getContainer(info.getContainerID()); - Assert.assertEquals(100000000L, - updatedContainer.getNumberOfKeys()); - Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes()); - - for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { - LambdaTestUtils.intercept(SCMException.class, "No entry " - + "exist for containerId:", () -> mapping.getStateManager() - .getContainerReplicas(ContainerID.valueof(c.getContainerID()))); - } - - mapping.processContainerReports(TestUtils.randomDatanodeDetails(), - crBuilder.build(), true); - for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { - Assert.assertTrue(mapping.getStateManager().getContainerReplicas( - ContainerID.valueof(c.getContainerID())).size() > 0); - } - } - - @Test - public void testListContainerAfterReport() throws Exception { - ContainerInfo info1 = createContainer(); - ContainerInfo info2 = createContainer(); - DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); - List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports = - new ArrayList<>(); - StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = - StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); - long cID1 = info1.getContainerID(); - long cID2 = info2.getContainerID(); - ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2") - .setSize(1000000000L) - .setUsed(987654321L) - .setKeyCount(100000000L) - .setReadBytes(1000000000L) - .setWriteBytes(1000000000L) - .setContainerID(cID1); - reports.add(ciBuilder.build()); - - ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9") - .setSize(1000000000L) - .setUsed(123456789L) - .setKeyCount(200000000L) - .setReadBytes(3000000000L) - .setWriteBytes(4000000000L) - .setContainerID(cID2); - reports.add(ciBuilder.build()); - - ContainerReportsProto.Builder crBuilder = ContainerReportsProto - .newBuilder(); - crBuilder.addAllReports(reports); - - mapping.processContainerReports(datanodeDetails, crBuilder.build(), false); - - List<ContainerInfo> list = mapping.listContainer(0, 50); - Assert.assertEquals(2, list.stream().filter( - x -> x.getContainerID() == cID1 || x.getContainerID() == cID2).count()); - Assert.assertEquals(300000000L, list.stream().filter( - x -> x.getContainerID() == cID1 || x.getContainerID() == cID2) - .mapToLong(x -> x.getNumberOfKeys()).sum()); - Assert.assertEquals(1111111110L, list.stream().filter( - x -> x.getContainerID() == cID1 || x.getContainerID() == cID2) - .mapToLong(x -> x.getUsedBytes()).sum()); - } - - @Test - public void testCloseContainer() throws IOException { - ContainerInfo info = createContainer(); - mapping.updateContainerState(info.getContainerID(), - HddsProtos.LifeCycleEvent.FINALIZE); - NavigableSet<ContainerID> pendingCloseContainers = mapping.getStateManager() - .getMatchingContainerIDs( - containerOwner, - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - HddsProtos.LifeCycleState.CLOSING); - Assert.assertTrue(pendingCloseContainers.contains(info.containerID())); - mapping.updateContainerState(info.getContainerID(), - HddsProtos.LifeCycleEvent.CLOSE); - NavigableSet<ContainerID> closeContainers = mapping.getStateManager() - .getMatchingContainerIDs( - containerOwner, - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - HddsProtos.LifeCycleState.CLOSED); - Assert.assertTrue(closeContainers.contains(info.containerID())); - } - - /** - * Creates a container with the given name in ContainerMapping. - * @throws IOException - */ - private ContainerInfo createContainer() - throws IOException { - nodeManager.setChillmode(false); - ContainerWithPipeline containerWithPipeline = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - ContainerInfo containerInfo = containerWithPipeline.getContainerInfo(); - mapping.updateContainerState(containerInfo.getContainerID(), - HddsProtos.LifeCycleEvent.CREATE); - mapping.updateContainerState(containerInfo.getContainerID(), - HddsProtos.LifeCycleEvent.CREATED); - return containerInfo; - } - - @Test - public void testFlushAllContainers() throws IOException { - ContainerInfo info = createContainer(); - List<ContainerInfo> containers = mapping.getStateManager() - .getAllContainers(); - Assert.assertTrue(containers.size() > 0); - mapping.flushContainerInfo(); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java index f79ae1e..7f32be5 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java @@ -74,10 +74,10 @@ public class TestContainerReportHandler implements EventPublisher { public void test() throws IOException { //GIVEN OzoneConfiguration conf = new OzoneConfiguration(); - Mapping mapping = Mockito.mock(Mapping.class); + ContainerManager containerManager = Mockito.mock(ContainerManager.class); PipelineSelector selector = Mockito.mock(PipelineSelector.class); - when(mapping.getContainer(anyLong())) + when(containerManager.getContainer(anyLong())) .thenAnswer( (Answer<ContainerInfo>) invocation -> new Builder() @@ -88,15 +88,15 @@ public class TestContainerReportHandler implements EventPublisher { ); ContainerStateManager containerStateManager = - new ContainerStateManager(conf, mapping, selector); + new ContainerStateManager(conf, containerManager, selector); - when(mapping.getStateManager()).thenReturn(containerStateManager); + when(containerManager.getStateManager()).thenReturn(containerStateManager); ReplicationActivityStatus replicationActivityStatus = new ReplicationActivityStatus(); ContainerReportHandler reportHandler = - new ContainerReportHandler(mapping, nodeManager, + new ContainerReportHandler(containerManager, nodeManager, replicationActivityStatus); DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java index b857740..785753b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java @@ -41,7 +41,7 @@ public class TestContainerStateManager { @Before public void init() throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); - Mapping mapping = Mockito.mock(Mapping.class); + ContainerManager mapping = Mockito.mock(ContainerManager.class); PipelineSelector selector = Mockito.mock(PipelineSelector.class); containerStateManager = new ContainerStateManager(conf, mapping, selector); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java new file mode 100644 index 0000000..b067ac9 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java @@ -0,0 +1,378 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.container; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; +import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.container.common.SCMTestUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableSet; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +/** + * Tests for Container ContainerManager. + */ +public class TestSCMContainerManager { + private static SCMContainerManager containerManager; + private static MockNodeManager nodeManager; + private static File testDir; + private static XceiverClientManager xceiverClientManager; + private static String containerOwner = "OZONE"; + private static Random random; + + private static final long TIMEOUT = 10000; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = SCMTestUtils.getConf(); + + testDir = GenericTestUtils + .getTestDir(TestSCMContainerManager.class.getSimpleName()); + conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, + testDir.getAbsolutePath()); + conf.setTimeDuration( + ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT, + TIMEOUT, + TimeUnit.MILLISECONDS); + boolean folderExisted = testDir.exists() || testDir.mkdirs(); + if (!folderExisted) { + throw new IOException("Unable to create test directory path"); + } + nodeManager = new MockNodeManager(true, 10); + containerManager = new SCMContainerManager(conf, nodeManager, 128, + new EventQueue()); + xceiverClientManager = new XceiverClientManager(conf); + random = new Random(); + } + + @AfterClass + public static void cleanup() throws IOException { + if(containerManager != null) { + containerManager.close(); + } + FileUtil.fullyDelete(testDir); + } + + @Before + public void clearChillMode() { + nodeManager.setChillmode(false); + } + + @Test + public void testallocateContainer() throws Exception { + ContainerWithPipeline containerInfo = containerManager.allocateContainer( + xceiverClientManager.getType(), + xceiverClientManager.getFactor(), + containerOwner); + Assert.assertNotNull(containerInfo); + } + + @Test + public void testallocateContainerDistributesAllocation() throws Exception { + /* This is a lame test, we should really be testing something like + z-score or make sure that we don't have 3sigma kind of events. Too lazy + to write all that code. This test very lamely tests if we have more than + 5 separate nodes from the list of 10 datanodes that got allocated a + container. + */ + Set<UUID> pipelineList = new TreeSet<>(); + for (int x = 0; x < 30; x++) { + ContainerWithPipeline containerInfo = containerManager.allocateContainer( + xceiverClientManager.getType(), + xceiverClientManager.getFactor(), + containerOwner); + + Assert.assertNotNull(containerInfo); + Assert.assertNotNull(containerInfo.getPipeline()); + pipelineList.add(containerInfo.getPipeline().getLeader() + .getUuid()); + } + Assert.assertTrue(pipelineList.size() > 5); + } + + @Test + public void testGetContainer() throws IOException { + ContainerWithPipeline containerInfo = containerManager.allocateContainer( + xceiverClientManager.getType(), + xceiverClientManager.getFactor(), + containerOwner); + Pipeline pipeline = containerInfo.getPipeline(); + Assert.assertNotNull(pipeline); + Pipeline newPipeline = containerInfo.getPipeline(); + Assert.assertEquals(pipeline.getLeader().getUuid(), + newPipeline.getLeader().getUuid()); + } + + @Test + public void testGetContainerWithPipeline() throws Exception { + ContainerWithPipeline containerWithPipeline = containerManager + .allocateContainer(xceiverClientManager.getType(), + xceiverClientManager.getFactor(), containerOwner); + ContainerInfo contInfo = containerWithPipeline.getContainerInfo(); + // Add dummy replicas for container. + DatanodeDetails dn1 = DatanodeDetails.newBuilder() + .setHostName("host1") + .setIpAddress("1.1.1.1") + .setUuid(UUID.randomUUID().toString()).build(); + DatanodeDetails dn2 = DatanodeDetails.newBuilder() + .setHostName("host2") + .setIpAddress("2.2.2.2") + .setUuid(UUID.randomUUID().toString()).build(); + containerManager + .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CREATE); + containerManager.updateContainerState(contInfo.getContainerID(), + LifeCycleEvent.CREATED); + containerManager.updateContainerState(contInfo.getContainerID(), + LifeCycleEvent.FINALIZE); + containerManager + .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CLOSE); + ContainerInfo finalContInfo = contInfo; + LambdaTestUtils.intercept(SCMException.class, "No entry exist for " + + "containerId:", () -> containerManager.getContainerWithPipeline( + finalContInfo.getContainerID())); + + containerManager.getStateManager().getContainerStateMap() + .addContainerReplica(contInfo.containerID(), dn1, dn2); + + contInfo = containerManager.getContainer(contInfo.getContainerID()); + Assert.assertEquals(contInfo.getState(), LifeCycleState.CLOSED); + Pipeline pipeline = containerWithPipeline.getPipeline(); + containerManager.getPipelineSelector().finalizePipeline(pipeline); + + ContainerWithPipeline containerWithPipeline2 = containerManager + .getContainerWithPipeline(contInfo.getContainerID()); + pipeline = containerWithPipeline2.getPipeline(); + Assert.assertNotEquals(containerWithPipeline, containerWithPipeline2); + Assert.assertNotNull("Pipeline should not be null", pipeline); + Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn1.getHostName())); + Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn2.getHostName())); + } + + @Test + public void testgetNoneExistentContainer() throws IOException { + thrown.expectMessage("Specified key does not exist."); + containerManager.getContainer(random.nextLong()); + } + + @Test + public void testContainerCreationLeaseTimeout() throws IOException, + InterruptedException { + nodeManager.setChillmode(false); + ContainerWithPipeline containerInfo = containerManager.allocateContainer( + xceiverClientManager.getType(), + xceiverClientManager.getFactor(), + containerOwner); + containerManager.updateContainerState(containerInfo.getContainerInfo() + .getContainerID(), HddsProtos.LifeCycleEvent.CREATE); + Thread.sleep(TIMEOUT + 1000); + + NavigableSet<ContainerID> deleteContainers = containerManager + .getStateManager().getMatchingContainerIDs("OZONE", + xceiverClientManager.getType(), + xceiverClientManager.getFactor(), + HddsProtos.LifeCycleState.DELETING); + Assert.assertTrue(deleteContainers + .contains(containerInfo.getContainerInfo().containerID())); + + thrown.expect(IOException.class); + thrown.expectMessage("Lease Exception"); + containerManager + .updateContainerState(containerInfo.getContainerInfo().getContainerID(), + HddsProtos.LifeCycleEvent.CREATED); + } + + @Test + public void testFullContainerReport() throws Exception { + ContainerInfo info = createContainer(); + DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); + List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports = + new ArrayList<>(); + StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = + StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); + ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2") + .setSize(5368709120L) + .setUsed(2000000000L) + .setKeyCount(100000000L) + .setReadCount(100000000L) + .setWriteCount(100000000L) + .setReadBytes(2000000000L) + .setWriteBytes(2000000000L) + .setContainerID(info.getContainerID()) + .setDeleteTransactionId(0); + + reports.add(ciBuilder.build()); + + ContainerReportsProto.Builder crBuilder = ContainerReportsProto + .newBuilder(); + crBuilder.addAllReports(reports); + + containerManager.processContainerReports( + datanodeDetails, crBuilder.build(), false); + + ContainerInfo updatedContainer = + containerManager.getContainer(info.getContainerID()); + Assert.assertEquals(100000000L, + updatedContainer.getNumberOfKeys()); + Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes()); + + for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { + LambdaTestUtils.intercept(SCMException.class, "No entry " + + "exist for containerId:", () -> containerManager.getStateManager() + .getContainerReplicas(ContainerID.valueof(c.getContainerID()))); + } + + containerManager.processContainerReports(TestUtils.randomDatanodeDetails(), + crBuilder.build(), true); + for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { + Assert.assertTrue(containerManager.getStateManager() + .getContainerReplicas( + ContainerID.valueof(c.getContainerID())).size() > 0); + } + } + + @Test + public void testListContainerAfterReport() throws Exception { + ContainerInfo info1 = createContainer(); + ContainerInfo info2 = createContainer(); + DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); + List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports = + new ArrayList<>(); + StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = + StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); + long cID1 = info1.getContainerID(); + long cID2 = info2.getContainerID(); + ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2") + .setSize(1000000000L) + .setUsed(987654321L) + .setKeyCount(100000000L) + .setReadBytes(1000000000L) + .setWriteBytes(1000000000L) + .setContainerID(cID1); + reports.add(ciBuilder.build()); + + ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9") + .setSize(1000000000L) + .setUsed(123456789L) + .setKeyCount(200000000L) + .setReadBytes(3000000000L) + .setWriteBytes(4000000000L) + .setContainerID(cID2); + reports.add(ciBuilder.build()); + + ContainerReportsProto.Builder crBuilder = ContainerReportsProto + .newBuilder(); + crBuilder.addAllReports(reports); + + containerManager.processContainerReports( + datanodeDetails, crBuilder.build(), false); + + List<ContainerInfo> list = containerManager.listContainer(0, 50); + Assert.assertEquals(2, list.stream().filter( + x -> x.getContainerID() == cID1 || x.getContainerID() == cID2).count()); + Assert.assertEquals(300000000L, list.stream().filter( + x -> x.getContainerID() == cID1 || x.getContainerID() == cID2) + .mapToLong(x -> x.getNumberOfKeys()).sum()); + Assert.assertEquals(1111111110L, list.stream().filter( + x -> x.getContainerID() == cID1 || x.getContainerID() == cID2) + .mapToLong(x -> x.getUsedBytes()).sum()); + } + + @Test + public void testCloseContainer() throws IOException { + ContainerInfo info = createContainer(); + containerManager.updateContainerState(info.getContainerID(), + HddsProtos.LifeCycleEvent.FINALIZE); + NavigableSet<ContainerID> pendingCloseContainers = containerManager + .getStateManager().getMatchingContainerIDs(containerOwner, + xceiverClientManager.getType(), + xceiverClientManager.getFactor(), + HddsProtos.LifeCycleState.CLOSING); + Assert.assertTrue(pendingCloseContainers.contains(info.containerID())); + containerManager.updateContainerState(info.getContainerID(), + HddsProtos.LifeCycleEvent.CLOSE); + NavigableSet<ContainerID> closeContainers = containerManager + .getStateManager().getMatchingContainerIDs(containerOwner, + xceiverClientManager.getType(), + xceiverClientManager.getFactor(), + HddsProtos.LifeCycleState.CLOSED); + Assert.assertTrue(closeContainers.contains(info.containerID())); + } + + /** + * Creates a container with the given name in SCMContainerManager. + * @throws IOException + */ + private ContainerInfo createContainer() + throws IOException { + nodeManager.setChillmode(false); + ContainerWithPipeline containerWithPipeline = containerManager + .allocateContainer(xceiverClientManager.getType(), + xceiverClientManager.getFactor(), containerOwner); + ContainerInfo containerInfo = containerWithPipeline.getContainerInfo(); + containerManager.updateContainerState(containerInfo.getContainerID(), + HddsProtos.LifeCycleEvent.CREATE); + containerManager.updateContainerState(containerInfo.getContainerID(), + HddsProtos.LifeCycleEvent.CREATED); + return containerInfo; + } + + @Test + public void testFlushAllContainers() throws IOException { + ContainerInfo info = createContainer(); + List<ContainerInfo> containers = containerManager.getStateManager() + .getAllContainers(); + Assert.assertTrue(containers.size() > 0); + containerManager.flushContainerInfo(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index cd79d44..175180a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.placement.algorithms .ContainerPlacementPolicy; @@ -97,12 +97,13 @@ public class TestContainerPlacement { return nodeManager; } - ContainerMapping createContainerManager(Configuration config, + SCMContainerManager createContainerManager(Configuration config, NodeManager scmNodeManager) throws IOException { EventQueue eventQueue = new EventQueue(); final int cacheSize = config.getInt(OZONE_SCM_DB_CACHE_SIZE_MB, OZONE_SCM_DB_CACHE_SIZE_DEFAULT); - return new ContainerMapping(config, scmNodeManager, cacheSize, eventQueue); + return new SCMContainerManager(config, scmNodeManager, cacheSize, + eventQueue); } @@ -131,7 +132,7 @@ public class TestContainerPlacement { SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class); SCMNodeManager nodeManager = createNodeManager(conf); - ContainerMapping containerManager = + SCMContainerManager containerManager = createContainerManager(conf, nodeManager); List<DatanodeDetails> datanodes = TestUtils.getListOfRegisteredDatanodeDetails(nodeManager, nodeCount); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java index 7bba032..9d6927d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java @@ -31,8 +31,8 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolPro import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerStateManager; -import org.apache.hadoop.hdds.scm.container.Mapping; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest; @@ -69,7 +69,7 @@ public class TestDeadNodeHandler { public void setup() throws IOException { OzoneConfiguration conf = new OzoneConfiguration(); containerStateManager = new ContainerStateManager(conf, - Mockito.mock(Mapping.class), + Mockito.mock(ContainerManager.class), Mockito.mock(PipelineSelector.class)); eventQueue = new EventQueue(); nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java index 56c3830..27195a1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler .CloseContainerRetryableReq; import org.apache.hadoop.hdds.scm.container.CloseContainerWatcher; import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.server.events.EventHandler; @@ -61,8 +61,8 @@ public class TestCloseContainerWatcher implements EventHandler<ContainerID> { private static EventWatcher<CloseContainerRetryableReq, CloseContainerStatus> watcher; private static LeaseManager<Long> leaseManager; - private static ContainerMapping containerMapping = Mockito - .mock(ContainerMapping.class); + private static SCMContainerManager containerManager = Mockito + .mock(SCMContainerManager.class); private static EventQueue queue; @Rule public Timeout timeout = new Timeout(1000*15); @@ -230,7 +230,7 @@ public class TestCloseContainerWatcher implements EventHandler<ContainerID> { time); leaseManager.start(); watcher = new CloseContainerWatcher(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ, - SCMEvents.CLOSE_CONTAINER_STATUS, leaseManager, containerMapping); + SCMEvents.CLOSE_CONTAINER_STATUS, leaseManager, containerManager); queue = new EventQueue(); watcher.start(queue); } @@ -274,8 +274,8 @@ public class TestCloseContainerWatcher implements EventHandler<ContainerID> { throws IOException { ContainerInfo containerInfo = Mockito.mock(ContainerInfo.class); ContainerInfo containerInfo2 = Mockito.mock(ContainerInfo.class); - when(containerMapping.getContainer(id1)).thenReturn(containerInfo); - when(containerMapping.getContainer(id2)).thenReturn(containerInfo2); + when(containerManager.getContainer(id1)).thenReturn(containerInfo); + when(containerManager.getContainer(id2)).thenReturn(containerInfo2); when(containerInfo.isContainerOpen()).thenReturn(true); when(containerInfo2.isContainerOpen()).thenReturn(isOpen); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java index a8f7e01..ed8b1e3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java @@ -55,7 +55,7 @@ public class TestContainerStateManagerIntegration { private MiniOzoneCluster cluster; private XceiverClientManager xceiverClientManager; private StorageContainerManager scm; - private ContainerMapping scmContainerMapping; + private ContainerManager containerManager; private ContainerStateManager containerStateManager; private PipelineSelector selector; private String containerOwner = "OZONE"; @@ -69,9 +69,9 @@ public class TestContainerStateManagerIntegration { cluster.waitTobeOutOfChillMode(); xceiverClientManager = new XceiverClientManager(conf); scm = cluster.getStorageContainerManager(); - scmContainerMapping = (ContainerMapping) scm.getScmContainerManager(); - containerStateManager = scmContainerMapping.getStateManager(); - selector = scmContainerMapping.getPipelineSelector(); + containerManager = scm.getContainerManager(); + containerStateManager = containerManager.getStateManager(); + selector = containerManager.getPipelineSelector(); } @After @@ -128,7 +128,7 @@ public class TestContainerStateManagerIntegration { xceiverClientManager.getFactor(), containerOwner); containers.add(container.getContainerInfo()); if (i >= 5) { - scm.getScmContainerManager().updateContainerState(container + scm.getContainerManager().updateContainerState(container .getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATE); } @@ -137,7 +137,7 @@ public class TestContainerStateManagerIntegration { // New instance of ContainerStateManager should load all the containers in // container store. ContainerStateManager stateManager = - new ContainerStateManager(conf, scmContainerMapping, selector); + new ContainerStateManager(conf, containerManager, selector); int matchCount = stateManager .getMatchingContainerIDs(containerOwner, xceiverClientManager.getType(), xceiverClientManager.getFactor(), @@ -154,10 +154,10 @@ public class TestContainerStateManagerIntegration { ContainerWithPipeline container1 = scm.getClientProtocolServer(). allocateContainer(xceiverClientManager.getType(), xceiverClientManager.getFactor(), containerOwner); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATE); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATED); @@ -179,10 +179,10 @@ public class TestContainerStateManagerIntegration { Assert.assertEquals(container2.getContainerInfo().getContainerID(), info.getContainerID()); - scmContainerMapping + containerManager .updateContainerState(container2.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATE); - scmContainerMapping + containerManager .updateContainerState(container2.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATED); @@ -216,7 +216,7 @@ public class TestContainerStateManagerIntegration { HddsProtos.LifeCycleState.ALLOCATED).size(); Assert.assertEquals(1, containers); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATE); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -224,7 +224,7 @@ public class TestContainerStateManagerIntegration { HddsProtos.LifeCycleState.CREATING).size(); Assert.assertEquals(1, containers); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATED); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -232,7 +232,7 @@ public class TestContainerStateManagerIntegration { HddsProtos.LifeCycleState.OPEN).size(); Assert.assertEquals(1, containers); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.FINALIZE); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -240,7 +240,7 @@ public class TestContainerStateManagerIntegration { HddsProtos.LifeCycleState.CLOSING).size(); Assert.assertEquals(1, containers); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CLOSE); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -248,7 +248,7 @@ public class TestContainerStateManagerIntegration { HddsProtos.LifeCycleState.CLOSED).size(); Assert.assertEquals(1, containers); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.DELETE); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -256,7 +256,7 @@ public class TestContainerStateManagerIntegration { HddsProtos.LifeCycleState.DELETING).size(); Assert.assertEquals(1, containers); - scmContainerMapping + containerManager .updateContainerState(container1.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CLEANUP); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -270,10 +270,10 @@ public class TestContainerStateManagerIntegration { .allocateContainer( xceiverClientManager.getType(), xceiverClientManager.getFactor(), containerOwner); - scmContainerMapping + containerManager .updateContainerState(container2.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATE); - scmContainerMapping + containerManager .updateContainerState(container2.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.TIMEOUT); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -287,16 +287,16 @@ public class TestContainerStateManagerIntegration { .allocateContainer( xceiverClientManager.getType(), xceiverClientManager.getFactor(), containerOwner); - scmContainerMapping + containerManager .updateContainerState(container3.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATE); - scmContainerMapping + containerManager .updateContainerState(container3.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATED); - scmContainerMapping + containerManager .updateContainerState(container3.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.FINALIZE); - scmContainerMapping + containerManager .updateContainerState(container3.getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CLOSE); containers = containerStateManager.getMatchingContainerIDs(containerOwner, @@ -310,10 +310,10 @@ public class TestContainerStateManagerIntegration { ContainerWithPipeline container1 = scm.getClientProtocolServer() .allocateContainer(xceiverClientManager.getType(), xceiverClientManager.getFactor(), containerOwner); - scmContainerMapping.updateContainerState(container1 + containerManager.updateContainerState(container1 .getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATE); - scmContainerMapping.updateContainerState(container1 + containerManager.updateContainerState(container1 .getContainerInfo().getContainerID(), HddsProtos.LifeCycleEvent.CREATED); @@ -330,8 +330,8 @@ public class TestContainerStateManagerIntegration { Assert.assertEquals(container1.getContainerInfo().getContainerID(), info.getContainerID()); - ContainerMapping containerMapping = - (ContainerMapping) scmContainerMapping; + SCMContainerManager containerMapping = + (SCMContainerManager) containerManager; // manually trigger a flush, this will persist the allocated bytes value // to disk containerMapping.flushContainerInfo(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java index e61a909..c0a6989 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers .ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID; @@ -51,7 +51,7 @@ public class TestNode2PipelineMap { private static StorageContainerManager scm; private static ContainerWithPipeline ratisContainer; private static ContainerStateMap stateMap; - private static ContainerMapping mapping; + private static ContainerManager containerManager; private static PipelineSelector pipelineSelector; /** @@ -65,10 +65,11 @@ public class TestNode2PipelineMap { cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); cluster.waitForClusterToBeReady(); scm = cluster.getStorageContainerManager(); - mapping = (ContainerMapping)scm.getScmContainerManager(); - stateMap = mapping.getStateManager().getContainerStateMap(); - ratisContainer = mapping.allocateContainer(RATIS, THREE, "testOwner"); - pipelineSelector = mapping.getPipelineSelector(); + containerManager = scm.getContainerManager(); + stateMap = containerManager.getStateManager().getContainerStateMap(); + ratisContainer = containerManager.allocateContainer( + RATIS, THREE, "testOwner"); + pipelineSelector = containerManager.getPipelineSelector(); } /** @@ -106,13 +107,13 @@ public class TestNode2PipelineMap { // Now close the container and it should not show up while fetching // containers by pipeline - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.CREATE); - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.CREATED); - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.FINALIZE); - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.CLOSE); Set<ContainerID> set2 = pipelineSelector.getOpenContainerIDsByPipeline( ratisContainer.getPipeline().getId()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java index 7e3969c..f3e1ece 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers .ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; @@ -52,7 +52,7 @@ public class TestNodeFailure { private static OzoneConfiguration conf; private static ContainerWithPipeline ratisContainer1; private static ContainerWithPipeline ratisContainer2; - private static ContainerMapping mapping; + private static ContainerManager containerManager; private static long timeForFailure; /** @@ -75,9 +75,11 @@ public class TestNodeFailure { .build(); cluster.waitForClusterToBeReady(); StorageContainerManager scm = cluster.getStorageContainerManager(); - mapping = (ContainerMapping)scm.getScmContainerManager(); - ratisContainer1 = mapping.allocateContainer(RATIS, THREE, "testOwner"); - ratisContainer2 = mapping.allocateContainer(RATIS, THREE, "testOwner"); + containerManager = scm.getContainerManager(); + ratisContainer1 = containerManager.allocateContainer( + RATIS, THREE, "testOwner"); + ratisContainer2 = containerManager.allocateContainer( + RATIS, THREE, "testOwner"); // At this stage, there should be 2 pipeline one with 1 open container each. // Try closing the both the pipelines, one with a closed container and // the other with an open container. @@ -113,12 +115,12 @@ public class TestNodeFailure { ratisContainer1.getPipeline().getLifeCycleState()); Assert.assertEquals(HddsProtos.LifeCycleState.OPEN, ratisContainer2.getPipeline().getLifeCycleState()); - Assert.assertNull( - mapping.getPipelineSelector().getPipeline(pipelineToFail.getId())); + Assert.assertNull(containerManager.getPipelineSelector() + .getPipeline(pipelineToFail.getId())); // Now restart the datanode and make sure that a new pipeline is created. cluster.restartHddsDatanode(dnToFail); ContainerWithPipeline ratisContainer3 = - mapping.allocateContainer(RATIS, THREE, "testOwner"); + containerManager.allocateContainer(RATIS, THREE, "testOwner"); //Assert that new container is not created from the ratis 2 pipeline Assert.assertNotEquals(ratisContainer3.getPipeline().getId(), ratisContainer2.getPipeline().getId()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java index b02eae2..7e6d5b4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers .ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; @@ -51,7 +51,7 @@ public class TestPipelineClose { private static ContainerWithPipeline ratisContainer1; private static ContainerWithPipeline ratisContainer2; private static ContainerStateMap stateMap; - private static ContainerMapping mapping; + private static ContainerManager containerManager; private static PipelineSelector pipelineSelector; /** @@ -65,11 +65,13 @@ public class TestPipelineClose { cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(6).build(); cluster.waitForClusterToBeReady(); scm = cluster.getStorageContainerManager(); - mapping = (ContainerMapping)scm.getScmContainerManager(); - stateMap = mapping.getStateManager().getContainerStateMap(); - ratisContainer1 = mapping.allocateContainer(RATIS, THREE, "testOwner"); - ratisContainer2 = mapping.allocateContainer(RATIS, THREE, "testOwner"); - pipelineSelector = mapping.getPipelineSelector(); + containerManager = scm.getContainerManager(); + stateMap = containerManager.getStateManager().getContainerStateMap(); + ratisContainer1 = containerManager + .allocateContainer(RATIS, THREE, "testOwner"); + ratisContainer2 = containerManager + .allocateContainer(RATIS, THREE, "testOwner"); + pipelineSelector = containerManager.getPipelineSelector(); // At this stage, there should be 2 pipeline one with 1 open container each. // Try closing the both the pipelines, one with a closed container and // the other with an open container. @@ -98,13 +100,13 @@ public class TestPipelineClose { // Now close the container and it should not show up while fetching // containers by pipeline - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.CREATE); - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.CREATED); - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.FINALIZE); - mapping + containerManager .updateContainerState(cId, HddsProtos.LifeCycleEvent.CLOSE); Set<ContainerID> setClosed = pipelineSelector.getOpenContainerIDsByPipeline( @@ -132,9 +134,9 @@ public class TestPipelineClose { Assert.assertEquals(1, setOpen.size()); long cId2 = ratisContainer2.getContainerInfo().getContainerID(); - mapping + containerManager .updateContainerState(cId2, HddsProtos.LifeCycleEvent.CREATE); - mapping + containerManager .updateContainerState(cId2, HddsProtos.LifeCycleEvent.CREATED); pipelineSelector.finalizePipeline(ratisContainer2.getPipeline()); Assert.assertEquals(ratisContainer2.getPipeline().getLifeCycleState(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java index fb94b3c..bac4022 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hdds.scm.pipeline; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; +import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.MiniOzoneCluster; @@ -46,8 +46,8 @@ public class TestSCMRestart { private static OzoneConfiguration conf; private static Pipeline ratisPipeline1; private static Pipeline ratisPipeline2; - private static ContainerMapping mapping; - private static ContainerMapping newMapping; + private static ContainerManager containerManager; + private static ContainerManager newContainerManager; /** * Create a MiniDFSCluster for testing. @@ -64,17 +64,17 @@ public class TestSCMRestart { .build(); cluster.waitForClusterToBeReady(); StorageContainerManager scm = cluster.getStorageContainerManager(); - mapping = (ContainerMapping)scm.getScmContainerManager(); - ratisPipeline1 = - mapping.allocateContainer(RATIS, THREE, "Owner1").getPipeline(); - ratisPipeline2 = - mapping.allocateContainer(RATIS, ONE, "Owner2").getPipeline(); + containerManager = scm.getContainerManager(); + ratisPipeline1 = containerManager.allocateContainer( + RATIS, THREE, "Owner1").getPipeline(); + ratisPipeline2 = containerManager.allocateContainer( + RATIS, ONE, "Owner2").getPipeline(); // At this stage, there should be 2 pipeline one with 1 open container // each. Try restarting the SCM and then discover that pipeline are in // correct state. cluster.restartStorageContainerManager(); - newMapping = (ContainerMapping)(cluster.getStorageContainerManager() - .getScmContainerManager()); + newContainerManager = cluster.getStorageContainerManager() + .getContainerManager(); } /** @@ -90,10 +90,10 @@ public class TestSCMRestart { @Test public void testPipelineWithScmRestart() throws IOException { // After restart make sure that the pipeline are still present - Pipeline ratisPipeline1AfterRestart = newMapping.getPipelineSelector() - .getPipeline(ratisPipeline1.getId()); - Pipeline ratisPipeline2AfterRestart = newMapping.getPipelineSelector() - .getPipeline(ratisPipeline2.getId()); + Pipeline ratisPipeline1AfterRestart = newContainerManager + .getPipelineSelector().getPipeline(ratisPipeline1.getId()); + Pipeline ratisPipeline2AfterRestart = newContainerManager + .getPipelineSelector().getPipeline(ratisPipeline2.getId()); Assert.assertNotSame(ratisPipeline1AfterRestart, ratisPipeline1); Assert.assertNotSame(ratisPipeline2AfterRestart, ratisPipeline2); Assert.assertEquals(ratisPipeline1AfterRestart, ratisPipeline1); @@ -111,9 +111,8 @@ public class TestSCMRestart { // Try creating a new ratis pipeline, it should be from the same pipeline // as was before restart - Pipeline newRatisPipeline = - newMapping.allocateContainer(RATIS, THREE, "Owner1") - .getPipeline(); + Pipeline newRatisPipeline = newContainerManager + .allocateContainer(RATIS, THREE, "Owner1").getPipeline(); Assert.assertEquals(newRatisPipeline.getId(), ratisPipeline1.getId()); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java index 7787b53..16e66ba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java @@ -44,13 +44,13 @@ public class OzoneTestUtils { StorageContainerManager scm) throws IOException { return performOperationOnKeyContainers((blockID) -> { try { - scm.getScmContainerManager() + scm.getContainerManager() .updateContainerState(blockID.getContainerID(), HddsProtos.LifeCycleEvent.FINALIZE); - scm.getScmContainerManager() + scm.getContainerManager() .updateContainerState(blockID.getContainerID(), HddsProtos.LifeCycleEvent.CLOSE); - Assert.assertFalse(scm.getScmContainerManager() + Assert.assertFalse(scm.getContainerManager() .getContainerWithPipeline(blockID.getContainerID()) .getContainerInfo().isContainerOpen()); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java index b4a0ba7..70ca1fa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java @@ -445,7 +445,7 @@ public class TestOzoneRestClient { // Sum the data size from chunks in Container via containerID // and localID, make sure the size equals to the actually value size. Pipeline pipeline = cluster.getStorageContainerManager() - .getScmContainerManager().getContainerWithPipeline(containerID) + .getContainerManager().getContainerWithPipeline(containerID) .getPipeline(); List<DatanodeDetails> datanodes = pipeline.getMachines(); Assert.assertEquals(datanodes.size(), 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/095c2696/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java index 83421b2..f606263 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java @@ -301,7 +301,7 @@ public class TestCloseContainerHandlingByClient { Assert.assertTrue(!containerIdList.isEmpty()); for (long containerID : containerIdList) { Pipeline pipeline = - cluster.getStorageContainerManager().getScmContainerManager() + cluster.getStorageContainerManager().getContainerManager() .getContainerWithPipeline(containerID).getPipeline(); pipelineList.add(pipeline); List<DatanodeDetails> datanodes = pipeline.getMachines(); @@ -349,7 +349,7 @@ public class TestCloseContainerHandlingByClient { new ArrayList<>(groupOutputStream.getLocationInfoList()); long containerID = locationInfos.get(0).getContainerID(); List<DatanodeDetails> datanodes = - cluster.getStorageContainerManager().getScmContainerManager() + cluster.getStorageContainerManager().getContainerManager() .getContainerWithPipeline(containerID).getPipeline().getMachines(); Assert.assertEquals(1, datanodes.size()); waitForContainerClose(keyName, key, HddsProtos.ReplicationType.STAND_ALONE); @@ -451,7 +451,7 @@ public class TestCloseContainerHandlingByClient { groupOutputStream.getLocationInfoList(); long containerID = locationInfos.get(0).getContainerID(); List<DatanodeDetails> datanodes = - cluster.getStorageContainerManager().getScmContainerManager() + cluster.getStorageContainerManager().getContainerManager() .getContainerWithPipeline(containerID).getPipeline().getMachines(); Assert.assertEquals(1, datanodes.size()); // move the container on the datanode to Closing state, this will ensure --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
