http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java deleted file mode 100644 index f9a881e..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java +++ /dev/null @@ -1,380 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.hdds.scm.container; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.container.common.SCMTestUtils; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.test.LambdaTestUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.NavigableSet; -import java.util.Random; -import java.util.Set; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -/** - * Tests for Container Mapping. - */ -public class TestContainerMapping { - private static ContainerMapping mapping; - private static MockNodeManager nodeManager; - private static File testDir; - private static XceiverClientManager xceiverClientManager; - private static String containerOwner = "OZONE"; - private static Random random; - - private static final long TIMEOUT = 10000; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - @BeforeClass - public static void setUp() throws Exception { - Configuration conf = SCMTestUtils.getConf(); - - testDir = GenericTestUtils - .getTestDir(TestContainerMapping.class.getSimpleName()); - conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, - testDir.getAbsolutePath()); - conf.setTimeDuration( - ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT, - TIMEOUT, - TimeUnit.MILLISECONDS); - boolean folderExisted = testDir.exists() || testDir.mkdirs(); - if (!folderExisted) { - throw new IOException("Unable to create test directory path"); - } - nodeManager = new MockNodeManager(true, 10); - mapping = new ContainerMapping(conf, nodeManager, 128, - new EventQueue()); - xceiverClientManager = new XceiverClientManager(conf); - random = new Random(); - } - - @AfterClass - public static void cleanup() throws IOException { - if(mapping != null) { - mapping.close(); - } - FileUtil.fullyDelete(testDir); - } - - @Before - public void clearChillMode() { - nodeManager.setChillmode(false); - } - - @Test - public void testallocateContainer() throws Exception { - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - Assert.assertNotNull(containerInfo); - } - - @Test - public void testallocateContainerDistributesAllocation() throws Exception { - /* This is a lame test, we should really be testing something like - z-score or make sure that we don't have 3sigma kind of events. Too lazy - to write all that code. This test very lamely tests if we have more than - 5 separate nodes from the list of 10 datanodes that got allocated a - container. - */ - Set<UUID> pipelineList = new TreeSet<>(); - for (int x = 0; x < 30; x++) { - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - - Assert.assertNotNull(containerInfo); - Assert.assertNotNull(containerInfo.getPipeline()); - pipelineList.add(containerInfo.getPipeline().getLeader() - .getUuid()); - } - Assert.assertTrue(pipelineList.size() > 5); - } - - @Test - public void testGetContainer() throws IOException { - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - Pipeline pipeline = containerInfo.getPipeline(); - Assert.assertNotNull(pipeline); - Pipeline newPipeline = containerInfo.getPipeline(); - Assert.assertEquals(pipeline.getLeader().getUuid(), - newPipeline.getLeader().getUuid()); - } - - @Test - public void testGetContainerWithPipeline() throws Exception { - ContainerWithPipeline containerWithPipeline = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - ContainerInfo contInfo = containerWithPipeline.getContainerInfo(); - // Add dummy replicas for container. - DatanodeDetails dn1 = DatanodeDetails.newBuilder() - .setHostName("host1") - .setIpAddress("1.1.1.1") - .setUuid(UUID.randomUUID().toString()).build(); - DatanodeDetails dn2 = DatanodeDetails.newBuilder() - .setHostName("host2") - .setIpAddress("2.2.2.2") - .setUuid(UUID.randomUUID().toString()).build(); - mapping - .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CREATE); - mapping.updateContainerState(contInfo.getContainerID(), - LifeCycleEvent.CREATED); - mapping.updateContainerState(contInfo.getContainerID(), - LifeCycleEvent.FINALIZE); - mapping - .updateContainerState(contInfo.getContainerID(), LifeCycleEvent.CLOSE); - ContainerInfo finalContInfo = contInfo; - LambdaTestUtils.intercept(SCMException.class, "No entry exist for " - + "containerId:", () -> mapping.getContainerWithPipeline( - finalContInfo.getContainerID())); - - mapping.getStateManager().getContainerStateMap() - .addContainerReplica(contInfo.containerID(), dn1, dn2); - - contInfo = mapping.getContainer(contInfo.getContainerID()); - Assert.assertEquals(contInfo.getState(), LifeCycleState.CLOSED); - Pipeline pipeline = containerWithPipeline.getPipeline(); - mapping.getPipelineSelector().finalizePipeline(pipeline); - - ContainerWithPipeline containerWithPipeline2 = mapping - .getContainerWithPipeline(contInfo.getContainerID()); - pipeline = containerWithPipeline2.getPipeline(); - Assert.assertNotEquals(containerWithPipeline, containerWithPipeline2); - Assert.assertNotNull("Pipeline should not be null", pipeline); - Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn1.getHostName())); - Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn2.getHostName())); - } - - @Test - public void testgetNoneExistentContainer() throws IOException { - thrown.expectMessage("Specified key does not exist."); - mapping.getContainer(random.nextLong()); - } - - @Test - public void testContainerCreationLeaseTimeout() throws IOException, - InterruptedException { - nodeManager.setChillmode(false); - ContainerWithPipeline containerInfo = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - mapping.updateContainerState(containerInfo.getContainerInfo() - .getContainerID(), HddsProtos.LifeCycleEvent.CREATE); - Thread.sleep(TIMEOUT + 1000); - - NavigableSet<ContainerID> deleteContainers = mapping.getStateManager() - .getMatchingContainerIDs( - "OZONE", - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - HddsProtos.LifeCycleState.DELETING); - Assert.assertTrue(deleteContainers - .contains(containerInfo.getContainerInfo().containerID())); - - thrown.expect(IOException.class); - thrown.expectMessage("Lease Exception"); - mapping - .updateContainerState(containerInfo.getContainerInfo().getContainerID(), - HddsProtos.LifeCycleEvent.CREATED); - } - - @Test - public void testFullContainerReport() throws Exception { - ContainerInfo info = createContainer(); - DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); - List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports = - new ArrayList<>(); - StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = - StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); - ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2") - .setSize(5368709120L) - .setUsed(2000000000L) - .setKeyCount(100000000L) - .setReadCount(100000000L) - .setWriteCount(100000000L) - .setReadBytes(2000000000L) - .setWriteBytes(2000000000L) - .setContainerID(info.getContainerID()) - .setDeleteTransactionId(0); - - reports.add(ciBuilder.build()); - - ContainerReportsProto.Builder crBuilder = ContainerReportsProto - .newBuilder(); - crBuilder.addAllReports(reports); - - mapping.processContainerReports(datanodeDetails, crBuilder.build(), false); - - ContainerInfo updatedContainer = - mapping.getContainer(info.getContainerID()); - Assert.assertEquals(100000000L, - updatedContainer.getNumberOfKeys()); - Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes()); - - for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { - LambdaTestUtils.intercept(SCMException.class, "No entry " - + "exist for containerId:", () -> mapping.getStateManager() - .getContainerReplicas(ContainerID.valueof(c.getContainerID()))); - } - - mapping.processContainerReports(TestUtils.randomDatanodeDetails(), - crBuilder.build(), true); - for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) { - Assert.assertTrue(mapping.getStateManager().getContainerReplicas( - ContainerID.valueof(c.getContainerID())).size() > 0); - } - } - - @Test - public void testListContainerAfterReport() throws Exception { - ContainerInfo info1 = createContainer(); - ContainerInfo info2 = createContainer(); - DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails(); - List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports = - new ArrayList<>(); - StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder = - StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); - long cID1 = info1.getContainerID(); - long cID2 = info2.getContainerID(); - ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2") - .setSize(1000000000L) - .setUsed(987654321L) - .setKeyCount(100000000L) - .setReadBytes(1000000000L) - .setWriteBytes(1000000000L) - .setContainerID(cID1); - reports.add(ciBuilder.build()); - - ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9") - .setSize(1000000000L) - .setUsed(123456789L) - .setKeyCount(200000000L) - .setReadBytes(3000000000L) - .setWriteBytes(4000000000L) - .setContainerID(cID2); - reports.add(ciBuilder.build()); - - ContainerReportsProto.Builder crBuilder = ContainerReportsProto - .newBuilder(); - crBuilder.addAllReports(reports); - - mapping.processContainerReports(datanodeDetails, crBuilder.build(), false); - - List<ContainerInfo> list = mapping.listContainer(0, 50); - Assert.assertEquals(2, list.stream().filter( - x -> x.getContainerID() == cID1 || x.getContainerID() == cID2).count()); - Assert.assertEquals(300000000L, list.stream().filter( - x -> x.getContainerID() == cID1 || x.getContainerID() == cID2) - .mapToLong(x -> x.getNumberOfKeys()).sum()); - Assert.assertEquals(1111111110L, list.stream().filter( - x -> x.getContainerID() == cID1 || x.getContainerID() == cID2) - .mapToLong(x -> x.getUsedBytes()).sum()); - } - - @Test - public void testCloseContainer() throws IOException { - ContainerInfo info = createContainer(); - mapping.updateContainerState(info.getContainerID(), - HddsProtos.LifeCycleEvent.FINALIZE); - NavigableSet<ContainerID> pendingCloseContainers = mapping.getStateManager() - .getMatchingContainerIDs( - containerOwner, - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - HddsProtos.LifeCycleState.CLOSING); - Assert.assertTrue(pendingCloseContainers.contains(info.containerID())); - mapping.updateContainerState(info.getContainerID(), - HddsProtos.LifeCycleEvent.CLOSE); - NavigableSet<ContainerID> closeContainers = mapping.getStateManager() - .getMatchingContainerIDs( - containerOwner, - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - HddsProtos.LifeCycleState.CLOSED); - Assert.assertTrue(closeContainers.contains(info.containerID())); - } - - /** - * Creates a container with the given name in ContainerMapping. - * @throws IOException - */ - private ContainerInfo createContainer() - throws IOException { - nodeManager.setChillmode(false); - ContainerWithPipeline containerWithPipeline = mapping.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), - containerOwner); - ContainerInfo containerInfo = containerWithPipeline.getContainerInfo(); - mapping.updateContainerState(containerInfo.getContainerID(), - HddsProtos.LifeCycleEvent.CREATE); - mapping.updateContainerState(containerInfo.getContainerID(), - HddsProtos.LifeCycleEvent.CREATED); - return containerInfo; - } - - @Test - public void testFlushAllContainers() throws IOException { - ContainerInfo info = createContainer(); - List<ContainerInfo> containers = mapping.getStateManager() - .getAllContainers(); - Assert.assertTrue(containers.size() > 0); - mapping.flushContainerInfo(); - } - -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java deleted file mode 100644 index f79ae1e..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java +++ /dev/null @@ -1,239 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.hdds.scm.container; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo - .Builder; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID; -import org.apache.hadoop.hdds.scm.container.replication - .ReplicationActivityStatus; -import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest; -import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector; -import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher - .ContainerReportFromDatanode; -import org.apache.hadoop.hdds.server.events.Event; -import org.apache.hadoop.hdds.server.events.EventPublisher; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import static org.mockito.Matchers.anyLong; -import org.mockito.Mockito; -import static org.mockito.Mockito.when; -import org.mockito.stubbing.Answer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Test the behaviour of the ContainerReportHandler. - */ -public class TestContainerReportHandler implements EventPublisher { - - private List<Object> publishedEvents = new ArrayList<>(); - private final NodeManager nodeManager = new MockNodeManager(true, 1); - - private static final Logger LOG = - LoggerFactory.getLogger(TestContainerReportHandler.class); - - @Before - public void resetEventCollector() { - publishedEvents.clear(); - } - - @Test - public void test() throws IOException { - //GIVEN - OzoneConfiguration conf = new OzoneConfiguration(); - Mapping mapping = Mockito.mock(Mapping.class); - PipelineSelector selector = Mockito.mock(PipelineSelector.class); - - when(mapping.getContainer(anyLong())) - .thenAnswer( - (Answer<ContainerInfo>) invocation -> - new Builder() - .setReplicationFactor(ReplicationFactor.THREE) - .setContainerID((Long) invocation.getArguments()[0]) - .setState(LifeCycleState.CLOSED) - .build() - ); - - ContainerStateManager containerStateManager = - new ContainerStateManager(conf, mapping, selector); - - when(mapping.getStateManager()).thenReturn(containerStateManager); - - ReplicationActivityStatus replicationActivityStatus = - new ReplicationActivityStatus(); - - ContainerReportHandler reportHandler = - new ContainerReportHandler(mapping, nodeManager, - replicationActivityStatus); - - DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); - DatanodeDetails dn2 = TestUtils.randomDatanodeDetails(); - DatanodeDetails dn3 = TestUtils.randomDatanodeDetails(); - DatanodeDetails dn4 = TestUtils.randomDatanodeDetails(); - nodeManager.addDatanodeInContainerMap(dn1.getUuid(), new HashSet<>()); - nodeManager.addDatanodeInContainerMap(dn2.getUuid(), new HashSet<>()); - nodeManager.addDatanodeInContainerMap(dn3.getUuid(), new HashSet<>()); - nodeManager.addDatanodeInContainerMap(dn4.getUuid(), new HashSet<>()); - PipelineSelector pipelineSelector = Mockito.mock(PipelineSelector.class); - - Pipeline pipeline = new Pipeline("leader", LifeCycleState.CLOSED, - ReplicationType.STAND_ALONE, ReplicationFactor.THREE, - PipelineID.randomId()); - - when(pipelineSelector.getReplicationPipeline(ReplicationType.STAND_ALONE, - ReplicationFactor.THREE)).thenReturn(pipeline); - - ContainerInfo cont1 = containerStateManager - .allocateContainer(pipelineSelector, ReplicationType.STAND_ALONE, - ReplicationFactor.THREE, "root").getContainerInfo(); - ContainerInfo cont2 = containerStateManager - .allocateContainer(pipelineSelector, ReplicationType.STAND_ALONE, - ReplicationFactor.THREE, "root").getContainerInfo(); - // Open Container - ContainerInfo cont3 = containerStateManager - .allocateContainer(pipelineSelector, ReplicationType.STAND_ALONE, - ReplicationFactor.THREE, "root").getContainerInfo(); - - long c1 = cont1.getContainerID(); - long c2 = cont2.getContainerID(); - long c3 = cont3.getContainerID(); - - // Close remaining containers - TestUtils.closeContainer(containerStateManager, cont1); - TestUtils.closeContainer(containerStateManager, cont2); - - //when - - //initial reports before replication is enabled. 2 containers w 3 replicas. - reportHandler.onMessage( - new ContainerReportFromDatanode(dn1, - createContainerReport(new long[] {c1, c2, c3})), this); - - reportHandler.onMessage( - new ContainerReportFromDatanode(dn2, - createContainerReport(new long[] {c1, c2, c3})), this); - - reportHandler.onMessage( - new ContainerReportFromDatanode(dn3, - createContainerReport(new long[] {c1, c2})), this); - - reportHandler.onMessage( - new ContainerReportFromDatanode(dn4, - createContainerReport(new long[] {})), this); - - Assert.assertEquals(0, publishedEvents.size()); - - replicationActivityStatus.enableReplication(); - - //no problem here - reportHandler.onMessage( - new ContainerReportFromDatanode(dn1, - createContainerReport(new long[] {c1, c2})), this); - - Assert.assertEquals(0, publishedEvents.size()); - - //container is missing from d2 - reportHandler.onMessage( - new ContainerReportFromDatanode(dn2, - createContainerReport(new long[] {c1})), this); - - Assert.assertEquals(1, publishedEvents.size()); - ReplicationRequest replicationRequest = - (ReplicationRequest) publishedEvents.get(0); - - Assert.assertEquals(c2, replicationRequest.getContainerId()); - Assert.assertEquals(3, replicationRequest.getExpecReplicationCount()); - Assert.assertEquals(2, replicationRequest.getReplicationCount()); - - //container was replicated to dn4 - reportHandler.onMessage( - new ContainerReportFromDatanode(dn4, - createContainerReport(new long[] {c2})), this); - - //no more event, everything is perfect - Assert.assertEquals(1, publishedEvents.size()); - - //c2 was found at dn2 (it was missing before, magic) - reportHandler.onMessage( - new ContainerReportFromDatanode(dn2, - createContainerReport(new long[] {c1, c2})), this); - - //c2 is over replicated (dn1,dn2,dn3,dn4) - Assert.assertEquals(2, publishedEvents.size()); - - replicationRequest = - (ReplicationRequest) publishedEvents.get(1); - - Assert.assertEquals(c2, replicationRequest.getContainerId()); - Assert.assertEquals(3, replicationRequest.getExpecReplicationCount()); - Assert.assertEquals(4, replicationRequest.getReplicationCount()); - - } - - private ContainerReportsProto createContainerReport(long[] containerIds) { - - ContainerReportsProto.Builder crBuilder = - ContainerReportsProto.newBuilder(); - - for (long containerId : containerIds) { - org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder - ciBuilder = org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder(); - ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2") - .setSize(5368709120L) - .setUsed(2000000000L) - .setKeyCount(100000000L) - .setReadCount(100000000L) - .setWriteCount(100000000L) - .setReadBytes(2000000000L) - .setWriteBytes(2000000000L) - .setContainerID(containerId) - .setDeleteTransactionId(0); - - crBuilder.addReports(ciBuilder.build()); - } - - return crBuilder.build(); - } - - @Override - public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent( - EVENT_TYPE event, PAYLOAD payload) { - LOG.info("Event is published: {}", payload); - publishedEvents.add(payload); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java deleted file mode 100644 index b857740..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.hdds.scm.container; - -import java.io.IOException; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest; - -import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -/** - * Testing ContainerStatemanager. - */ -public class TestContainerStateManager { - - private ContainerStateManager containerStateManager; - - @Before - public void init() throws IOException { - OzoneConfiguration conf = new OzoneConfiguration(); - Mapping mapping = Mockito.mock(Mapping.class); - PipelineSelector selector = Mockito.mock(PipelineSelector.class); - containerStateManager = new ContainerStateManager(conf, mapping, selector); - - } - - @Test - public void checkReplicationStateOK() throws IOException { - //GIVEN - ContainerInfo c1 = TestUtils.allocateContainer(containerStateManager); - - DatanodeDetails d1 = TestUtils.randomDatanodeDetails(); - DatanodeDetails d2 = TestUtils.randomDatanodeDetails(); - DatanodeDetails d3 = TestUtils.randomDatanodeDetails(); - - addReplica(c1, d1); - addReplica(c1, d2); - addReplica(c1, d3); - - //WHEN - ReplicationRequest replicationRequest = containerStateManager - .checkReplicationState(new ContainerID(c1.getContainerID())); - - //THEN - Assert.assertNull(replicationRequest); - } - - @Test - public void checkReplicationStateMissingReplica() throws IOException { - //GIVEN - - ContainerInfo c1 = TestUtils.allocateContainer(containerStateManager); - - DatanodeDetails d1 = TestUtils.randomDatanodeDetails(); - DatanodeDetails d2 = TestUtils.randomDatanodeDetails(); - - addReplica(c1, d1); - addReplica(c1, d2); - - //WHEN - ReplicationRequest replicationRequest = containerStateManager - .checkReplicationState(new ContainerID(c1.getContainerID())); - - Assert - .assertEquals(c1.getContainerID(), replicationRequest.getContainerId()); - Assert.assertEquals(2, replicationRequest.getReplicationCount()); - Assert.assertEquals(3, replicationRequest.getExpecReplicationCount()); - } - - private void addReplica(ContainerInfo c1, DatanodeDetails d1) { - containerStateManager - .addContainerReplica(new ContainerID(c1.getContainerID()), d1); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java deleted file mode 100644 index 2f35719..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -/** - * Make CheckStyle happy. - */ -package org.apache.hadoop.hdds.scm.container.closer; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java deleted file mode 100644 index f93aea6..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -/** - * Make CheckStyle Happy. - */ -package org.apache.hadoop.hdds.scm.container; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java deleted file mode 100644 index 764daff..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.hdds.scm.container.placement.algorithms; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.scm.node.NodeManager; - -import org.junit.Assert; -import org.junit.Test; -import static org.mockito.Matchers.anyObject; -import org.mockito.Mockito; -import static org.mockito.Mockito.when; - -public class TestSCMContainerPlacementCapacity { - @Test - public void chooseDatanodes() throws SCMException { - //given - Configuration conf = new OzoneConfiguration(); - - List<DatanodeDetails> datanodes = new ArrayList<>(); - for (int i = 0; i < 7; i++) { - datanodes.add(TestUtils.randomDatanodeDetails()); - } - - NodeManager mockNodeManager = Mockito.mock(NodeManager.class); - when(mockNodeManager.getNodes(NodeState.HEALTHY)) - .thenReturn(new ArrayList<>(datanodes)); - - when(mockNodeManager.getNodeStat(anyObject())) - .thenReturn(new SCMNodeMetric(100L, 0L, 100L)); - when(mockNodeManager.getNodeStat(datanodes.get(2))) - .thenReturn(new SCMNodeMetric(100L, 90L, 10L)); - when(mockNodeManager.getNodeStat(datanodes.get(3))) - .thenReturn(new SCMNodeMetric(100L, 80L, 20L)); - when(mockNodeManager.getNodeStat(datanodes.get(4))) - .thenReturn(new SCMNodeMetric(100L, 70L, 30L)); - - SCMContainerPlacementCapacity scmContainerPlacementRandom = - new SCMContainerPlacementCapacity(mockNodeManager, conf); - - List<DatanodeDetails> existingNodes = new ArrayList<>(); - existingNodes.add(datanodes.get(0)); - existingNodes.add(datanodes.get(1)); - - Map<DatanodeDetails, Integer> selectedCount = new HashMap<>(); - for (DatanodeDetails datanode : datanodes) { - selectedCount.put(datanode, 0); - } - - for (int i = 0; i < 1000; i++) { - - //when - List<DatanodeDetails> datanodeDetails = - scmContainerPlacementRandom.chooseDatanodes(existingNodes, 1, 15); - - //then - Assert.assertEquals(1, datanodeDetails.size()); - DatanodeDetails datanode0Details = datanodeDetails.get(0); - - Assert.assertNotEquals( - "Datanode 0 should not been selected: excluded by parameter", - datanodes.get(0), datanode0Details); - Assert.assertNotEquals( - "Datanode 1 should not been selected: excluded by parameter", - datanodes.get(1), datanode0Details); - Assert.assertNotEquals( - "Datanode 2 should not been selected: not enough space there", - datanodes.get(2), datanode0Details); - - selectedCount - .put(datanode0Details, selectedCount.get(datanode0Details) + 1); - - } - - //datanode 4 has less space. Should be selected less times. - Assert.assertTrue(selectedCount.get(datanodes.get(3)) > selectedCount - .get(datanodes.get(6))); - Assert.assertTrue(selectedCount.get(datanodes.get(4)) > selectedCount - .get(datanodes.get(6))); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java deleted file mode 100644 index b652b6b..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.hdds.scm.container.placement.algorithms; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.scm.node.NodeManager; - -import org.junit.Assert; -import org.junit.Test; -import static org.mockito.Matchers.anyObject; -import org.mockito.Mockito; -import static org.mockito.Mockito.when; - -public class TestSCMContainerPlacementRandom { - - @Test - public void chooseDatanodes() throws SCMException { - //given - Configuration conf = new OzoneConfiguration(); - - List<DatanodeDetails> datanodes = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - datanodes.add(TestUtils.randomDatanodeDetails()); - } - - NodeManager mockNodeManager = Mockito.mock(NodeManager.class); - when(mockNodeManager.getNodes(NodeState.HEALTHY)) - .thenReturn(new ArrayList<>(datanodes)); - - when(mockNodeManager.getNodeStat(anyObject())) - .thenReturn(new SCMNodeMetric(100L, 0L, 100L)); - when(mockNodeManager.getNodeStat(datanodes.get(2))) - .thenReturn(new SCMNodeMetric(100L, 90L, 10L)); - - SCMContainerPlacementRandom scmContainerPlacementRandom = - new SCMContainerPlacementRandom(mockNodeManager, conf); - - List<DatanodeDetails> existingNodes = new ArrayList<>(); - existingNodes.add(datanodes.get(0)); - existingNodes.add(datanodes.get(1)); - - for (int i = 0; i < 100; i++) { - //when - List<DatanodeDetails> datanodeDetails = - scmContainerPlacementRandom.chooseDatanodes(existingNodes, 1, 15); - - //then - Assert.assertEquals(1, datanodeDetails.size()); - DatanodeDetails datanode0Details = datanodeDetails.get(0); - - Assert.assertNotEquals( - "Datanode 0 should not been selected: excluded by parameter", - datanodes.get(0), datanode0Details); - Assert.assertNotEquals( - "Datanode 1 should not been selected: excluded by parameter", - datanodes.get(1), datanode0Details); - Assert.assertNotEquals( - "Datanode 2 should not been selected: not enough space there", - datanodes.get(2), datanode0Details); - - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java deleted file mode 100644 index a4615fc..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationActivityStatus.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements.ââSee the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership.ââThe ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License.ââYou may obtain a copy of the License at - * - * ââââ http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.scm.container.replication; - -import static org.junit.Assert.*; - -import java.util.concurrent.TimeoutException; -import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Tests for ReplicationActivityStatus. - */ -public class TestReplicationActivityStatus { - - private static EventQueue eventQueue; - private static ReplicationActivityStatus replicationActivityStatus; - - @BeforeClass - public static void setup() { - eventQueue = new EventQueue(); - replicationActivityStatus = new ReplicationActivityStatus(); - eventQueue.addHandler(SCMEvents.START_REPLICATION, - replicationActivityStatus.getReplicationStatusListener()); - eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, - replicationActivityStatus.getChillModeStatusListener()); - } - - @Test - public void testReplicationStatusForChillMode() - throws TimeoutException, InterruptedException { - assertFalse(replicationActivityStatus.isReplicationEnabled()); - // In chill mode replication process should be stopped. - eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, true); - assertFalse(replicationActivityStatus.isReplicationEnabled()); - - // Replication should be enabled when chill mode if off. - eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false); - GenericTestUtils.waitFor(() -> { - return replicationActivityStatus.isReplicationEnabled(); - }, 10, 1000*5); - assertTrue(replicationActivityStatus.isReplicationEnabled()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java deleted file mode 100644 index 06beb7c..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.hdds.scm.container.replication; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; - -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerStateManager; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID; -import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy; -import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationRequestToRepeat; -import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.ozone.lease.LeaseManager; -import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; - -import com.google.common.base.Preconditions; -import static org.apache.hadoop.hdds.scm.events.SCMEvents.TRACK_REPLICATE_COMMAND; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import static org.mockito.Matchers.anyObject; -import org.mockito.Mockito; -import static org.mockito.Mockito.when; - -/** - * Test behaviour of the TestReplication. - */ -public class TestReplicationManager { - - private EventQueue queue; - - private List<ReplicationRequestToRepeat> trackReplicationEvents; - - private List<CommandForDatanode<ReplicateContainerCommandProto>> copyEvents; - - private ContainerStateManager containerStateManager; - - private ContainerPlacementPolicy containerPlacementPolicy; - private List<DatanodeDetails> listOfDatanodeDetails; - private LeaseManager<Long> leaseManager; - private ReplicationManager replicationManager; - - @Before - public void initReplicationManager() throws IOException { - - listOfDatanodeDetails = new ArrayList<>(); - listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails()); - listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails()); - listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails()); - listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails()); - listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails()); - - containerPlacementPolicy = - (excludedNodes, nodesRequired, sizeRequired) -> listOfDatanodeDetails - .subList(2, 2 + nodesRequired); - - containerStateManager = Mockito.mock(ContainerStateManager.class); - - ContainerInfo containerInfo = new ContainerInfo.Builder() - .setState(LifeCycleState.CLOSED) - .build(); - - when(containerStateManager.getContainer(anyObject())) - .thenReturn(containerInfo); - - when(containerStateManager.getContainerReplicas(new ContainerID(1L))) - .thenReturn(new HashSet<>(Arrays.asList( - listOfDatanodeDetails.get(0), - listOfDatanodeDetails.get(1) - ))); - - - when(containerStateManager.getContainerReplicas(new ContainerID(3L))) - .thenReturn(new HashSet<>()); - - queue = new EventQueue(); - - trackReplicationEvents = new ArrayList<>(); - queue.addHandler(TRACK_REPLICATE_COMMAND, - (event, publisher) -> trackReplicationEvents.add(event)); - - copyEvents = new ArrayList<>(); - queue.addHandler(SCMEvents.DATANODE_COMMAND, - (event, publisher) -> copyEvents.add(event)); - - leaseManager = new LeaseManager<>("Test", 100000L); - - replicationManager = new ReplicationManager(containerPlacementPolicy, - containerStateManager, queue, leaseManager); - - - - } - - /** - * Container should be replicated but no source replicas. - */ - @Test() - public void testNoExistingReplicas() throws InterruptedException { - try { - leaseManager.start(); - replicationManager.start(); - - //WHEN - queue.fireEvent(SCMEvents.REPLICATE_CONTAINER, - new ReplicationRequest(3L, (short) 2, System.currentTimeMillis(), - (short) 3)); - - Thread.sleep(500L); - queue.processAll(1000L); - - //THEN - Assert.assertEquals(0, trackReplicationEvents.size()); - Assert.assertEquals(0, copyEvents.size()); - - } finally { - if (leaseManager != null) { - leaseManager.shutdown(); - } - } - } - - @Test - public void testEventSending() throws InterruptedException, IOException { - - //GIVEN - try { - leaseManager.start(); - - replicationManager.start(); - - //WHEN - queue.fireEvent(SCMEvents.REPLICATE_CONTAINER, - new ReplicationRequest(1L, (short) 2, System.currentTimeMillis(), - (short) 3)); - - Thread.sleep(500L); - queue.processAll(1000L); - - //THEN - Assert.assertEquals(1, trackReplicationEvents.size()); - Assert.assertEquals(1, copyEvents.size()); - } finally { - if (leaseManager != null) { - leaseManager.shutdown(); - } - } - } - - @Test - public void testCommandWatcher() throws InterruptedException, IOException { - LeaseManager<Long> rapidLeaseManager = - new LeaseManager<>("Test", 1000L); - - replicationManager = new ReplicationManager(containerPlacementPolicy, - containerStateManager, queue, rapidLeaseManager); - - try { - rapidLeaseManager.start(); - replicationManager.start(); - - queue.fireEvent(SCMEvents.REPLICATE_CONTAINER, - new ReplicationRequest(1L, (short) 2, System.currentTimeMillis(), - (short) 3)); - - Thread.sleep(500L); - - queue.processAll(1000L); - - Assert.assertEquals(1, trackReplicationEvents.size()); - Assert.assertEquals(1, copyEvents.size()); - - Assert.assertEquals(trackReplicationEvents.get(0).getId(), - copyEvents.get(0).getCommand().getId()); - - //event is timed out - Thread.sleep(1500); - - queue.processAll(1000L); - - //original copy command + retry - Assert.assertEquals(2, trackReplicationEvents.size()); - Assert.assertEquals(2, copyEvents.size()); - - } finally { - if (rapidLeaseManager != null) { - rapidLeaseManager.shutdown(); - } - } - } - - public static Pipeline createPipeline(Iterable<DatanodeDetails> ids) - throws IOException { - Objects.requireNonNull(ids, "ids == null"); - final Iterator<DatanodeDetails> i = ids.iterator(); - Preconditions.checkArgument(i.hasNext()); - final DatanodeDetails leader = i.next(); - final Pipeline pipeline = - new Pipeline(leader.getUuidString(), LifeCycleState.OPEN, - ReplicationType.STAND_ALONE, ReplicationFactor.ONE, - PipelineID.randomId()); - pipeline.addMember(leader); - while (i.hasNext()) { - pipeline.addMember(i.next()); - } - return pipeline; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java deleted file mode 100644 index 9dd4fe3..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationQueue.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdds.scm.container.replication; - -import java.util.Random; -import java.util.UUID; -import org.apache.hadoop.util.Time; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * Test class for ReplicationQueue. - */ -public class TestReplicationQueue { - - private ReplicationQueue replicationQueue; - private Random random; - - @Before - public void setUp() { - replicationQueue = new ReplicationQueue(); - random = new Random(); - } - - @Test - public void testDuplicateAddOp() throws InterruptedException { - long contId = random.nextLong(); - String nodeId = UUID.randomUUID().toString(); - ReplicationRequest obj1, obj2, obj3; - long time = Time.monotonicNow(); - obj1 = new ReplicationRequest(contId, (short) 2, time, (short) 3); - obj2 = new ReplicationRequest(contId, (short) 2, time + 1, (short) 3); - obj3 = new ReplicationRequest(contId, (short) 1, time+2, (short) 3); - - replicationQueue.add(obj1); - replicationQueue.add(obj2); - replicationQueue.add(obj3); - Assert.assertEquals("Should add only 1 msg as second one is duplicate", - 1, replicationQueue.size()); - ReplicationRequest temp = replicationQueue.take(); - Assert.assertEquals(temp, obj3); - } - - @Test - public void testPollOp() throws InterruptedException { - long contId = random.nextLong(); - String nodeId = UUID.randomUUID().toString(); - ReplicationRequest msg1, msg2, msg3, msg4, msg5; - msg1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(), - (short) 3); - long time = Time.monotonicNow(); - msg2 = new ReplicationRequest(contId + 1, (short) 4, time, (short) 3); - msg3 = new ReplicationRequest(contId + 2, (short) 0, time, (short) 3); - msg4 = new ReplicationRequest(contId, (short) 2, time, (short) 3); - // Replication message for same container but different nodeId - msg5 = new ReplicationRequest(contId + 1, (short) 2, time, (short) 3); - - replicationQueue.add(msg1); - replicationQueue.add(msg2); - replicationQueue.add(msg3); - replicationQueue.add(msg4); - replicationQueue.add(msg5); - Assert.assertEquals("Should have 3 objects", - 3, replicationQueue.size()); - - // Since Priority queue orders messages according to replication count, - // message with lowest replication should be first - ReplicationRequest temp; - temp = replicationQueue.take(); - Assert.assertEquals("Should have 2 objects", - 2, replicationQueue.size()); - Assert.assertEquals(temp, msg3); - - temp = replicationQueue.take(); - Assert.assertEquals("Should have 1 objects", - 1, replicationQueue.size()); - Assert.assertEquals(temp, msg5); - - // Message 2 should be ordered before message 5 as both have same - // replication number but message 2 has earlier timestamp. - temp = replicationQueue.take(); - Assert.assertEquals("Should have 0 objects", - replicationQueue.size(), 0); - Assert.assertEquals(temp, msg4); - } - - @Test - public void testRemoveOp() { - long contId = random.nextLong(); - String nodeId = UUID.randomUUID().toString(); - ReplicationRequest obj1, obj2, obj3; - obj1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(), - (short) 3); - obj2 = new ReplicationRequest(contId + 1, (short) 2, Time.monotonicNow(), - (short) 3); - obj3 = new ReplicationRequest(contId + 2, (short) 3, Time.monotonicNow(), - (short) 3); - - replicationQueue.add(obj1); - replicationQueue.add(obj2); - replicationQueue.add(obj3); - Assert.assertEquals("Should have 3 objects", - 3, replicationQueue.size()); - - replicationQueue.remove(obj3); - Assert.assertEquals("Should have 2 objects", - 2, replicationQueue.size()); - - replicationQueue.remove(obj2); - Assert.assertEquals("Should have 1 objects", - 1, replicationQueue.size()); - - replicationQueue.remove(obj1); - Assert.assertEquals("Should have 0 objects", - 0, replicationQueue.size()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java deleted file mode 100644 index 1423c99..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -/** - * SCM Testing and Mocking Utils. - */ -package org.apache.hadoop.hdds.scm.container.replication; -// Test classes for Replication functionality. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java deleted file mode 100644 index 63cc9bf..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerAttribute.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package org.apache.hadoop.hdds.scm.container.states; - -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.Arrays; -import java.util.List; - -/** - * Test ContainerAttribute management. - */ -public class TestContainerAttribute { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Test - public void testInsert() throws SCMException { - ContainerAttribute<Integer> containerAttribute = new ContainerAttribute<>(); - ContainerID id = new ContainerID(42); - containerAttribute.insert(1, id); - Assert.assertEquals(1, - containerAttribute.getCollection(1).size()); - Assert.assertTrue(containerAttribute.getCollection(1).contains(id)); - - // Insert again and verify that it overwrites an existing value. - ContainerID newId = - new ContainerID(42); - containerAttribute.insert(1, newId); - Assert.assertEquals(1, - containerAttribute.getCollection(1).size()); - Assert.assertTrue(containerAttribute.getCollection(1).contains(newId)); - } - - @Test - public void testHasKey() throws SCMException { - ContainerAttribute<Integer> containerAttribute = new ContainerAttribute<>(); - - for (int x = 1; x < 42; x++) { - containerAttribute.insert(1, new ContainerID(x)); - } - Assert.assertTrue(containerAttribute.hasKey(1)); - for (int x = 1; x < 42; x++) { - Assert.assertTrue(containerAttribute.hasContainerID(1, x)); - } - - Assert.assertFalse(containerAttribute.hasContainerID(1, - new ContainerID(42))); - } - - @Test - public void testClearSet() throws SCMException { - List<String> keyslist = Arrays.asList("Key1", "Key2", "Key3"); - ContainerAttribute<String> containerAttribute = new ContainerAttribute<>(); - for (String k : keyslist) { - for (int x = 1; x < 101; x++) { - containerAttribute.insert(k, new ContainerID(x)); - } - } - for (String k : keyslist) { - Assert.assertEquals(100, - containerAttribute.getCollection(k).size()); - } - containerAttribute.clearSet("Key1"); - Assert.assertEquals(0, - containerAttribute.getCollection("Key1").size()); - } - - @Test - public void testRemove() throws SCMException { - - List<String> keyslist = Arrays.asList("Key1", "Key2", "Key3"); - ContainerAttribute<String> containerAttribute = new ContainerAttribute<>(); - - for (String k : keyslist) { - for (int x = 1; x < 101; x++) { - containerAttribute.insert(k, new ContainerID(x)); - } - } - for (int x = 1; x < 101; x += 2) { - containerAttribute.remove("Key1", new ContainerID(x)); - } - - for (int x = 1; x < 101; x += 2) { - Assert.assertFalse(containerAttribute.hasContainerID("Key1", - new ContainerID(x))); - } - - Assert.assertEquals(100, - containerAttribute.getCollection("Key2").size()); - - Assert.assertEquals(100, - containerAttribute.getCollection("Key3").size()); - - Assert.assertEquals(50, - containerAttribute.getCollection("Key1").size()); - } - - @Test - public void tesUpdate() throws SCMException { - String key1 = "Key1"; - String key2 = "Key2"; - String key3 = "Key3"; - - ContainerAttribute<String> containerAttribute = new ContainerAttribute<>(); - ContainerID id = new ContainerID(42); - - containerAttribute.insert(key1, id); - Assert.assertTrue(containerAttribute.hasContainerID(key1, id)); - Assert.assertFalse(containerAttribute.hasContainerID(key2, id)); - - // This should move the id from key1 bucket to key2 bucket. - containerAttribute.update(key1, key2, id); - Assert.assertFalse(containerAttribute.hasContainerID(key1, id)); - Assert.assertTrue(containerAttribute.hasContainerID(key2, id)); - - // This should fail since we cannot find this id in the key3 bucket. - thrown.expect(SCMException.class); - containerAttribute.update(key3, key1, id); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java deleted file mode 100644 index 795dfc1..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -/** - * Make CheckStyle Happy. - */ -package org.apache.hadoop.hdds.scm.container.states; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java deleted file mode 100644 index 6f8534d..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.scm.node; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.container.ContainerMapping; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -import org.apache.hadoop.hdds.scm.container.placement.algorithms - .ContainerPlacementPolicy; -import org.apache.hadoop.hdds.scm.container.placement.algorithms - .SCMContainerPlacementCapacity; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.test.PathUtils; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mockito; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeoutException; - -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_DB_CACHE_SIZE_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys - .OZONE_SCM_DB_CACHE_SIZE_MB; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState - .HEALTHY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test for different container placement policy. - */ -public class TestContainerPlacement { - @Rule - public ExpectedException thrown = ExpectedException.none(); - private static XceiverClientManager xceiverClientManager = - new XceiverClientManager(new OzoneConfiguration()); - - /** - * Returns a new copy of Configuration. - * - * @return Config - */ - OzoneConfiguration getConf() { - return new OzoneConfiguration(); - } - - /** - * Creates a NodeManager. - * - * @param config - Config for the node manager. - * @return SCNNodeManager - * @throws IOException - */ - - SCMNodeManager createNodeManager(OzoneConfiguration config) - throws IOException { - EventQueue eventQueue = new EventQueue(); - eventQueue.addHandler(SCMEvents.NEW_NODE, - Mockito.mock(NewNodeHandler.class)); - eventQueue.addHandler(SCMEvents.STALE_NODE, - Mockito.mock(StaleNodeHandler.class)); - eventQueue.addHandler(SCMEvents.DEAD_NODE, - Mockito.mock(DeadNodeHandler.class)); - SCMNodeManager nodeManager = new SCMNodeManager(config, - UUID.randomUUID().toString(), null, eventQueue); - assertFalse("Node manager should be in chill mode", - nodeManager.isOutOfChillMode()); - return nodeManager; - } - - ContainerMapping createContainerManager(Configuration config, - NodeManager scmNodeManager) throws IOException { - EventQueue eventQueue = new EventQueue(); - final int cacheSize = config.getInt(OZONE_SCM_DB_CACHE_SIZE_MB, - OZONE_SCM_DB_CACHE_SIZE_DEFAULT); - return new ContainerMapping(config, scmNodeManager, cacheSize, eventQueue); - - } - - /** - * Test capacity based container placement policy with node reports. - * - * @throws IOException - * @throws InterruptedException - * @throws TimeoutException - */ - @Test - @Ignore - public void testContainerPlacementCapacity() throws IOException, - InterruptedException, TimeoutException { - OzoneConfiguration conf = getConf(); - final int nodeCount = 4; - final long capacity = 10L * OzoneConsts.GB; - final long used = 2L * OzoneConsts.GB; - final long remaining = capacity - used; - - final File testDir = PathUtils.getTestDir( - TestContainerPlacement.class); - conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, - testDir.getAbsolutePath()); - conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, - SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class); - - SCMNodeManager nodeManager = createNodeManager(conf); - ContainerMapping containerManager = - createContainerManager(conf, nodeManager); - List<DatanodeDetails> datanodes = - TestUtils.getListOfRegisteredDatanodeDetails(nodeManager, nodeCount); - try { - for (DatanodeDetails datanodeDetails : datanodes) { - nodeManager.processHeartbeat(datanodeDetails); - } - - //TODO: wait for heartbeat to be processed - Thread.sleep(4 * 1000); - assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY)); - assertEquals(capacity * nodeCount, - (long) nodeManager.getStats().getCapacity().get()); - assertEquals(used * nodeCount, - (long) nodeManager.getStats().getScmUsed().get()); - assertEquals(remaining * nodeCount, - (long) nodeManager.getStats().getRemaining().get()); - - assertTrue(nodeManager.isOutOfChillMode()); - - ContainerWithPipeline containerWithPipeline = containerManager - .allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), "OZONE"); - assertEquals(xceiverClientManager.getFactor().getNumber(), - containerWithPipeline.getPipeline().getMachines().size()); - } finally { - IOUtils.closeQuietly(containerManager); - IOUtils.closeQuietly(nodeManager); - FileUtil.fullyDelete(testDir); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java deleted file mode 100644 index 7bba032..0000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.scm.node; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; -import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerStateManager; -import org.apache.hadoop.hdds.scm.container.Mapping; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; -import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest; -import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector; -import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode; -import org.apache.hadoop.hdds.server.events.EventPublisher; - -import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import static org.mockito.Matchers.eq; -import org.mockito.Mockito; - -/** - * Test DeadNodeHandler. - */ -public class TestDeadNodeHandler { - - private List<ReplicationRequest> sentEvents = new ArrayList<>(); - private SCMNodeManager nodeManager; - private ContainerStateManager containerStateManager; - private NodeReportHandler nodeReportHandler; - private DeadNodeHandler deadNodeHandler; - private EventPublisher publisher; - private EventQueue eventQueue; - - @Before - public void setup() throws IOException { - OzoneConfiguration conf = new OzoneConfiguration(); - containerStateManager = new ContainerStateManager(conf, - Mockito.mock(Mapping.class), - Mockito.mock(PipelineSelector.class)); - eventQueue = new EventQueue(); - nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue); - deadNodeHandler = new DeadNodeHandler(nodeManager, - containerStateManager); - eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler); - publisher = Mockito.mock(EventPublisher.class); - nodeReportHandler = new NodeReportHandler(nodeManager); - } - - @Test - public void testOnMessage() throws IOException { - //GIVEN - DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails(); - DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails(); - - ContainerInfo container1 = - TestUtils.allocateContainer(containerStateManager); - ContainerInfo container2 = - TestUtils.allocateContainer(containerStateManager); - ContainerInfo container3 = - TestUtils.allocateContainer(containerStateManager); - - registerReplicas(datanode1, container1, container2); - registerReplicas(datanode2, container1, container3); - - registerReplicas(containerStateManager, container1, datanode1, datanode2); - registerReplicas(containerStateManager, container2, datanode1); - registerReplicas(containerStateManager, container3, datanode2); - - TestUtils.closeContainer(containerStateManager, container1); - - deadNodeHandler.onMessage(datanode1, publisher); - - Set<DatanodeDetails> container1Replicas = - containerStateManager.getContainerStateMap() - .getContainerReplicas(new ContainerID(container1.getContainerID())); - Assert.assertEquals(1, container1Replicas.size()); - Assert.assertEquals(datanode2, container1Replicas.iterator().next()); - - Set<DatanodeDetails> container2Replicas = - containerStateManager.getContainerStateMap() - .getContainerReplicas(new ContainerID(container2.getContainerID())); - Assert.assertEquals(0, container2Replicas.size()); - - Set<DatanodeDetails> container3Replicas = - containerStateManager.getContainerStateMap() - .getContainerReplicas(new ContainerID(container3.getContainerID())); - Assert.assertEquals(1, container3Replicas.size()); - Assert.assertEquals(datanode2, container3Replicas.iterator().next()); - - ArgumentCaptor<ReplicationRequest> replicationRequestParameter = - ArgumentCaptor.forClass(ReplicationRequest.class); - - Mockito.verify(publisher) - .fireEvent(eq(SCMEvents.REPLICATE_CONTAINER), - replicationRequestParameter.capture()); - - Assert - .assertEquals(container1.getContainerID(), - replicationRequestParameter.getValue().getContainerId()); - Assert - .assertEquals(1, - replicationRequestParameter.getValue().getReplicationCount()); - Assert - .assertEquals(3, - replicationRequestParameter.getValue().getExpecReplicationCount()); - } - - @Test - public void testStatisticsUpdate() throws Exception { - //GIVEN - DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails(); - DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails(); - String storagePath1 = GenericTestUtils.getRandomizedTempPath() - .concat("/" + datanode1.getUuidString()); - String storagePath2 = GenericTestUtils.getRandomizedTempPath() - .concat("/" + datanode2.getUuidString()); - - StorageReportProto storageOne = TestUtils.createStorageReport( - datanode1.getUuid(), storagePath1, 100, 10, 90, null); - StorageReportProto storageTwo = TestUtils.createStorageReport( - datanode2.getUuid(), storagePath2, 200, 20, 180, null); - nodeReportHandler.onMessage(getNodeReport(datanode1, storageOne), - Mockito.mock(EventPublisher.class)); - nodeReportHandler.onMessage(getNodeReport(datanode2, storageTwo), - Mockito.mock(EventPublisher.class)); - - ContainerInfo container1 = - TestUtils.allocateContainer(containerStateManager); - registerReplicas(datanode1, container1); - - SCMNodeStat stat = nodeManager.getStats(); - Assert.assertTrue(stat.getCapacity().get() == 300); - Assert.assertTrue(stat.getRemaining().get() == 270); - Assert.assertTrue(stat.getScmUsed().get() == 30); - - SCMNodeMetric nodeStat = nodeManager.getNodeStat(datanode1); - Assert.assertTrue(nodeStat.get().getCapacity().get() == 100); - Assert.assertTrue(nodeStat.get().getRemaining().get() == 90); - Assert.assertTrue(nodeStat.get().getScmUsed().get() == 10); - - //WHEN datanode1 is dead. - eventQueue.fireEvent(SCMEvents.DEAD_NODE, datanode1); - Thread.sleep(100); - - //THEN statistics in SCM should changed. - stat = nodeManager.getStats(); - Assert.assertTrue(stat.getCapacity().get() == 200); - Assert.assertTrue(stat.getRemaining().get() == 180); - Assert.assertTrue(stat.getScmUsed().get() == 20); - - nodeStat = nodeManager.getNodeStat(datanode1); - Assert.assertTrue(nodeStat.get().getCapacity().get() == 0); - Assert.assertTrue(nodeStat.get().getRemaining().get() == 0); - Assert.assertTrue(nodeStat.get().getScmUsed().get() == 0); - } - - @Test - public void testOnMessageReplicaFailure() throws Exception { - DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); - GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer - .captureLogs(DeadNodeHandler.getLogger()); - String storagePath1 = GenericTestUtils.getRandomizedTempPath() - .concat("/" + dn1.getUuidString()); - - StorageReportProto storageOne = TestUtils.createStorageReport( - dn1.getUuid(), storagePath1, 100, 10, 90, null); - nodeReportHandler.onMessage(getNodeReport(dn1, storageOne), - Mockito.mock(EventPublisher.class)); - - ContainerInfo container1 = - TestUtils.allocateContainer(containerStateManager); - registerReplicas(dn1, container1); - - deadNodeHandler.onMessage(dn1, eventQueue); - Assert.assertTrue(logCapturer.getOutput().contains( - "DataNode " + dn1.getUuid() + " doesn't have replica for container " - + container1.getContainerID())); - } - - private void registerReplicas(ContainerStateManager csm, - ContainerInfo container, DatanodeDetails... datanodes) { - csm.getContainerStateMap() - .addContainerReplica(new ContainerID(container.getContainerID()), - datanodes); - } - - private void registerReplicas(DatanodeDetails datanode, - ContainerInfo... containers) - throws SCMException { - nodeManager - .addDatanodeInContainerMap(datanode.getUuid(), - Arrays.stream(containers) - .map(container -> new ContainerID(container.getContainerID())) - .collect(Collectors.toSet())); - } - - private NodeReportFromDatanode getNodeReport(DatanodeDetails dn, - StorageReportProto... reports) { - NodeReportProto nodeReportProto = TestUtils.createNodeReport(reports); - return new NodeReportFromDatanode(dn, nodeReportProto); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org