HDDS-234. Add SCM node report handler. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/556d9b36 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/556d9b36 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/556d9b36 Branch: refs/heads/YARN-7402 Commit: 556d9b36be4b0b759646b8f6030c9e693b97bdb8 Parents: 5ee90ef Author: Anu Engineer <aengin...@apache.org> Authored: Thu Jul 12 12:09:31 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Thu Jul 12 12:09:31 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hdds/scm/node/NodeManager.java | 9 ++ .../hadoop/hdds/scm/node/NodeReportHandler.java | 19 +++- .../hadoop/hdds/scm/node/SCMNodeManager.java | 11 +++ .../hdds/scm/container/MockNodeManager.java | 11 +++ .../hdds/scm/node/TestNodeReportHandler.java | 95 ++++++++++++++++++++ .../testutils/ReplicationNodeManagerMock.java | 10 +++ 6 files changed, 152 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index 5e2969d..deb1628 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdds.scm.node; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; @@ -138,4 +139,12 @@ public interface NodeManager extends StorageContainerNodeProtocol, * @param command */ void addDatanodeCommand(UUID dnId, SCMCommand command); + + /** + * Process node report. + * + * @param dnUuid + * @param nodeReport + */ + void processNodeReport(UUID dnUuid, NodeReportProto nodeReport); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java index aa78d53..331bfed 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,25 +18,38 @@ package org.apache.hadoop.hdds.scm.node; +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher .NodeReportFromDatanode; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Handles Node Reports from datanode. */ public class NodeReportHandler implements EventHandler<NodeReportFromDatanode> { + private static final Logger LOGGER = LoggerFactory + .getLogger(NodeReportHandler.class); private final NodeManager nodeManager; public NodeReportHandler(NodeManager nodeManager) { + Preconditions.checkNotNull(nodeManager); this.nodeManager = nodeManager; } @Override public void onMessage(NodeReportFromDatanode nodeReportFromDatanode, - EventPublisher publisher) { - //TODO: process node report. + EventPublisher publisher) { + Preconditions.checkNotNull(nodeReportFromDatanode); + DatanodeDetails dn = nodeReportFromDatanode.getDatanodeDetails(); + Preconditions.checkNotNull(dn, "NodeReport is " + + "missing DatanodeDetails."); + LOGGER.trace("Processing node report for dn: {}", dn); + nodeManager + .processNodeReport(dn.getUuid(), nodeReportFromDatanode.getReport()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 2ba8067..7370b07 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -423,6 +423,17 @@ public class SCMNodeManager } /** + * Process node report. + * + * @param dnUuid + * @param nodeReport + */ + @Override + public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) { + this.updateNodeStat(dnUuid, nodeReport); + } + + /** * Returns the aggregated node stats. * @return the aggregated node stats. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 5e83c28..593b780 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -295,6 +295,17 @@ public class MockNodeManager implements NodeManager { } } + /** + * Empty implementation for processNodeReport. + * + * @param dnUuid + * @param nodeReport + */ + @Override + public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) { + // do nothing + } + // Returns the number of commands that is queued to this node manager. public int getCommandCount(DatanodeDetails dd) { List<SCMCommand> list = commandMap.get(dd.getUuid()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java new file mode 100644 index 0000000..3cbde4b --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; +import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode; +import org.apache.hadoop.hdds.server.events.Event; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestNodeReportHandler implements EventPublisher { + + private static Logger LOG = LoggerFactory + .getLogger(TestNodeReportHandler.class); + private NodeReportHandler nodeReportHandler; + private SCMNodeManager nodeManager; + private String storagePath = GenericTestUtils.getRandomizedTempPath() + .concat("/" + UUID.randomUUID().toString()); + ; + + @Before + public void resetEventCollector() throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(); + nodeManager = new SCMNodeManager(conf, "cluster1", null, new EventQueue()); + nodeReportHandler = new NodeReportHandler(nodeManager); + } + + @Test + public void testNodeReport() throws IOException { + DatanodeDetails dn = TestUtils.getDatanodeDetails(); + List<StorageReportProto> reports = + TestUtils.createStorageReport(100, 10, 90, storagePath, null, + dn.getUuid().toString(), 1); + + nodeReportHandler.onMessage( + getNodeReport(dn, reports), this); + SCMNodeMetric nodeMetric = nodeManager.getNodeStat(dn); + + Assert.assertTrue(nodeMetric.get().getCapacity().get() == 100); + Assert.assertTrue(nodeMetric.get().getRemaining().get() == 90); + Assert.assertTrue(nodeMetric.get().getScmUsed().get() == 10); + + reports = + TestUtils.createStorageReport(100, 10, 90, storagePath, null, + dn.getUuid().toString(), 2); + nodeReportHandler.onMessage( + getNodeReport(dn, reports), this); + nodeMetric = nodeManager.getNodeStat(dn); + + Assert.assertTrue(nodeMetric.get().getCapacity().get() == 200); + Assert.assertTrue(nodeMetric.get().getRemaining().get() == 180); + Assert.assertTrue(nodeMetric.get().getScmUsed().get() == 20); + + } + + private NodeReportFromDatanode getNodeReport(DatanodeDetails dn, + List<StorageReportProto> reports) { + NodeReportProto nodeReportProto = TestUtils.createNodeReport(reports); + return new NodeReportFromDatanode(dn, nodeReportProto); + } + + @Override + public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent( + EVENT_TYPE event, PAYLOAD payload) { + LOG.info("Event is published: {}", payload); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index 2d27d71..a0249aa 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -289,6 +289,16 @@ public class ReplicationNodeManagerMock implements NodeManager { this.commandQueue.addCommand(dnId, command); } + /** + * Empty implementation for processNodeReport. + * @param dnUuid + * @param nodeReport + */ + @Override + public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) { + // do nothing. + } + @Override public void onMessage(CommandForDatanode commandForDatanode, EventPublisher publisher) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
