HDDS-234. Add SCM node report handler.
Contributed by Ajay Kumar.

Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/556d9b36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/556d9b36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/556d9b36

Branch: refs/heads/YARN-7402
Commit: 556d9b36be4b0b759646b8f6030c9e693b97bdb8
Parents: 5ee90ef
Author: Anu Engineer <aengin...@apache.org>
Authored: Thu Jul 12 12:09:31 2018 -0700
Committer: Anu Engineer <aengin...@apache.org>
Committed: Thu Jul 12 12:09:31 2018 -0700

----------------------------------------------------------------------
 .../hadoop/hdds/scm/node/NodeManager.java       |  9 ++
 .../hadoop/hdds/scm/node/NodeReportHandler.java | 19 +++-
 .../hadoop/hdds/scm/node/SCMNodeManager.java    | 11 +++
 .../hdds/scm/container/MockNodeManager.java     | 11 +++
 .../hdds/scm/node/TestNodeReportHandler.java    | 95 ++++++++++++++++++++
 .../testutils/ReplicationNodeManagerMock.java   | 10 +++
 6 files changed, 152 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 5e2969d..deb1628 100644
--- 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdds.scm.node;
 
+import 
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
@@ -138,4 +139,12 @@ public interface NodeManager extends 
StorageContainerNodeProtocol,
    * @param command
    */
   void addDatanodeCommand(UUID dnId, SCMCommand command);
+
+  /**
+   * Process node report.
+   *
+   * @param dnUuid
+   * @param nodeReport
+   */
+  void processNodeReport(UUID dnUuid, NodeReportProto nodeReport);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
index aa78d53..331bfed 100644
--- 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
+++ 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,25 +18,38 @@
 
 package org.apache.hadoop.hdds.scm.node;
 
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
     .NodeReportFromDatanode;
 import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Handles Node Reports from datanode.
  */
 public class NodeReportHandler implements EventHandler<NodeReportFromDatanode> 
{
 
+  private static final Logger LOGGER = LoggerFactory
+      .getLogger(NodeReportHandler.class);
   private final NodeManager nodeManager;
 
   public NodeReportHandler(NodeManager nodeManager) {
+    Preconditions.checkNotNull(nodeManager);
     this.nodeManager = nodeManager;
   }
 
   @Override
   public void onMessage(NodeReportFromDatanode nodeReportFromDatanode,
-                        EventPublisher publisher) {
-    //TODO: process node report.
+      EventPublisher publisher) {
+    Preconditions.checkNotNull(nodeReportFromDatanode);
+    DatanodeDetails dn = nodeReportFromDatanode.getDatanodeDetails();
+    Preconditions.checkNotNull(dn, "NodeReport is "
+        + "missing DatanodeDetails.");
+    LOGGER.trace("Processing node report for dn: {}", dn);
+    nodeManager
+        .processNodeReport(dn.getUuid(), nodeReportFromDatanode.getReport());
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 2ba8067..7370b07 100644
--- 
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ 
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -423,6 +423,17 @@ public class SCMNodeManager
   }
 
   /**
+   * Process node report.
+   *
+   * @param dnUuid
+   * @param nodeReport
+   */
+  @Override
+  public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) {
+      this.updateNodeStat(dnUuid, nodeReport);
+  }
+
+  /**
    * Returns the aggregated node stats.
    * @return the aggregated node stats.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
 
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 5e83c28..593b780 100644
--- 
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ 
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -295,6 +295,17 @@ public class MockNodeManager implements NodeManager {
     }
   }
 
+  /**
+   * Empty implementation for processNodeReport.
+   *
+   * @param dnUuid
+   * @param nodeReport
+   */
+  @Override
+  public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) {
+    // do nothing
+  }
+
   // Returns the number of commands that is queued to this node manager.
   public int getCommandCount(DatanodeDetails dd) {
     List<SCMCommand> list = commandMap.get(dd.getUuid());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
 
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
new file mode 100644
index 0000000..3cbde4b
--- /dev/null
+++ 
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations 
under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.UUID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import 
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import 
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
+import 
org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestNodeReportHandler implements EventPublisher {
+
+  private static Logger LOG = LoggerFactory
+      .getLogger(TestNodeReportHandler.class);
+  private NodeReportHandler nodeReportHandler;
+  private SCMNodeManager nodeManager;
+  private String storagePath = GenericTestUtils.getRandomizedTempPath()
+      .concat("/" + UUID.randomUUID().toString());
+  ;
+
+  @Before
+  public void resetEventCollector() throws IOException {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    nodeManager = new SCMNodeManager(conf, "cluster1", null, new EventQueue());
+    nodeReportHandler = new NodeReportHandler(nodeManager);
+  }
+
+  @Test
+  public void testNodeReport() throws IOException {
+    DatanodeDetails dn = TestUtils.getDatanodeDetails();
+    List<StorageReportProto> reports =
+        TestUtils.createStorageReport(100, 10, 90, storagePath, null,
+            dn.getUuid().toString(), 1);
+
+    nodeReportHandler.onMessage(
+        getNodeReport(dn, reports), this);
+    SCMNodeMetric nodeMetric = nodeManager.getNodeStat(dn);
+
+    Assert.assertTrue(nodeMetric.get().getCapacity().get() == 100);
+    Assert.assertTrue(nodeMetric.get().getRemaining().get() == 90);
+    Assert.assertTrue(nodeMetric.get().getScmUsed().get() == 10);
+
+    reports =
+        TestUtils.createStorageReport(100, 10, 90, storagePath, null,
+            dn.getUuid().toString(), 2);
+    nodeReportHandler.onMessage(
+        getNodeReport(dn, reports), this);
+    nodeMetric = nodeManager.getNodeStat(dn);
+
+    Assert.assertTrue(nodeMetric.get().getCapacity().get() == 200);
+    Assert.assertTrue(nodeMetric.get().getRemaining().get() == 180);
+    Assert.assertTrue(nodeMetric.get().getScmUsed().get() == 20);
+
+  }
+
+  private NodeReportFromDatanode getNodeReport(DatanodeDetails dn,
+      List<StorageReportProto> reports) {
+    NodeReportProto nodeReportProto = TestUtils.createNodeReport(reports);
+    return new NodeReportFromDatanode(dn, nodeReportProto);
+  }
+
+  @Override
+  public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
+      EVENT_TYPE event, PAYLOAD payload) {
+    LOG.info("Event is published: {}", payload);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/556d9b36/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git 
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
 
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 2d27d71..a0249aa 100644
--- 
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ 
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -289,6 +289,16 @@ public class ReplicationNodeManagerMock implements 
NodeManager {
     this.commandQueue.addCommand(dnId, command);
   }
 
+  /**
+   * Empty implementation for processNodeReport.
+   * @param dnUuid
+   * @param nodeReport
+   */
+  @Override
+  public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) {
+    // do nothing.
+  }
+
   @Override
   public void onMessage(CommandForDatanode commandForDatanode,
                         EventPublisher publisher) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to