Author: llu
Date: Thu Apr 4 07:07:59 2013
New Revision: 1464301
URL: http://svn.apache.org/r1464301
Log:
HADOOP-9451. Fault single-layer config if node group topology is enabled.
(Junping Du via llu)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1464301&r1=1464300&r2=1464301&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Apr
4 07:07:59 2013
@@ -165,6 +165,9 @@ Trunk (Unreleased)
BUG FIXES
+ HADOOP-9451. Fault single-layer config if node group topology is enabled.
+ (Junping Du via llu)
+
HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
HADOOP-9041. FsUrlStreamHandlerFactory could cause an infinite loop in
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java?rev=1464301&r1=1464300&r2=1464301&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
Thu Apr 4 07:07:59 2013
@@ -191,7 +191,12 @@ public class NetworkTopologyWithNodeGrou
}
rack = getNode(nodeGroup.getNetworkLocation());
- if (rack != null && !(rack instanceof InnerNode)) {
+ // rack should be an innerNode and with parent.
+ // note: rack's null parent case is: node's topology only has one layer,
+ // so rack is recognized as "/" and no parent.
+ // This will be recognized as a node with fault topology.
+ if (rack != null &&
+ (!(rack instanceof InnerNode) || rack.getParent() == null)) {
throw new IllegalArgumentException("Unexpected data node "
+ node.toString()
+ " at an illegal network location");
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java?rev=1464301&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
Thu Apr 4 07:07:59 2013
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+public class TestNetworkTopologyWithNodeGroup {
+ private final static NetworkTopologyWithNodeGroup cluster = new
+ NetworkTopologyWithNodeGroup();
+
+ private final static NodeBase dataNodes[] = new NodeBase[] {
+ new NodeBase("h1", "/d1/r1/s1"),
+ new NodeBase("h2", "/d1/r1/s1"),
+ new NodeBase("h3", "/d1/r1/s2"),
+ new NodeBase("h4", "/d1/r2/s3"),
+ new NodeBase("h5", "/d1/r2/s3"),
+ new NodeBase("h6", "/d1/r2/s4"),
+ new NodeBase("h7", "/d2/r3/s5"),
+ new NodeBase("h8", "/d2/r3/s6")
+ };
+
+ private final static NodeBase computeNode = new NodeBase("/d1/r1/s1/h9");
+
+ private final static NodeBase rackOnlyNode = new NodeBase("h10", "/r2");
+
+ static {
+ for(int i=0; i<dataNodes.length; i++) {
+ cluster.add(dataNodes[i]);
+ }
+ }
+
+ @Test
+ public void testNumOfChildren() throws Exception {
+ assertEquals(cluster.getNumOfLeaves(), dataNodes.length);
+ }
+
+ @Test
+ public void testNumOfRacks() throws Exception {
+ assertEquals(cluster.getNumOfRacks(), 3);
+ }
+
+ @Test
+ public void testRacks() throws Exception {
+ assertEquals(cluster.getNumOfRacks(), 3);
+ assertTrue(cluster.isOnSameRack(dataNodes[0], dataNodes[1]));
+ assertTrue(cluster.isOnSameRack(dataNodes[1], dataNodes[2]));
+ assertFalse(cluster.isOnSameRack(dataNodes[2], dataNodes[3]));
+ assertTrue(cluster.isOnSameRack(dataNodes[3], dataNodes[4]));
+ assertTrue(cluster.isOnSameRack(dataNodes[4], dataNodes[5]));
+ assertFalse(cluster.isOnSameRack(dataNodes[5], dataNodes[6]));
+ assertTrue(cluster.isOnSameRack(dataNodes[6], dataNodes[7]));
+ }
+
+ @Test
+ public void testNodeGroups() throws Exception {
+ assertEquals(cluster.getNumOfRacks(), 3);
+ assertTrue(cluster.isOnSameNodeGroup(dataNodes[0], dataNodes[1]));
+ assertFalse(cluster.isOnSameNodeGroup(dataNodes[1], dataNodes[2]));
+ assertFalse(cluster.isOnSameNodeGroup(dataNodes[2], dataNodes[3]));
+ assertTrue(cluster.isOnSameNodeGroup(dataNodes[3], dataNodes[4]));
+ assertFalse(cluster.isOnSameNodeGroup(dataNodes[4], dataNodes[5]));
+ assertFalse(cluster.isOnSameNodeGroup(dataNodes[5], dataNodes[6]));
+ assertFalse(cluster.isOnSameNodeGroup(dataNodes[6], dataNodes[7]));
+ }
+
+ @Test
+ public void testGetDistance() throws Exception {
+ assertEquals(cluster.getDistance(dataNodes[0], dataNodes[0]), 0);
+ assertEquals(cluster.getDistance(dataNodes[0], dataNodes[1]), 2);
+ assertEquals(cluster.getDistance(dataNodes[0], dataNodes[2]), 4);
+ assertEquals(cluster.getDistance(dataNodes[0], dataNodes[3]), 6);
+ assertEquals(cluster.getDistance(dataNodes[0], dataNodes[6]), 8);
+ }
+
+ @Test
+ public void testPseudoSortByDistance() throws Exception {
+ NodeBase[] testNodes = new NodeBase[4];
+
+ // array contains both local node, local node group & local rack node
+ testNodes[0] = dataNodes[1];
+ testNodes[1] = dataNodes[2];
+ testNodes[2] = dataNodes[3];
+ testNodes[3] = dataNodes[0];
+ cluster.pseudoSortByDistance(dataNodes[0], testNodes );
+ assertTrue(testNodes[0] == dataNodes[0]);
+ assertTrue(testNodes[1] == dataNodes[1]);
+ assertTrue(testNodes[2] == dataNodes[2]);
+ assertTrue(testNodes[3] == dataNodes[3]);
+
+ // array contains local node & local node group
+ testNodes[0] = dataNodes[3];
+ testNodes[1] = dataNodes[4];
+ testNodes[2] = dataNodes[1];
+ testNodes[3] = dataNodes[0];
+ cluster.pseudoSortByDistance(dataNodes[0], testNodes );
+ assertTrue(testNodes[0] == dataNodes[0]);
+ assertTrue(testNodes[1] == dataNodes[1]);
+
+ // array contains local node & rack node
+ testNodes[0] = dataNodes[5];
+ testNodes[1] = dataNodes[3];
+ testNodes[2] = dataNodes[2];
+ testNodes[3] = dataNodes[0];
+ cluster.pseudoSortByDistance(dataNodes[0], testNodes );
+ assertTrue(testNodes[0] == dataNodes[0]);
+ assertTrue(testNodes[1] == dataNodes[2]);
+
+ // array contains local-nodegroup node (not a data node also) & rack node
+ testNodes[0] = dataNodes[6];
+ testNodes[1] = dataNodes[7];
+ testNodes[2] = dataNodes[2];
+ testNodes[3] = dataNodes[0];
+ cluster.pseudoSortByDistance(computeNode, testNodes );
+ assertTrue(testNodes[0] == dataNodes[0]);
+ assertTrue(testNodes[1] == dataNodes[2]);
+ }
+
+ /**
+ * This picks a large number of nodes at random in order to ensure coverage
+ *
+ * @param numNodes the number of nodes
+ * @param excludedScope the excluded scope
+ * @return the frequency that nodes were chosen
+ */
+ private Map<Node, Integer> pickNodesAtRandom(int numNodes,
+ String excludedScope) {
+ Map<Node, Integer> frequency = new HashMap<Node, Integer>();
+ for (NodeBase dnd : dataNodes) {
+ frequency.put(dnd, 0);
+ }
+
+ for (int j = 0; j < numNodes; j++) {
+ Node random = cluster.chooseRandom(excludedScope);
+ frequency.put(random, frequency.get(random) + 1);
+ }
+ return frequency;
+ }
+
+ /**
+ * This test checks that chooseRandom works for an excluded node.
+ */
+ /**
+ * Test replica placement policy in case last node is invalid.
+ * We create 6 nodes but the last node is in fault topology (with rack info),
+ * so cannot be added to cluster. We should test proper exception is thrown
in
+ * adding node but shouldn't affect the cluster.
+ */
+ @Test
+ public void testChooseRandomExcludedNode() {
+ String scope = "~" + NodeBase.getPath(dataNodes[0]);
+ Map<Node, Integer> frequency = pickNodesAtRandom(100, scope);
+
+ for (Node key : dataNodes) {
+ // all nodes except the first should be more than zero
+ assertTrue(frequency.get(key) > 0 || key == dataNodes[0]);
+ }
+ }
+
+ /**
+ * This test checks that adding a node with invalid topology will be failed
+ * with an exception to show topology is invalid.
+ */
+ @Test
+ public void testAddNodeWithInvalidTopology() {
+ // The last node is a node with invalid topology
+ try {
+ cluster.add(rackOnlyNode);
+ fail("Exception should be thrown, so we should not have reached here.");
+ } catch (Exception e) {
+ if (!(e instanceof IllegalArgumentException)) {
+ fail("Expecting IllegalArgumentException, but caught:" + e);
+ }
+ assertTrue(e.getMessage().contains("illegal network location"));
+ }
+ }
+
+}
