http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1feb4ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java index d05c75c..1e2326b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java @@ -29,7 +29,9 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -536,4 +538,30 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { Assert.assertTrue("Should failed when #labels > 1 on a host after add", failed); } + + @Test (timeout = 5000) + public void testUpdateNodeLabels() throws Exception { + boolean failed = false; + + // should fail: label isn't exist + try { + mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance( + "p1", false))); + } catch (YarnException e) { + failed = true; + } + Assert.assertTrue("Should fail since the node label doesn't exist", failed); + + mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + + mgr.updateNodeLabels(Arrays.asList( + NodeLabel.newInstance("p1", false), NodeLabel.newInstance("p2", true))); + Assert.assertEquals("p1", mgr.lastUpdatedNodeLabels.get(0).getNodeLabel()); + Assert.assertFalse(mgr.lastUpdatedNodeLabels.get(0).getIsExclusive()); + Assert.assertTrue(mgr.lastUpdatedNodeLabels.get(1).getIsExclusive()); + + // Check exclusive for p1/p2 + Assert.assertFalse(mgr.isExclusiveNodeLabel("p1")); + Assert.assertTrue(mgr.isExclusiveNodeLabel("p2")); + } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1feb4ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 5cc026a..6694290 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.InlineDispatcher; import org.junit.After; @@ -188,7 +189,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { } @SuppressWarnings({ "unchecked", "rawtypes" }) - @Test//(timeout = 10000) + @Test (timeout = 10000) public void testSerilizationAfterRecovery() throws Exception { mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); mgr.addToCluserNodeLabels(toSet("p4")); @@ -218,6 +219,14 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { * p4: n4 * p6: n6, n7 */ + + mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("p2", false))); + mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("p6", false))); + + /* + * Set p2/p6 to be exclusive + */ + // shutdown mgr and start a new mgr mgr.stop(); @@ -239,6 +248,10 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { "p4", toSet(toNodeId("n4")), "p2", toSet(toNodeId("n2")))); + Assert.assertFalse(mgr.isExclusiveNodeLabel("p2")); + Assert.assertTrue(mgr.isExclusiveNodeLabel("p4")); + Assert.assertFalse(mgr.isExclusiveNodeLabel("p6")); + /* * Add label p7,p8 then shutdown */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1feb4ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 6180995..22b92c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -77,6 +77,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLa import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; @@ -663,6 +665,28 @@ public class AdminService extends CompositeService implements throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); } } + + @Override + public UpdateNodeLabelsResponse updateNodeLabels( + UpdateNodeLabelsRequest request) throws YarnException, IOException { + String argName = "updateNodeLabels"; + final String msg = "update node labels"; + UserGroupInformation user = checkAcls(argName); + + checkRMStatus(user.getShortUserName(), argName, msg); + + UpdateNodeLabelsResponse response = UpdateNodeLabelsResponse.newInstance(); + + try { + rmContext.getNodeLabelManager().updateNodeLabels( + request.getNodeLabels()); + RMAuditLogger + .logSuccess(user.getShortUserName(), argName, "AdminService"); + return response; + } catch (YarnException ioe) { + throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); + } + } private void checkRMStatus(String user, String argName, String msg) throws StandbyException { @@ -673,11 +697,11 @@ public class AdminService extends CompositeService implements } } - private YarnException logAndWrapException(IOException ioe, String user, + private YarnException logAndWrapException(Exception exception, String user, String argName, String msg) throws YarnException { - LOG.info("Exception " + msg, ioe); + LOG.info("Exception " + msg, exception); RMAuditLogger.logFailure(user, argName, "", "AdminService", "Exception " + msg); - return RPCUtil.getRemoteException(ioe); + return RPCUtil.getRemoteException(exception); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1feb4ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index e5abdc9..574e24c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -36,7 +36,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; -import org.apache.hadoop.yarn.nodelabels.NodeLabel; +import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.collect.ImmutableSet; public class RMNodeLabelsManager extends CommonNodeLabelsManager { - protected static class Queue { protected Set<String> acccessibleNodeLabels; protected Resource resource; @@ -156,7 +155,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { throws IOException { try { writeLock.lock(); - + // get nodesCollection before edition Map<String, Host> before = cloneNodeMap(replaceLabelsToNode.keySet()); @@ -171,7 +170,6 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { writeLock.unlock(); } } - /* * Following methods are used for setting if a node is up and running, and it @@ -201,7 +199,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { Set<String> labelsForNode = getLabelsByNode(nodeId); if (labelsForNode != null) { for (String label : labelsForNode) { - NodeLabel labelInfo = labelCollections.get(label); + RMNodeLabel labelInfo = labelCollections.get(label); if(labelInfo != null) { labelInfo.addNodeId(nodeId); } @@ -383,7 +381,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { // no label in the past if (oldLabels.isEmpty()) { // update labels - NodeLabel label = labelCollections.get(NO_LABEL); + RMNodeLabel label = labelCollections.get(NO_LABEL); label.removeNode(oldNM.resource); // update queues, all queue can access this node @@ -393,7 +391,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { } else { // update labels for (String labelName : oldLabels) { - NodeLabel label = labelCollections.get(labelName); + RMNodeLabel label = labelCollections.get(labelName); if (null == label) { continue; } @@ -418,7 +416,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { // no label in the past if (newLabels.isEmpty()) { // update labels - NodeLabel label = labelCollections.get(NO_LABEL); + RMNodeLabel label = labelCollections.get(NO_LABEL); label.addNode(newNM.resource); // update queues, all queue can access this node @@ -428,7 +426,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { } else { // update labels for (String labelName : newLabels) { - NodeLabel label = labelCollections.get(labelName); + RMNodeLabel label = labelCollections.get(labelName); label.addNode(newNM.resource); } @@ -499,13 +497,13 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { this.rmContext = rmContext; } - public List<NodeLabel> pullRMNodeLabelsInfo() { + public List<RMNodeLabel> pullRMNodeLabelsInfo() { try { readLock.lock(); - List<NodeLabel> infos = new ArrayList<NodeLabel>(); + List<RMNodeLabel> infos = new ArrayList<RMNodeLabel>(); - for (Entry<String, NodeLabel> entry : labelCollections.entrySet()) { - NodeLabel label = entry.getValue(); + for (Entry<String, RMNodeLabel> entry : labelCollections.entrySet()) { + RMNodeLabel label = entry.getValue(); infos.add(label.getCopy()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1feb4ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java index 5e8c1ed..7458558 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java @@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; -import org.apache.hadoop.yarn.nodelabels.NodeLabel; +import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.webapp.SubView; @@ -55,7 +55,7 @@ public class NodeLabelsPage extends RmView { tbody(); RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager(); - for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) { + for (RMNodeLabel info : nlm.pullRMNodeLabelsInfo()) { TR<TBODY<TABLE<Hamlet>>> row = tbody.tr().td( info.getLabelName().isEmpty() ? "<NO_LABEL>" : info http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1feb4ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java index b1be525..14d96a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java @@ -20,11 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore; @@ -63,6 +65,12 @@ public class NullRMNodeLabelsManager extends RMNodeLabelsManager { public void close() throws IOException { // do nothing } + + @Override + public void updateNodeLabels(List<NodeLabel> updatedNodeLabels) + throws IOException { + // do nothing + } }; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1feb4ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index 8a37c24..ace8742 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; -import org.apache.hadoop.yarn.nodelabels.NodeLabel; +import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -476,8 +476,8 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { } - private void checkNodeLabelInfo(List<NodeLabel> infos, String labelName, int activeNMs, int memory) { - for (NodeLabel info : infos) { + private void checkNodeLabelInfo(List<RMNodeLabel> infos, String labelName, int activeNMs, int memory) { + for (RMNodeLabel info : infos) { if (info.getLabelName().equals(labelName)) { Assert.assertEquals(activeNMs, info.getNumActiveNMs()); Assert.assertEquals(memory, info.getResource().getMemory()); @@ -499,7 +499,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y"))); // x, y, z and "" - List<NodeLabel> infos = mgr.pullRMNodeLabelsInfo(); + List<RMNodeLabel> infos = mgr.pullRMNodeLabelsInfo(); Assert.assertEquals(4, infos.size()); checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20); checkNodeLabelInfo(infos, "x", 2, 20);