YARN-3413. Changed Nodelabel attributes (like exclusivity) to be settable only via addToClusterNodeLabels but not changeable at runtime. (Wangda Tan via vinodkv)
(cherry picked from commit f5fe35e297ed4a00a1ba93d090207ef67cebcc9d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ec6e7e7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ec6e7e7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ec6e7e7 Branch: refs/heads/branch-2 Commit: 0ec6e7e7f5fd6780713d3bc0a93b88739f461f2d Parents: 48d8d88 Author: Vinod Kumar Vavilapalli <vino...@apache.org> Authored: Thu Apr 23 11:19:55 2015 -0700 Committer: Vinod Kumar Vavilapalli <vino...@apache.org> Committed: Thu Apr 23 11:25:30 2015 -0700 ---------------------------------------------------------------------- .../hadoop/mapred/ResourceMgrDelegate.java | 3 +- .../GetClusterNodeLabelsResponse.java | 9 +- .../hadoop/yarn/api/records/NodeLabel.java | 79 +++++++--- .../ResourceManagerAdministrationProtocol.java | 8 - .../AddToClusterNodeLabelsRequest.java | 24 +-- .../UpdateNodeLabelsRequest.java | 49 ------- .../UpdateNodeLabelsResponse.java | 37 ----- ...esourcemanager_administration_protocol.proto | 1 - ..._server_resourcemanager_service_protos.proto | 7 +- .../src/main/proto/yarn_protos.proto | 2 +- .../src/main/proto/yarn_service_protos.proto | 2 +- .../TestDistributedShellWithNodeLabels.java | 2 +- .../hadoop/yarn/client/api/YarnClient.java | 3 +- .../yarn/client/api/impl/YarnClientImpl.java | 3 +- .../hadoop/yarn/client/cli/ClusterCLI.java | 18 +-- .../hadoop/yarn/client/cli/RMAdminCLI.java | 73 +++++++++- .../hadoop/yarn/client/cli/TestClusterCLI.java | 21 ++- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 60 ++++++-- .../pb/GetClusterNodeLabelsResponsePBImpl.java | 119 +++++++++------ .../api/records/impl/pb/NodeLabelPBImpl.java | 18 +-- .../nodelabels/CommonNodeLabelsManager.java | 95 ++++++------ .../nodelabels/FileSystemNodeLabelsStore.java | 40 ++--- .../hadoop/yarn/nodelabels/NodeLabelsStore.java | 10 +- .../hadoop/yarn/nodelabels/RMNodeLabel.java | 19 ++- .../event/NodeLabelsStoreEventType.java | 3 +- .../event/StoreNewClusterNodeLabels.java | 10 +- .../event/StoreUpdateNodeLabelsEvent.java | 36 ----- ...nagerAdministrationProtocolPBClientImpl.java | 21 +-- ...agerAdministrationProtocolPBServiceImpl.java | 23 --- .../pb/AddToClusterNodeLabelsRequestPBImpl.java | 119 +++++++++------ .../impl/pb/UpdateNodeLabelsRequestPBImpl.java | 145 ------------------- .../impl/pb/UpdateNodeLabelsResponsePBImpl.java | 67 --------- .../hadoop/yarn/api/TestPBImplRecords.java | 18 +-- .../DummyCommonNodeLabelsManager.java | 11 +- .../yarn/nodelabels/NodeLabelTestBase.java | 1 + .../nodelabels/TestCommonNodeLabelsManager.java | 104 ++++++------- .../TestFileSystemNodeLabelsStore.java | 52 +++---- .../server/resourcemanager/AdminService.java | 24 --- .../scheduler/capacity/CSQueueUtils.java | 2 +- .../scheduler/capacity/ParentQueue.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 4 +- .../resourcemanager/TestClientRMService.java | 12 +- .../server/resourcemanager/TestRMRestart.java | 6 +- .../TestResourceTrackerService.java | 12 +- .../nodelabels/NullRMNodeLabelsManager.java | 9 +- .../nodelabels/TestRMNodeLabelsManager.java | 20 +-- .../capacity/TestCapacityScheduler.java | 2 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 2 +- .../capacity/TestContainerAllocation.java | 7 - .../TestNodeLabelContainerAllocation.java | 53 +++---- .../scheduler/capacity/TestQueueParsing.java | 8 +- ...TestWorkPreservingRMRestartForNodeLabel.java | 4 +- 52 files changed, 600 insertions(+), 879 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 279c4f1..2b7cd5f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; @@ -461,7 +462,7 @@ public class ResourceMgrDelegate extends YarnClient { } @Override - public Set<String> getClusterNodeLabels() + public List<NodeLabel> getClusterNodeLabels() throws YarnException, IOException { return client.getClusterNodeLabels(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java index 7bd8223..cf6e683 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeLabelsResponse.java @@ -18,16 +18,17 @@ package org.apache.hadoop.yarn.api.protocolrecords; -import java.util.Set; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.util.Records; @Public @Evolving public abstract class GetClusterNodeLabelsResponse { - public static GetClusterNodeLabelsResponse newInstance(Set<String> labels) { + public static GetClusterNodeLabelsResponse newInstance(List<NodeLabel> labels) { GetClusterNodeLabelsResponse request = Records.newRecord(GetClusterNodeLabelsResponse.class); request.setNodeLabels(labels); @@ -36,9 +37,9 @@ public abstract class GetClusterNodeLabelsResponse { @Public @Evolving - public abstract void setNodeLabels(Set<String> labels); + public abstract void setNodeLabels(List<NodeLabel> labels); @Public @Evolving - public abstract Set<String> getNodeLabels(); + public abstract List<NodeLabel> getNodeLabels(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java index 23da1f4..aba6962 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -25,31 +26,73 @@ import org.apache.hadoop.yarn.util.Records; @Public @Unstable -public abstract class NodeLabel { - @Public +public abstract class NodeLabel implements Comparable<NodeLabel> { + /** + * By default, node label is exclusive or not + */ + @Private + @Unstable + public static final boolean DEFAULT_NODE_LABEL_EXCLUSIVITY = true; + + @Private + @Unstable + public static NodeLabel newInstance(String name) { + return newInstance(name, DEFAULT_NODE_LABEL_EXCLUSIVITY); + } + + @Private @Unstable - public static NodeLabel newInstance(String nodeLabel, - boolean isExclusive) { - NodeLabel request = - Records.newRecord(NodeLabel.class); - request.setNodeLabel(nodeLabel); - request.setIsExclusive(isExclusive); + public static NodeLabel newInstance(String name, boolean isExclusive) { + NodeLabel request = Records.newRecord(NodeLabel.class); + request.setName(name); + request.setExclusivity(isExclusive); return request; } - + @Public @Stable - public abstract String getNodeLabel(); - - @Public + public abstract String getName(); + + @Private @Unstable - public abstract void setNodeLabel(String nodeLabel); - + public abstract void setName(String name); + @Public @Stable - public abstract boolean getIsExclusive(); - - @Public + public abstract boolean isExclusive(); + + @Private @Unstable - public abstract void setIsExclusive(boolean isExclusive); + public abstract void setExclusivity(boolean isExclusive); + + @Override + public int compareTo(NodeLabel other) { + return getName().compareTo(other.getName()); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof NodeLabel) { + NodeLabel nl = (NodeLabel) obj; + return nl.getName().equals(getName()) + && nl.isExclusive() == isExclusive(); + } + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("<"); + sb.append(getName()); + sb.append(":exclusivity="); + sb.append(isExclusive()); + sb.append(">"); + return sb.toString(); + } + + @Override + public int hashCode() { + return (getName().hashCode() << 16) + (isExclusive() ? 1 : 0); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index a23f1fb..0cfa8ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -50,8 +50,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLa import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; @@ -142,12 +140,6 @@ public interface ResourceManagerAdministrationProtocol extends GetUserMappingsPr @Public @Evolving @Idempotent - public UpdateNodeLabelsResponse updateNodeLabels( - UpdateNodeLabelsRequest request) throws YarnException, IOException; - - @Public - @Evolving - @Idempotent public CheckForDecommissioningNodesResponse checkForDecommissioningNodes( CheckForDecommissioningNodesRequest checkForDecommissioningNodesRequest) throws YarnException, IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java index 68fe995..c1ea07d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java @@ -18,27 +18,31 @@ package org.apache.hadoop.yarn.server.api.protocolrecords; -import java.util.Set; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.util.Records; @Public -@Evolving +@Unstable public abstract class AddToClusterNodeLabelsRequest { - public static AddToClusterNodeLabelsRequest newInstance(Set<String> labels) { + @Public + @Unstable + public static AddToClusterNodeLabelsRequest newInstance( + List<NodeLabel> NodeLabels) { AddToClusterNodeLabelsRequest request = - Records.newRecord(AddToClusterNodeLabelsRequest.class); - request.setNodeLabels(labels); + Records.newRecord(AddToClusterNodeLabelsRequest.class); + request.setNodeLabels(NodeLabels); return request; } @Public - @Evolving - public abstract void setNodeLabels(Set<String> labels); + @Unstable + public abstract void setNodeLabels(List<NodeLabel> NodeLabels); @Public - @Evolving - public abstract Set<String> getNodeLabels(); + @Unstable + public abstract List<NodeLabel> getNodeLabels(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java deleted file mode 100644 index 44bdc65..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import java.util.List; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.NodeLabel; -import org.apache.hadoop.yarn.util.Records; - -@Public -@Unstable -public abstract class UpdateNodeLabelsRequest { - @Public - @Unstable - public static UpdateNodeLabelsRequest newInstance( - List<NodeLabel> NodeLabels) { - UpdateNodeLabelsRequest request = - Records.newRecord(UpdateNodeLabelsRequest.class); - request.setNodeLabels(NodeLabels); - return request; - } - - @Public - @Unstable - public abstract void setNodeLabels( - List<NodeLabel> NodeLabels); - - @Public - @Unstable - public abstract List<NodeLabel> getNodeLabels(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java deleted file mode 100644 index eb704c0..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.util.Records; - -@Public -@Unstable -public class UpdateNodeLabelsResponse { - - @Private - @Unstable - public static UpdateNodeLabelsResponse newInstance() { - UpdateNodeLabelsResponse response = - Records.newRecord(UpdateNodeLabelsResponse.class); - return response; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index c7be907..5097084 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -42,6 +42,5 @@ service ResourceManagerAdministrationProtocolService { rpc addToClusterNodeLabels(AddToClusterNodeLabelsRequestProto) returns (AddToClusterNodeLabelsResponseProto); rpc removeFromClusterNodeLabels(RemoveFromClusterNodeLabelsRequestProto) returns (RemoveFromClusterNodeLabelsResponseProto); rpc replaceLabelsOnNodes(ReplaceLabelsOnNodeRequestProto) returns (ReplaceLabelsOnNodeResponseProto); - rpc updateNodeLabels(UpdateNodeLabelsRequestProto) returns (UpdateNodeLabelsResponseProto); rpc checkForDecommissioningNodes(CheckForDecommissioningNodesRequestProto) returns (CheckForDecommissioningNodesResponseProto); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index cf6fe3f..d6d8713 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -77,7 +77,7 @@ message UpdateNodeResourceResponseProto { } message AddToClusterNodeLabelsRequestProto { - repeated string nodeLabels = 1; + repeated NodeLabelProto nodeLabels = 1; } message AddToClusterNodeLabelsResponseProto { @@ -98,11 +98,6 @@ message ReplaceLabelsOnNodeResponseProto { } -message UpdateNodeLabelsRequestProto { - repeated NodeLabelProto nodeLabels = 1; -} - - message UpdateNodeLabelsResponseProto { } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 50d7165..c45081a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -262,7 +262,7 @@ message LabelsToNodeIdsProto { } message NodeLabelProto { - optional string nodeLabel = 1; + optional string name = 1; optional bool isExclusive = 2 [default = true]; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 33d1207..410b663 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -213,7 +213,7 @@ message GetClusterNodeLabelsRequestProto { } message GetClusterNodeLabelsResponseProto { - repeated string nodeLabels = 1; + repeated NodeLabelProto nodeLabels = 1; } ////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java index c04b7fe..b62b091 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java @@ -54,7 +54,7 @@ public class TestDistributedShellWithNodeLabels { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); Set<String> labels = new HashSet<String>(); labels.add("x"); - labelsMgr.addToCluserNodeLabels(labels); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(labels); // Setup queue access to node labels distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index aa49d86..5ce626c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -663,6 +664,6 @@ public abstract class YarnClient extends AbstractService { */ @Public @Unstable - public abstract Set<String> getClusterNodeLabels() + public abstract List<NodeLabel> getClusterNodeLabels() throws YarnException, IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 5eda2c8..42dd5cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -815,7 +816,7 @@ public class YarnClientImpl extends YarnClient { } @Override - public Set<String> getClusterNodeLabels() throws YarnException, IOException { + public List<NodeLabel> getClusterNodeLabels() throws YarnException, IOException { return rmClient.getClusterNodeLabels( GetClusterNodeLabelsRequest.newInstance()).getNodeLabels(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java index 3924803..8765e41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java @@ -25,9 +25,7 @@ import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Set; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -38,6 +36,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; @@ -113,23 +112,16 @@ public class ClusterCLI extends YarnCLI { return 0; } - private List<String> sortStrSet(Set<String> labels) { - List<String> list = new ArrayList<String>(); - list.addAll(labels); - Collections.sort(list); - return list; - } - void printClusterNodeLabels() throws YarnException, IOException { - Set<String> nodeLabels = null; + List<NodeLabel> nodeLabels = null; if (accessLocal) { nodeLabels = - getNodeLabelManagerInstance(getConf()).getClusterNodeLabels(); + new ArrayList<>(getNodeLabelManagerInstance(getConf()).getClusterNodeLabels()); } else { - nodeLabels = client.getClusterNodeLabels(); + nodeLabels = new ArrayList<>(client.getClusterNodeLabels()); } sysout.println(String.format("Node Labels: %s", - StringUtils.join(sortStrSet(nodeLabels).iterator(), ","))); + StringUtils.join(nodeLabels.iterator(), ","))); } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index b2ed730..44abe6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -40,6 +40,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.RMHAServiceTarget; import org.apache.hadoop.yarn.conf.HAUtil; @@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; @Private @Unstable @@ -80,6 +82,9 @@ public class RMAdminCLI extends HAAdmin { "No node-to-labels mappings are specified"; private static final String INVALID_TIMEOUT_ERR_MSG = "Invalid timeout specified : "; + private static final String ADD_LABEL_FORMAT_ERR_MSG = + "Input format for adding node-labels is not correct, it should be " + + "labelName1[(exclusive=true/false)],LabelName2[] .."; protected final static Map<String, UsageInfo> ADMIN_USAGE = ImmutableMap.<String, UsageInfo>builder() @@ -104,7 +109,8 @@ public class RMAdminCLI extends HAAdmin { .put("-getGroups", new UsageInfo("[username]", "Get the groups which given user belongs to.")) .put("-addToClusterNodeLabels", - new UsageInfo("[label1,label2,label3] (label splitted by \",\")", + new UsageInfo("[label1(exclusive=true)," + + "label2(exclusive=false),label3]", "add to cluster node labels ")) .put("-removeFromClusterNodeLabels", new UsageInfo("[label1,label2,label3] (label splitted by \",\")", @@ -407,7 +413,66 @@ public class RMAdminCLI extends HAAdmin { return localNodeLabelsManager; } - private Set<String> buildNodeLabelsSetFromStr(String args) { + private List<NodeLabel> buildNodeLabelsFromStr(String args) { + List<NodeLabel> nodeLabels = new ArrayList<>(); + for (String p : args.split(",")) { + if (!p.trim().isEmpty()) { + String labelName = p; + + // Try to parse exclusive + boolean exclusive = NodeLabel.DEFAULT_NODE_LABEL_EXCLUSIVITY; + int leftParenthesisIdx = p.indexOf("("); + int rightParenthesisIdx = p.indexOf(")"); + + if ((leftParenthesisIdx == -1 && rightParenthesisIdx != -1) + || (leftParenthesisIdx != -1 && rightParenthesisIdx == -1)) { + // Parenthese not match + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + + if (leftParenthesisIdx > 0 && rightParenthesisIdx > 0) { + if (leftParenthesisIdx > rightParenthesisIdx) { + // Parentese not match + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + + String property = p.substring(p.indexOf("(") + 1, p.indexOf(")")); + if (property.contains("=")) { + String key = property.substring(0, property.indexOf("=")).trim(); + String value = + property + .substring(property.indexOf("=") + 1, property.length()) + .trim(); + + // Now we only support one property, which is exclusive, so check if + // key = exclusive and value = {true/false} + if (key.equals("exclusive") + && ImmutableSet.of("true", "false").contains(value)) { + exclusive = Boolean.parseBoolean(value); + } else { + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + } else if (!property.trim().isEmpty()) { + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + } + + // Try to get labelName if there's "(..)" + if (labelName.contains("(")) { + labelName = labelName.substring(0, labelName.indexOf("(")).trim(); + } + + nodeLabels.add(NodeLabel.newInstance(labelName, exclusive)); + } + } + + if (nodeLabels.isEmpty()) { + throw new IllegalArgumentException(NO_LABEL_ERR_MSG); + } + return nodeLabels; + } + + private Set<String> buildNodeLabelNamesFromStr(String args) { Set<String> labels = new HashSet<String>(); for (String p : args.split(",")) { if (!p.trim().isEmpty()) { @@ -423,7 +488,7 @@ public class RMAdminCLI extends HAAdmin { private int addToClusterNodeLabels(String args) throws IOException, YarnException { - Set<String> labels = buildNodeLabelsSetFromStr(args); + List<NodeLabel> labels = buildNodeLabelsFromStr(args); if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).addToCluserNodeLabels(labels); @@ -439,7 +504,7 @@ public class RMAdminCLI extends HAAdmin { private int removeFromClusterNodeLabels(String args) throws IOException, YarnException { - Set<String> labels = buildNodeLabelsSetFromStr(args); + Set<String> labels = buildNodeLabelNamesFromStr(args); if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).removeFromClusterNodeLabels( http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java index f9ccf87..d9c7de9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java @@ -27,8 +27,10 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.PrintWriter; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.Arrays; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.junit.Before; @@ -55,7 +57,8 @@ public class TestClusterCLI { public void testGetClusterNodeLabels() throws Exception { YarnClient client = mock(YarnClient.class); when(client.getClusterNodeLabels()).thenReturn( - ImmutableSet.of("label1", "label2")); + Arrays.asList(NodeLabel.newInstance("label1"), + NodeLabel.newInstance("label2"))); ClusterCLI cli = new ClusterCLI(); cli.setClient(client); cli.setSysOutPrintStream(sysOut); @@ -67,7 +70,7 @@ public class TestClusterCLI { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); - pw.print("Node Labels: label1,label2"); + pw.print("Node Labels: <label1:exclusivity=true>,<label2:exclusivity=true>"); pw.close(); verify(sysOut).println(baos.toString("UTF-8")); } @@ -76,14 +79,16 @@ public class TestClusterCLI { public void testGetClusterNodeLabelsWithLocalAccess() throws Exception { YarnClient client = mock(YarnClient.class); when(client.getClusterNodeLabels()).thenReturn( - ImmutableSet.of("remote1", "remote2")); + Arrays.asList(NodeLabel.newInstance("remote1"), + NodeLabel.newInstance("remote2"))); ClusterCLI cli = new ClusterCLI(); cli.setClient(client); cli.setSysOutPrintStream(sysOut); cli.setSysErrPrintStream(sysErr); ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); - when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels()) - .thenReturn(ImmutableSet.of("local1", "local2")); + when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels()).thenReturn( + Arrays.asList(NodeLabel.newInstance("local1"), + NodeLabel.newInstance("local2"))); int rc = cli.run(new String[] { ClusterCLI.CMD, @@ -94,7 +99,7 @@ public class TestClusterCLI { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); // it should return local* instead of remote* - pw.print("Node Labels: local1,local2"); + pw.print("Node Labels: <local1:exclusivity=true>,<local2:exclusivity=true>"); pw.close(); verify(sysOut).println(baos.toString("UTF-8")); } @@ -102,7 +107,7 @@ public class TestClusterCLI { @Test public void testGetEmptyClusterNodeLabels() throws Exception { YarnClient client = mock(YarnClient.class); - when(client.getClusterNodeLabels()).thenReturn(new HashSet<String>()); + when(client.getClusterNodeLabels()).thenReturn(new ArrayList<NodeLabel>()); ClusterCLI cli = new ClusterCLI(); cli.setClient(client); cli.setSysOutPrintStream(sysOut); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index f64a1f0..f8f23f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -454,7 +454,7 @@ public class TestRMAdminCLI { String[] args = { "-addToClusterNodeLabels", "x,y", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x", "y"))); // reset localNodeLabelsManager @@ -466,7 +466,7 @@ public class TestRMAdminCLI { new String[] { "-addToClusterNodeLabels", "-directlyAccessNodeLabelStore", "x,y" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x", "y"))); // local node labels manager will be close after running @@ -480,7 +480,7 @@ public class TestRMAdminCLI { assertEquals(0, rmAdminCLI.run(args)); // localNodeLabelsManager shouldn't accessed - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().isEmpty()); + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().isEmpty()); // remote node labels manager accessed assertTrue(remoteAdminServiceAccessed); @@ -492,7 +492,7 @@ public class TestRMAdminCLI { String[] args = { "-addToClusterNodeLabels", "x", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x"))); // no labels, should fail @@ -518,19 +518,61 @@ public class TestRMAdminCLI { new String[] { "-addToClusterNodeLabels", ",x,,", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x"))); } @Test + public void testAddToClusterNodeLabelsWithExclusivitySetting() + throws Exception { + // Parenthese not match + String[] args = new String[] { "-addToClusterNodeLabels", "x(" }; + assertTrue(0 != rmAdminCLI.run(args)); + + args = new String[] { "-addToClusterNodeLabels", "x)" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // Not expected key=value specifying inner parentese + args = new String[] { "-addToClusterNodeLabels", "x(key=value)" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // Not key is expected, but value not + args = new String[] { "-addToClusterNodeLabels", "x(exclusive=)" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // key=value both set + args = + new String[] { "-addToClusterNodeLabels", + "w,x(exclusive=true), y(exclusive=false),z()", + "-directlyAccessNodeLabelStore" }; + assertTrue(0 == rmAdminCLI.run(args)); + + assertTrue(dummyNodeLabelsManager.isExclusiveNodeLabel("w")); + assertTrue(dummyNodeLabelsManager.isExclusiveNodeLabel("x")); + assertFalse(dummyNodeLabelsManager.isExclusiveNodeLabel("y")); + assertTrue(dummyNodeLabelsManager.isExclusiveNodeLabel("z")); + + // key=value both set, and some spaces need to be handled + args = + new String[] { "-addToClusterNodeLabels", + "a (exclusive= true) , b( exclusive =false),c ", + "-directlyAccessNodeLabelStore" }; + assertTrue(0 == rmAdminCLI.run(args)); + + assertTrue(dummyNodeLabelsManager.isExclusiveNodeLabel("a")); + assertFalse(dummyNodeLabelsManager.isExclusiveNodeLabel("b")); + assertTrue(dummyNodeLabelsManager.isExclusiveNodeLabel("c")); + } + + @Test public void testRemoveFromClusterNodeLabels() throws Exception { // Successfully remove labels - dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + dummyNodeLabelsManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); String[] args = { "-removeFromClusterNodeLabels", "x,,y", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().isEmpty()); + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().isEmpty()); // no labels, should fail args = new String[] { "-removeFromClusterNodeLabels" }; @@ -555,7 +597,7 @@ public class TestRMAdminCLI { public void testReplaceLabelsOnNode() throws Exception { // Successfully replace labels dummyNodeLabelsManager - .addToCluserNodeLabels(ImmutableSet.of("x", "y", "Y")); + .addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "Y")); String[] args = { "-replaceLabelsOnNode", "node1:8000,x node2:8000=y node3,x node4=Y", @@ -590,7 +632,7 @@ public class TestRMAdminCLI { @Test public void testReplaceMultipleLabelsOnSingleNode() throws Exception { // Successfully replace labels - dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + dummyNodeLabelsManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); String[] args = { "-replaceLabelsOnNode", "node1,x,y", "-directlyAccessNodeLabelStore" }; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java index 8d8331f..f569fb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java @@ -18,22 +18,26 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; -import java.util.HashSet; -import java.util.Set; +import java.util.ArrayList; +import java.util.List; + import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeLabelsResponseProtoOrBuilder; public class GetClusterNodeLabelsResponsePBImpl extends GetClusterNodeLabelsResponse { - Set<String> labels; GetClusterNodeLabelsResponseProto proto = GetClusterNodeLabelsResponseProto .getDefaultInstance(); GetClusterNodeLabelsResponseProto.Builder builder = null; + private List<NodeLabel> updatedNodeLabels; boolean viaProto = false; public GetClusterNodeLabelsResponsePBImpl() { - this.builder = GetClusterNodeLabelsResponseProto.newBuilder(); + builder = GetClusterNodeLabelsResponseProto.newBuilder(); } public GetClusterNodeLabelsResponsePBImpl( @@ -42,18 +46,11 @@ public class GetClusterNodeLabelsResponsePBImpl extends viaProto = true; } - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = GetClusterNodeLabelsResponseProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.labels != null && !this.labels.isEmpty()) { - builder.clearNodeLabels(); - builder.addAllNodeLabels(this.labels); - } + public GetClusterNodeLabelsResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; } private void mergeLocalToProto() { @@ -64,35 +61,30 @@ public class GetClusterNodeLabelsResponsePBImpl extends viaProto = true; } - public GetClusterNodeLabelsResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void initNodeLabels() { - if (this.labels != null) { - return; + private void mergeLocalToBuilder() { + if (this.updatedNodeLabels != null) { + addNodeLabelsToProto(); } - GetClusterNodeLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; - this.labels = new HashSet<String>(); - this.labels.addAll(p.getNodeLabelsList()); } - @Override - public void setNodeLabels(Set<String> labels) { + private void addNodeLabelsToProto() { maybeInitBuilder(); - if (labels == null || labels.isEmpty()) { - builder.clearNodeLabels(); + builder.clearNodeLabels(); + List<NodeLabelProto> protoList = new ArrayList<NodeLabelProto>(); + for (NodeLabel r : this.updatedNodeLabels) { + protoList.add(convertToProtoFormat(r)); } - this.labels = labels; + builder.addAllNodeLabels(protoList); } @Override - public Set<String> getNodeLabels() { - initNodeLabels(); - return this.labels; + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; } @Override @@ -100,14 +92,53 @@ public class GetClusterNodeLabelsResponsePBImpl extends assert false : "hashCode not designed"; return 0; } - + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetClusterNodeLabelsResponseProto.newBuilder(proto); + } + viaProto = false; + } + @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); + public void setNodeLabels(List<NodeLabel> updatedNodeLabels) { + maybeInitBuilder(); + this.updatedNodeLabels = new ArrayList<>(); + if (updatedNodeLabels == null) { + builder.clearNodeLabels(); + return; } - return false; + this.updatedNodeLabels.addAll(updatedNodeLabels); + } + + private void initLocalNodeLabels() { + GetClusterNodeLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; + List<NodeLabelProto> attributesProtoList = p.getNodeLabelsList(); + this.updatedNodeLabels = new ArrayList<NodeLabel>(); + for (NodeLabelProto r : attributesProtoList) { + this.updatedNodeLabels.add(convertFromProtoFormat(r)); + } + } + + @Override + public List<NodeLabel> getNodeLabels() { + if (this.updatedNodeLabels != null) { + return this.updatedNodeLabels; + } + initLocalNodeLabels(); + return this.updatedNodeLabels; + } + + private NodeLabel convertFromProtoFormat(NodeLabelProto p) { + return new NodeLabelPBImpl(p); + } + + private NodeLabelProto convertToProtoFormat(NodeLabel t) { + return ((NodeLabelPBImpl) t).getProto(); + } + + @Override + public String toString() { + return getProto().toString(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeLabelPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeLabelPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeLabelPBImpl.java index 661359e..1283f36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeLabelPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeLabelPBImpl.java @@ -73,32 +73,32 @@ public class NodeLabelPBImpl extends NodeLabel { } @Override - public String getNodeLabel() { + public String getName() { NodeLabelProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasNodeLabel()) { + if (!p.hasName()) { return null; } - return (p.getNodeLabel()); + return (p.getName()); } @Override - public void setNodeLabel(String nodeLabel) { + public void setName(String name) { maybeInitBuilder(); - if (nodeLabel == null) { - builder.clearNodeLabel(); + if (name == null) { + builder.clearName(); return; } - builder.setNodeLabel(nodeLabel); + builder.setName(name); } @Override - public boolean getIsExclusive() { + public boolean isExclusive() { NodeLabelProtoOrBuilder p = viaProto ? proto : builder; return p.getIsExclusive(); } @Override - public void setIsExclusive(boolean isExclusive) { + public void setExclusivity(boolean isExclusive) { maybeInitBuilder(); builder.setIsExclusive(isExclusive); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index fe38164..7493169 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.nodelabels; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -47,13 +48,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEvent; import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEventType; import org.apache.hadoop.yarn.nodelabels.event.RemoveClusterNodeLabels; import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels; -import org.apache.hadoop.yarn.nodelabels.event.StoreUpdateNodeLabelsEvent; import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -185,13 +184,6 @@ public class CommonNodeLabelsManager extends AbstractService { store.updateNodeToLabelsMappings(updateNodeToLabelsMappingsEvent .getNodeToLabels()); break; - case UPDATE_NODE_LABELS: - StoreUpdateNodeLabelsEvent - storeSetNodeLabelsEventEvent = - (StoreUpdateNodeLabelsEvent) event; - store.updateNodeLabels(storeSetNodeLabelsEventEvent - .getUpdatedNodeLabels()); - break; } } catch (IOException e) { LOG.error("Failed to store label modification to storage"); @@ -274,14 +266,9 @@ public class CommonNodeLabelsManager extends AbstractService { } } - /** - * Add multiple node labels to repository - * - * @param labels - * new node labels added - */ @SuppressWarnings("unchecked") - public void addToCluserNodeLabels(Set<String> labels) throws IOException { + public void addToCluserNodeLabels(Collection<NodeLabel> labels) + throws IOException { if (!nodeLabelsEnabled) { LOG.error(NODE_LABELS_NOT_ENABLED_ERR); throw new IOException(NODE_LABELS_NOT_ENABLED_ERR); @@ -289,19 +276,19 @@ public class CommonNodeLabelsManager extends AbstractService { if (null == labels || labels.isEmpty()) { return; } - Set<String> newLabels = new HashSet<String>(); - labels = normalizeLabels(labels); + List<NodeLabel> newLabels = new ArrayList<NodeLabel>(); + normalizeNodeLabels(labels); // do a check before actual adding them, will throw exception if any of them // doesn't meet label name requirement - for (String label : labels) { - checkAndThrowLabelName(label); + for (NodeLabel label : labels) { + checkAndThrowLabelName(label.getName()); } - for (String label : labels) { + for (NodeLabel label : labels) { // shouldn't overwrite it to avoid changing the Label.resource - if (this.labelCollections.get(label) == null) { - this.labelCollections.put(label, new RMNodeLabel(label)); + if (this.labelCollections.get(label.getName()) == null) { + this.labelCollections.put(label.getName(), new RMNodeLabel(label)); newLabels.add(label); } } @@ -312,6 +299,22 @@ public class CommonNodeLabelsManager extends AbstractService { LOG.info("Add labels: [" + StringUtils.join(labels.iterator(), ",") + "]"); } + + /** + * Add multiple node labels to repository + * + * @param labels + * new node labels added + */ + @VisibleForTesting + public void addToCluserNodeLabelsWithDefaultExclusivity(Set<String> labels) + throws IOException { + Set<NodeLabel> nodeLabels = new HashSet<NodeLabel>(); + for (String label : labels) { + nodeLabels.add(NodeLabel.newInstance(label)); + } + addToCluserNodeLabels(nodeLabels); + } protected void checkAddLabelsToNode( Map<NodeId, Set<String>> addedLabelsToNode) throws IOException { @@ -780,7 +783,7 @@ public class CommonNodeLabelsManager extends AbstractService { * * @return existing valid labels in repository */ - public Set<String> getClusterNodeLabels() { + public Set<String> getClusterNodeLabelNames() { try { readLock.lock(); Set<String> labels = new HashSet<String>(labelCollections.keySet()); @@ -791,39 +794,17 @@ public class CommonNodeLabelsManager extends AbstractService { } } - private void checkUpdateNodeLabels( - List<NodeLabel> updatedNodeLabels) throws YarnException { - // pre-check - for (NodeLabel label : updatedNodeLabels) { - if (!labelCollections.containsKey(label.getNodeLabel())) { - String message = - String.format( - "Trying to update a non-existing node-label=%s", - label.getNodeLabel()); - LOG.error(message); - throw new YarnException(message); - } - } - } - - @SuppressWarnings("unchecked") - public void updateNodeLabels( - List<NodeLabel> updatedNodeLabels) throws YarnException { + public List<NodeLabel> getClusterNodeLabels() { try { - writeLock.lock(); - checkUpdateNodeLabels(updatedNodeLabels); - - for (NodeLabel label : updatedNodeLabels) { - RMNodeLabel rmLabel = labelCollections.get(label.getNodeLabel()); - rmLabel.setIsExclusive(label.getIsExclusive()); - } - - if (null != dispatcher && !updatedNodeLabels.isEmpty()) { - dispatcher.getEventHandler().handle( - new StoreUpdateNodeLabelsEvent(updatedNodeLabels)); + readLock.lock(); + List<NodeLabel> nodeLabels = new ArrayList<>(); + for (RMNodeLabel label : labelCollections.values()) { + nodeLabels.add(NodeLabel.newInstance(label.getLabelName(), + label.getIsExclusive())); } + return nodeLabels; } finally { - writeLock.unlock(); + readLock.unlock(); } } @@ -876,6 +857,12 @@ public class CommonNodeLabelsManager extends AbstractService { return newLabels; } + private void normalizeNodeLabels(Collection<NodeLabel> labels) { + for (NodeLabel label : labels) { + label.setName(normalizeLabel(label.getName())); + } + } + protected Node getNMInNodeSet(NodeId nodeId) { return getNMInNodeSet(nodeId, nodeCollections); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 0c7192f..ea185f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -41,15 +41,12 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; import com.google.common.collect.Sets; @@ -66,7 +63,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { protected static final String EDITLOG_FILENAME = "nodelabel.editlog"; protected enum SerializedLogType { - ADD_LABELS, NODE_TO_LABELS, REMOVE_LABELS, UPDATE_NODE_LABELS + ADD_LABELS, NODE_TO_LABELS, REMOVE_LABELS } Path fsWorkingPath; @@ -138,12 +135,12 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { } @Override - public void storeNewClusterNodeLabels(Set<String> labels) + public void storeNewClusterNodeLabels(List<NodeLabel> labels) throws IOException { ensureAppendEditlogFile(); editlogOs.writeInt(SerializedLogType.ADD_LABELS.ordinal()); - ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest.newInstance(labels)).getProto() - .writeDelimitedTo(editlogOs); + ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest + .newInstance(labels)).getProto().writeDelimitedTo(editlogOs); ensureCloseEditlogFile(); } @@ -156,16 +153,6 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { .newHashSet(labels.iterator()))).getProto().writeDelimitedTo(editlogOs); ensureCloseEditlogFile(); } - - @Override - public void updateNodeLabels(List<NodeLabel> updatedNodeLabels) - throws IOException { - ensureAppendEditlogFile(); - editlogOs.writeInt(SerializedLogType.UPDATE_NODE_LABELS.ordinal()); - ((UpdateNodeLabelsRequestPBImpl) UpdateNodeLabelsRequest - .newInstance(updatedNodeLabels)).getProto().writeDelimitedTo(editlogOs); - ensureCloseEditlogFile(); - } @Override public void recover() throws YarnException, IOException { @@ -192,7 +179,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { } if (null != is) { - Set<String> labels = + List<NodeLabel> labels = new AddToClusterNodeLabelsRequestPBImpl( AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)).getNodeLabels(); Map<NodeId, Set<String>> nodeToLabels = @@ -216,10 +203,11 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { switch (type) { case ADD_LABELS: { - Collection<String> labels = - AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is) - .getNodeLabelsList(); - mgr.addToCluserNodeLabels(Sets.newHashSet(labels.iterator())); + List<NodeLabel> labels = + new AddToClusterNodeLabelsRequestPBImpl( + AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)) + .getNodeLabels(); + mgr.addToCluserNodeLabels(labels); break; } case REMOVE_LABELS: { @@ -237,14 +225,6 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { mgr.replaceLabelsOnNode(map); break; } - case UPDATE_NODE_LABELS: { - List<NodeLabel> attributes = - new UpdateNodeLabelsRequestPBImpl( - UpdateNodeLabelsRequestProto.parseDelimitedFrom(is)) - .getNodeLabels(); - mgr.updateNodeLabels(attributes); - break; - } } } catch (EOFException e) { // EOF hit, break http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java index 6bd90db..47b7370 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java @@ -46,7 +46,7 @@ public abstract class NodeLabelsStore implements Closeable { /** * Store new labels */ - public abstract void storeNewClusterNodeLabels(Set<String> label) + public abstract void storeNewClusterNodeLabels(List<NodeLabel> label) throws IOException; /** @@ -54,13 +54,7 @@ public abstract class NodeLabelsStore implements Closeable { */ public abstract void removeClusterNodeLabels(Collection<String> labels) throws IOException; - - /** - * Update node labels - */ - public abstract void updateNodeLabels( - List<NodeLabel> updatedNodeLabels) throws IOException; - + /** * Recover labels and node to labels mappings from store */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java index 7638e21..1c4fd17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java @@ -20,9 +20,12 @@ package org.apache.hadoop.yarn.nodelabels; import java.util.HashSet; import java.util.Set; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.resource.Resources; public class RMNodeLabel implements Comparable<RMNodeLabel> { @@ -30,17 +33,25 @@ public class RMNodeLabel implements Comparable<RMNodeLabel> { private int numActiveNMs; private String labelName; private Set<NodeId> nodeIds; - private boolean exclusive = true; + private boolean exclusive; + + public RMNodeLabel(NodeLabel nodeLabel) { + this(nodeLabel.getName(), Resource.newInstance(0, 0), 0, + nodeLabel.isExclusive()); + } public RMNodeLabel(String labelName) { - this(labelName, Resource.newInstance(0, 0), 0); + this(labelName, Resource.newInstance(0, 0), 0, + NodeLabel.DEFAULT_NODE_LABEL_EXCLUSIVITY); } - protected RMNodeLabel(String labelName, Resource res, int activeNMs) { + protected RMNodeLabel(String labelName, Resource res, int activeNMs, + boolean exclusive) { this.labelName = labelName; this.resource = res; this.numActiveNMs = activeNMs; this.nodeIds = new HashSet<NodeId>(); + this.exclusive = exclusive; } public void addNodeId(NodeId node) { @@ -86,7 +97,7 @@ public class RMNodeLabel implements Comparable<RMNodeLabel> { } public RMNodeLabel getCopy() { - return new RMNodeLabel(labelName, resource, numActiveNMs); + return new RMNodeLabel(labelName, resource, numActiveNMs, exclusive); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java index ce29176..efa2dbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java @@ -21,6 +21,5 @@ package org.apache.hadoop.yarn.nodelabels.event; public enum NodeLabelsStoreEventType { REMOVE_LABELS, ADD_LABELS, - STORE_NODE_TO_LABELS, - UPDATE_NODE_LABELS + STORE_NODE_TO_LABELS } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java index b478c6b..c5a7d01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java @@ -18,17 +18,19 @@ package org.apache.hadoop.yarn.nodelabels.event; -import java.util.Set; +import java.util.List; + +import org.apache.hadoop.yarn.api.records.NodeLabel; public class StoreNewClusterNodeLabels extends NodeLabelsStoreEvent { - private Set<String> labels; + private List<NodeLabel> labels; - public StoreNewClusterNodeLabels(Set<String> labels) { + public StoreNewClusterNodeLabels(List<NodeLabel> labels) { super(NodeLabelsStoreEventType.ADD_LABELS); this.labels = labels; } - public Set<String> getLabels() { + public List<NodeLabel> getLabels() { return labels; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreUpdateNodeLabelsEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreUpdateNodeLabelsEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreUpdateNodeLabelsEvent.java deleted file mode 100644 index 7c05012..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreUpdateNodeLabelsEvent.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.nodelabels.event; - -import java.util.List; - -import org.apache.hadoop.yarn.api.records.NodeLabel; - -public class StoreUpdateNodeLabelsEvent extends NodeLabelsStoreEvent { - private List<NodeLabel> updatedNodeLabels; - - public StoreUpdateNodeLabelsEvent(List<NodeLabel> updateNodeLabels) { - super(NodeLabelsStoreEventType.UPDATE_NODE_LABELS); - this.updatedNodeLabels = updateNodeLabels; - } - - public List<NodeLabel> getUpdatedNodeLabels() { - return updatedNodeLabels; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index c6adfd8..02bf045 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -41,7 +41,6 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Refre import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; @@ -65,8 +64,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLa import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; @@ -89,8 +86,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClust import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; @@ -275,20 +270,6 @@ public class ResourceManagerAdministrationProtocolPBClientImpl implements Resour } @Override - public UpdateNodeLabelsResponse updateNodeLabels( - UpdateNodeLabelsRequest request) throws YarnException, IOException { - UpdateNodeLabelsRequestProto requestProto = - ((UpdateNodeLabelsRequestPBImpl) request).getProto(); - try { - return new UpdateNodeLabelsResponsePBImpl( - proxy.updateNodeLabels(null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } - - @Override public CheckForDecommissioningNodesResponse checkForDecommissioningNodes( CheckForDecommissioningNodesRequest checkForDecommissioningNodesRequest) throws YarnException, IOException { @@ -303,4 +284,4 @@ public class ResourceManagerAdministrationProtocolPBClientImpl implements Resour return null; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index adeca3c..0989347 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -44,8 +44,6 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Remov import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; @@ -61,8 +59,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsC import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; @@ -84,8 +80,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClust import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; @@ -282,23 +276,6 @@ public class ResourceManagerAdministrationProtocolPBServiceImpl implements Resou } @Override - public UpdateNodeLabelsResponseProto updateNodeLabels( - RpcController controller, UpdateNodeLabelsRequestProto proto) - throws ServiceException { - UpdateNodeLabelsRequest request = - new UpdateNodeLabelsRequestPBImpl(proto); - try { - UpdateNodeLabelsResponse response = - real.updateNodeLabels(request); - return ((UpdateNodeLabelsResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } - - @Override public CheckForDecommissioningNodesResponseProto checkForDecommissioningNodes( RpcController controller, CheckForDecommissioningNodesRequestProto proto) throws ServiceException {
