YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecf3c51f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecf3c51f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecf3c51f Branch: refs/heads/YARN-1011 Commit: ecf3c51ff3a4d20cf9db845cc1270162507598b4 Parents: 52ec505 Author: Miklos Szegedi <szege...@apache.org> Authored: Tue Nov 28 14:01:30 2017 -0800 Committer: Haibo Chen <haiboc...@apache.org> Committed: Mon Jun 4 21:01:33 2018 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/server/utils/BuilderUtils.java | 15 +++- .../yarn/server/utils/TestBuilderUtils.java | 54 ++++++++++++++ .../server/resourcemanager/RMAppManager.java | 6 +- .../server/resourcemanager/TestAppManager.java | 76 +++++++++++++++----- 4 files changed, 131 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index c906b7e..fd9a9fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -349,15 +349,28 @@ public class BuilderUtils { return priority; } + public static ResourceRequest newAMResourceRequest(Priority priority, + String hostName, Resource capability) { + return newResourceRequest(priority, hostName, capability, 1, + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true)); + } + public static ResourceRequest newResourceRequest(Priority priority, String hostName, Resource capability, int numContainers) { + return newResourceRequest(priority, hostName, capability, + numContainers, ExecutionTypeRequest.newInstance()); + } + + public static ResourceRequest newResourceRequest(Priority priority, + String hostName, Resource capability, int numContainers, + ExecutionTypeRequest executionTypeRequest) { ResourceRequest request = recordFactory .newRecordInstance(ResourceRequest.class); request.setPriority(priority); request.setResourceName(hostName); request.setCapability(capability); request.setNumContainers(numContainers); - request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance()); + request.setExecutionTypeRequest(executionTypeRequest); return request; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java new file mode 100644 index 0000000..4d5ab79 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.utils; + +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for BuilderUtils. + */ +public class TestBuilderUtils { + @Test + public void testNewAMResourceRequest() { + final Priority priority = Priority.newInstance(0); + final String hostName = "ResourceA"; + final Resource capacity = Resource.newInstance(1024, 1); + + ResourceRequest resourceRequest = + BuilderUtils.newAMResourceRequest(priority, hostName, capacity); + + Assert.assertEquals("unexpected priority", priority, + resourceRequest.getPriority()); + Assert.assertEquals("unexpected host name", hostName, + resourceRequest.getResourceName()); + Assert.assertEquals("unexpected capacity", capacity, + resourceRequest.getCapability()); + Assert.assertEquals("unexpected number of containers", 1, + resourceRequest.getNumContainers()); + Assert.assertEquals("unexpected execution type", + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true), + resourceRequest.getExecutionTypeRequest()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 3e64cfc..357f1c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -483,8 +483,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, if (amReqs == null || amReqs.isEmpty()) { if (submissionContext.getResource() != null) { amReqs = Collections.singletonList(BuilderUtils - .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, - ResourceRequest.ANY, submissionContext.getResource(), 1)); + .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, + ResourceRequest.ANY, submissionContext.getResource())); } else { throw new InvalidResourceRequestException("Invalid resource request, " + "no resources requested"); @@ -515,7 +515,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, for (ResourceRequest amReq : amReqs) { amReq.setCapability(anyReq.getCapability()); amReq.setExecutionTypeRequest( - ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true)); amReq.setNumContainers(1); amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecf3c51f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index e79ba08..c95ed85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -700,15 +700,45 @@ public class TestAppManager{ public void testRMAppSubmitAMContainerResourceRequests() throws Exception { asContext.setResource(Resources.createResource(1024)); asContext.setAMContainerResourceRequest( - ResourceRequest.newInstance(Priority.newInstance(0), - ResourceRequest.ANY, Resources.createResource(1024), 1, true)); + ResourceRequest.newBuilder().priority(Priority.newInstance(0)) + .resourceName(ResourceRequest.ANY) + .capability(Resources.createResource(1024)) + .numContainers(1) + .relaxLocality(true) + .executionTypeRequest(ExecutionTypeRequest.newInstance( + ExecutionType.GUARANTEED, true)) + .build()); List<ResourceRequest> reqs = new ArrayList<>(); - reqs.add(ResourceRequest.newInstance(Priority.newInstance(0), - ResourceRequest.ANY, Resources.createResource(1025), 1, false)); - reqs.add(ResourceRequest.newInstance(Priority.newInstance(0), - "/rack", Resources.createResource(1025), 1, false)); - reqs.add(ResourceRequest.newInstance(Priority.newInstance(0), - "/rack/node", Resources.createResource(1025), 1, true)); + reqs.add( + ResourceRequest.newBuilder().priority(Priority.newInstance(0)) + .resourceName(ResourceRequest.ANY) + .capability(Resources.createResource(1025)) + .numContainers(1) + .relaxLocality(false) + .executionTypeRequest(ExecutionTypeRequest.newInstance( + ExecutionType.GUARANTEED, true)) + .build() + ); + reqs.add( + ResourceRequest.newBuilder().priority(Priority.newInstance(0)) + .resourceName("/rack") + .capability(Resources.createResource(1025)) + .numContainers(1) + .relaxLocality(false) + .executionTypeRequest(ExecutionTypeRequest.newInstance( + ExecutionType.GUARANTEED, true)) + .build() + ); + reqs.add( + ResourceRequest.newBuilder().priority(Priority.newInstance(0)) + .resourceName("/rack/node") + .capability(Resources.createResource(1025)) + .numContainers(1) + .relaxLocality(true) + .executionTypeRequest(ExecutionTypeRequest.newInstance( + ExecutionType.GUARANTEED, true)) + .build() + ); asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs)); // getAMContainerResourceRequest uses the first entry of // getAMContainerResourceRequests @@ -730,9 +760,15 @@ public class TestAppManager{ asContext.setResource(Resources.createResource(1024)); asContext.setAMContainerResourceRequests(null); ResourceRequest req = - ResourceRequest.newInstance(Priority.newInstance(0), - ResourceRequest.ANY, Resources.createResource(1025), 1, true); - req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL); + ResourceRequest.newBuilder().priority(Priority.newInstance(0)) + .resourceName(ResourceRequest.ANY) + .capability(Resources.createResource(1025)) + .numContainers(1) + .relaxLocality(true) + .nodeLabelExpression(RMNodeLabelsManager.NO_LABEL) + .executionTypeRequest(ExecutionTypeRequest.newInstance( + ExecutionType.GUARANTEED, true)) + .build(); asContext.setAMContainerResourceRequest(cloneResourceRequest(req)); // getAMContainerResourceRequests uses a singleton list of // getAMContainerResourceRequest @@ -753,9 +789,16 @@ public class TestAppManager{ // setResource Assert.assertEquals(Collections.singletonList( - ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, - ResourceRequest.ANY, Resources.createResource(1024), 1, true, - "")), + ResourceRequest.newBuilder() + .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY) + .resourceName(ResourceRequest.ANY) + .capability(Resources.createResource(1024)) + .numContainers(1) + .relaxLocality(true) + .nodeLabelExpression("") + .executionTypeRequest(ExecutionTypeRequest.newInstance( + ExecutionType.GUARANTEED, true)) + .build()), app.getAMResourceRequests()); } @@ -797,7 +840,7 @@ public class TestAppManager{ for (ResourceRequest req : reqs) { req.setCapability(anyReq.getCapability()); req.setExecutionTypeRequest( - ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true)); req.setNumContainers(1); req.setPriority(Priority.newInstance(0)); } @@ -1110,7 +1153,8 @@ public class TestAppManager{ req.getNodeLabelExpression() != null ? new String(req.getNodeLabelExpression()) : null, ExecutionTypeRequest.newInstance( - req.getExecutionTypeRequest().getExecutionType())); + req.getExecutionTypeRequest().getExecutionType(), + req.getExecutionTypeRequest().getEnforceExecutionType())); } private static List<ResourceRequest> cloneResourceRequests( --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
