Repository: hadoop Updated Branches: refs/heads/branch-2 a7cdd3923 -> c00a468b0
YARN-6022. Revert changes of AbstractResourceRequest. (Contributed by Wangda Tan / Daniel Templeton) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c00a468b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c00a468b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c00a468b Branch: refs/heads/branch-2 Commit: c00a468b0baaf22e81accf0b40b3abd83955ba20 Parents: a7cdd39 Author: Wangda Tan <wan...@apache.org> Authored: Tue Jan 10 10:17:33 2017 -0800 Committer: Wangda Tan <wan...@apache.org> Committed: Tue Jan 10 10:17:33 2017 -0800 ---------------------------------------------------------------------- .../api/records/AbstractResourceRequest.java | 55 -------------------- .../yarn/api/records/ResourceRequest.java | 19 ++++++- .../api/records/UpdateContainerRequest.java | 18 ++++++- .../server/resourcemanager/RMAppManager.java | 2 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 7 ++- .../scheduler/SchedulerUtils.java | 18 ++++--- .../scheduler/YarnScheduler.java | 6 +-- .../scheduler/fair/FairScheduler.java | 5 +- 9 files changed, 55 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java deleted file mode 100644 index 819a607..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -/** - * {@code AbstractResourceRequest} represents a generic resource request made - * by an application to the {@code ResourceManager}. - * <p> - * It includes: - * <ul> - * <li>{@link Resource} capability required for each request.</li> - * </ul> - * - * @see Resource - */ -@Public -@Unstable -public abstract class AbstractResourceRequest { - - /** - * Set the <code>Resource</code> capability of the request - * @param capability <code>Resource</code> capability of the request - */ - @Public - @Stable - public abstract void setCapability(Resource capability); - - /** - * Get the <code>Resource</code> capability of the request. - * @return <code>Resource</code> capability of the request - */ - @Public - @Stable - public abstract Resource getCapability(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index ce85b21..5bedc87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -58,8 +58,7 @@ import org.apache.hadoop.yarn.util.Records; */ @Public @Stable -public abstract class ResourceRequest extends AbstractResourceRequest - implements Comparable<ResourceRequest>{ +public abstract class ResourceRequest implements Comparable<ResourceRequest> { @Public @Stable @@ -506,6 +505,22 @@ public abstract class ResourceRequest extends AbstractResourceRequest public void setAllocationRequestId(long allocationRequestID) { throw new UnsupportedOperationException(); } + + /** + * Set the <code>Resource</code> capability of the request. + * @param capability <code>Resource</code> capability of the request + */ + @Public + @Stable + public abstract void setCapability(Resource capability); + + /** + * Get the <code>Resource</code> capability of the request. + * @return <code>Resource</code> capability of the request + */ + @Public + @Stable + public abstract Resource getCapability(); @Override public int hashCode() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java index e4f7a82..2a8bb79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java @@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.util.Records; */ @InterfaceAudience.Public @InterfaceStability.Unstable -public abstract class UpdateContainerRequest extends AbstractResourceRequest { +public abstract class UpdateContainerRequest { @InterfaceAudience.Public @InterfaceStability.Unstable @@ -143,6 +143,22 @@ public abstract class UpdateContainerRequest extends AbstractResourceRequest { @InterfaceStability.Unstable public abstract void setExecutionType(ExecutionType executionType); + /** + * Set the <code>Resource</code> capability of the request. + * @param capability <code>Resource</code> capability of the request + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract void setCapability(Resource capability); + + /** + * Get the <code>Resource</code> capability of the request. + * @return <code>Resource</code> capability of the request + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract Resource getCapability(); + @Override public int hashCode() { final int prime = 2153; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index f0f923b..56d763b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -451,7 +451,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, throw e; } - scheduler.normalizeRequest(amReq); + amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability())); return amReq; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 74898ca..ff4c762 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -303,7 +303,7 @@ public class RMServerUtils { return false; } ResourceScheduler scheduler = rmContext.getScheduler(); - scheduler.normalizeRequest(request); + request.setCapability(scheduler.getNormalizedResource(request.getCapability())); return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 145b9b4..dc376a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -37,7 +37,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -1043,8 +1042,8 @@ public abstract class AbstractYarnScheduler } @Override - public void normalizeRequest(AbstractResourceRequest ask) { - SchedulerUtils.normalizeRequest(ask, + public Resource getNormalizedResource(Resource requestedResource) { + return SchedulerUtils.getNormalizedResource(requestedResource, getResourceCalculator(), getMinimumResourceCapability(), getMaximumResourceCapability(), @@ -1058,7 +1057,7 @@ public abstract class AbstractYarnScheduler */ protected void normalizeRequests(List<ResourceRequest> asks) { for (ResourceRequest ask: asks) { - normalizeRequest(ask); + ask.setCapability(getNormalizedResource(ask.getCapability())); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index b227523..b1e789e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.io.IOException; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -32,7 +33,6 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; @@ -127,29 +127,33 @@ public class SchedulerUtils { * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of minMemory and is not zero. */ + @VisibleForTesting public static void normalizeRequest( ResourceRequest ask, ResourceCalculator resourceCalculator, Resource minimumResource, Resource maximumResource) { - normalizeRequest(ask, resourceCalculator, - minimumResource, maximumResource, minimumResource); + ask.setCapability( + getNormalizedResource(ask.getCapability(), resourceCalculator, + minimumResource, maximumResource, minimumResource)); } /** * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of increment resource and is not zero. + * + * @return normalized resource */ - public static void normalizeRequest( - AbstractResourceRequest ask, + public static Resource getNormalizedResource( + Resource ask, ResourceCalculator resourceCalculator, Resource minimumResource, Resource maximumResource, Resource incrementResource) { Resource normalized = Resources.normalize( - resourceCalculator, ask.getCapability(), minimumResource, + resourceCalculator, ask, minimumResource, maximumResource, incrementResource); - ask.setCapability(normalized); + return normalized; } private static void normalizeNodeLabelExpressionInRequest( http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 608f275..c9ef7dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -385,7 +384,8 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> { /** * Normalize a resource request. * - * @param request the resource request to be normalized + * @param requestedResource the resource to be normalized + * @return the normalized resource */ - void normalizeRequest(AbstractResourceRequest request); + Resource getNormalizedResource(Resource requestedResource); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c00a468b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 4018681..c1e673e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -26,7 +26,6 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -986,8 +985,8 @@ public class FairScheduler extends } @Override - public void normalizeRequest(AbstractResourceRequest ask) { - SchedulerUtils.normalizeRequest(ask, + public Resource getNormalizedResource(Resource requestedResource) { + return SchedulerUtils.getNormalizedResource(requestedResource, DOMINANT_RESOURCE_CALCULATOR, minimumAllocation, getMaximumResourceCapability(), --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
