YARN-6022. Revert changes of AbstractResourceRequest (Contributed by Wangda Tan via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac16400e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac16400e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac16400e Branch: refs/heads/HADOOP-13345 Commit: ac16400e1fb85a4186e5bf5bbc9cf204735ae74f Parents: a59df15 Author: Daniel Templeton <templ...@apache.org> Authored: Sat Jan 7 08:23:06 2017 -0800 Committer: Daniel Templeton <templ...@apache.org> Committed: Sat Jan 7 08:24:30 2017 -0800 ---------------------------------------------------------------------- .../api/records/AbstractResourceRequest.java | 55 -------------------- .../yarn/api/records/ResourceRequest.java | 19 ++++++- .../api/records/UpdateContainerRequest.java | 18 ++++++- .../server/scheduler/SchedulerRequestKey.java | 1 - .../server/resourcemanager/RMAppManager.java | 2 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 7 ++- .../scheduler/SchedulerUtils.java | 18 ++++--- .../scheduler/YarnScheduler.java | 6 +-- .../scheduler/fair/FairScheduler.java | 5 +- 10 files changed, 55 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java deleted file mode 100644 index 819a607..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -/** - * {@code AbstractResourceRequest} represents a generic resource request made - * by an application to the {@code ResourceManager}. - * <p> - * It includes: - * <ul> - * <li>{@link Resource} capability required for each request.</li> - * </ul> - * - * @see Resource - */ -@Public -@Unstable -public abstract class AbstractResourceRequest { - - /** - * Set the <code>Resource</code> capability of the request - * @param capability <code>Resource</code> capability of the request - */ - @Public - @Stable - public abstract void setCapability(Resource capability); - - /** - * Get the <code>Resource</code> capability of the request. - * @return <code>Resource</code> capability of the request - */ - @Public - @Stable - public abstract Resource getCapability(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index ce85b21..5bedc87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -58,8 +58,7 @@ import org.apache.hadoop.yarn.util.Records; */ @Public @Stable -public abstract class ResourceRequest extends AbstractResourceRequest - implements Comparable<ResourceRequest>{ +public abstract class ResourceRequest implements Comparable<ResourceRequest> { @Public @Stable @@ -506,6 +505,22 @@ public abstract class ResourceRequest extends AbstractResourceRequest public void setAllocationRequestId(long allocationRequestID) { throw new UnsupportedOperationException(); } + + /** + * Set the <code>Resource</code> capability of the request. + * @param capability <code>Resource</code> capability of the request + */ + @Public + @Stable + public abstract void setCapability(Resource capability); + + /** + * Get the <code>Resource</code> capability of the request. + * @return <code>Resource</code> capability of the request + */ + @Public + @Stable + public abstract Resource getCapability(); @Override public int hashCode() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java index 925a7979..2060370 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java @@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.util.Records; */ @InterfaceAudience.Public @InterfaceStability.Unstable -public abstract class UpdateContainerRequest extends AbstractResourceRequest { +public abstract class UpdateContainerRequest { @InterfaceAudience.Public @InterfaceStability.Unstable @@ -143,6 +143,22 @@ public abstract class UpdateContainerRequest extends AbstractResourceRequest { @InterfaceStability.Unstable public abstract void setExecutionType(ExecutionType executionType); + /** + * Set the <code>Resource</code> capability of the request. + * @param capability <code>Resource</code> capability of the request + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract void setCapability(Resource capability); + + /** + * Get the <code>Resource</code> capability of the request. + * @return <code>Resource</code> capability of the request + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract Resource getCapability(); + @Override public int hashCode() { final int prime = 2153; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java index 36a9149..02539ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.scheduler; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 9dd544e..8232b88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -459,7 +459,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, throw e; } - scheduler.normalizeRequest(amReq); + amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability())); return amReq; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index ebbeb0f..94bfd58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -337,7 +337,7 @@ public class RMServerUtils { return false; } ResourceScheduler scheduler = rmContext.getScheduler(); - scheduler.normalizeRequest(request); + request.setCapability(scheduler.getNormalizedResource(request.getCapability())); return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index acfcde8..89f9ffa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -36,7 +36,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -1044,8 +1043,8 @@ public abstract class AbstractYarnScheduler } @Override - public void normalizeRequest(AbstractResourceRequest ask) { - SchedulerUtils.normalizeRequest(ask, + public Resource getNormalizedResource(Resource requestedResource) { + return SchedulerUtils.getNormalizedResource(requestedResource, getResourceCalculator(), getMinimumResourceCapability(), getMaximumResourceCapability(), @@ -1059,7 +1058,7 @@ public abstract class AbstractYarnScheduler */ protected void normalizeRequests(List<ResourceRequest> asks) { for (ResourceRequest ask: asks) { - normalizeRequest(ask); + ask.setCapability(getNormalizedResource(ask.getCapability())); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 5360665..148f023 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.io.IOException; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; @@ -134,29 +134,33 @@ public class SchedulerUtils { * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of minMemory and is not zero. */ + @VisibleForTesting public static void normalizeRequest( ResourceRequest ask, ResourceCalculator resourceCalculator, Resource minimumResource, Resource maximumResource) { - normalizeRequest(ask, resourceCalculator, - minimumResource, maximumResource, minimumResource); + ask.setCapability( + getNormalizedResource(ask.getCapability(), resourceCalculator, + minimumResource, maximumResource, minimumResource)); } /** * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of increment resource and is not zero. + * + * @return normalized resource */ - public static void normalizeRequest( - AbstractResourceRequest ask, + public static Resource getNormalizedResource( + Resource ask, ResourceCalculator resourceCalculator, Resource minimumResource, Resource maximumResource, Resource incrementResource) { Resource normalized = Resources.normalize( - resourceCalculator, ask.getCapability(), minimumResource, + resourceCalculator, ask, minimumResource, maximumResource, incrementResource); - ask.setCapability(normalized); + return normalized; } private static void normalizeNodeLabelExpressionInRequest( http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 3185dc1..5579efd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -381,7 +380,8 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> { /** * Normalize a resource request. * - * @param request the resource request to be normalized + * @param requestedResource the resource to be normalized + * @return the normalized resource */ - void normalizeRequest(AbstractResourceRequest request); + Resource getNormalizedResource(Resource requestedResource); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 78dfe61..6bbccc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.yarn.api.records.AbstractResourceRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -799,8 +798,8 @@ public class FairScheduler extends } @Override - public void normalizeRequest(AbstractResourceRequest ask) { - SchedulerUtils.normalizeRequest(ask, + public Resource getNormalizedResource(Resource requestedResource) { + return SchedulerUtils.getNormalizedResource(requestedResource, DOMINANT_RESOURCE_CALCULATOR, minimumAllocation, getMaximumResourceCapability(), --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
