Repository: hadoop Updated Branches: refs/heads/YARN-3926 f461a2d42 -> 7d9f92cf0
YARN-7043. Cleanup ResourceProfileManager. (wangda) Change-Id: I463356f37bf1f6a3f1fc3c594c79916e8c0ab913 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d9f92cf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d9f92cf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d9f92cf Branch: refs/heads/YARN-3926 Commit: 7d9f92cf0f35f542256480a8adc14fe89b10ec99 Parents: f461a2d Author: Wangda Tan <wan...@apache.org> Authored: Mon Aug 21 17:20:06 2017 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Mon Aug 21 17:20:06 2017 -0700 ---------------------------------------------------------------------- .../ams/ApplicationMasterServiceProcessor.java | 8 ++- .../yarn/api/ApplicationClientProtocol.java | 11 ++-- .../YARNFeatureNotEnabledException.java | 45 +++++++++++++++ .../yarn/util/resource/ResourceUtils.java | 21 +++++++ .../hadoop/yarn/client/api/YarnClient.java | 13 +++-- .../resource/DominantResourceCalculator.java | 6 +- .../hadoop/yarn/api/TestPBImplRecords.java | 1 + .../resourcemanager/AMSProcessingChain.java | 2 +- .../server/resourcemanager/ClientRMService.java | 26 ++------- .../resourcemanager/DefaultAMSProcessor.java | 12 ++-- ...pportunisticContainerAllocatorAMService.java | 3 +- .../server/resourcemanager/RMServerUtils.java | 14 ----- .../resource/ResourceProfilesManager.java | 32 +++++----- .../resource/ResourceProfilesManagerImpl.java | 61 +++++++++++++------- .../scheduler/AbstractYarnScheduler.java | 18 +++++- .../TestApplicationMasterService.java | 11 ++-- .../resource/TestResourceProfiles.java | 10 +++- 17 files changed, 191 insertions(+), 103 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java index b7d925a..8e76a11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java @@ -52,11 +52,13 @@ public interface ApplicationMasterServiceProcessor { * @param request Register Request. * @param response Register Response. * @throws IOException IOException. + * @throws YarnException in critical situation where invalid + * profiles/resources are added. */ - void registerApplicationMaster( - ApplicationAttemptId applicationAttemptId, + void registerApplicationMaster(ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException; + RegisterApplicationMasterResponse response) + throws IOException, YarnException; /** * Allocate call. http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index e745697..8456a8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -81,6 +81,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; /** * <p>The protocol between clients and the <code>ResourceManager</code> @@ -604,7 +605,8 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol { * @param request request to get all the resource profiles * @return Response containing a map of the profile name to Resource * capabilities - * @throws YarnException if resource profiles are not enabled on the RM + * @throws YARNFeatureNotEnabledException if resource-profile is disabled + * @throws YarnException if any error happens inside YARN * @throws IOException in case of other errors */ @Public @@ -618,8 +620,8 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol { * </p> * @param request request to get the details of a resource profile * @return Response containing the details for a particular resource profile - * @throws YarnException if resource profiles are not enabled on the RM or - * the profile cannot be found + * @throws YARNFeatureNotEnabledException if resource-profile is disabled + * @throws YarnException if any error happens inside YARN * @throws IOException in case of other errors */ @Public @@ -633,8 +635,7 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol { * </p> * @param request request to get the details of a resource profile * @return Response containing the details for a particular resource profile - * @throws YarnException if resource profiles are not enabled on the RM or - * the profile cannot be found + * @throws YarnException if any error happens inside YARN * @throws IOException in case of other errors */ @Public http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java new file mode 100644 index 0000000..62340fe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.exceptions; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This exception is thrown when a feature is being used which is not enabled + * yet. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class YARNFeatureNotEnabledException extends YarnException { + private static final long serialVersionUID = 898023752676L; + + public YARNFeatureNotEnabledException(Throwable cause) { + super(cause); + } + + public YARNFeatureNotEnabledException(String message) { + super(message); + } + + public YARNFeatureNotEnabledException(String message, Throwable cause) { + super(message, cause); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 997c2c0..b81c327 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -37,9 +38,12 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -561,4 +565,21 @@ public class ResourceUtils { } return ""; } + + /** + * Get all resource types information from known resource types. + * @return List of ResourceTypeInfo + */ + public static List<ResourceTypeInfo> getResourcesTypeInfo() { + List<ResourceTypeInfo> array = new ArrayList<>(); + // Add all resource types + Collection<ResourceInformation> resourcesInfo = + ResourceUtils.getResourceTypes().values(); + for (ResourceInformation resourceInfo : resourcesInfo) { + array.add(ResourceTypeInfo + .newInstance(resourceInfo.getName(), resourceInfo.getUnits(), + resourceInfo.getResourceType())); + } + return array; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 4f5b5bc..60e7813 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; +import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; @@ -863,7 +864,8 @@ public abstract class YarnClient extends AbstractService { * Get the resource profiles available in the RM. * </p> * @return a Map of the resource profile names to their capabilities - * @throws YarnException if resource profiles are not enabled + * @throws YARNFeatureNotEnabledException if resource-profile is disabled + * @throws YarnException if any error happens inside YARN * @throws IOException in case of other errors */ @Public @@ -876,9 +878,9 @@ public abstract class YarnClient extends AbstractService { * Get the details of a specific resource profile from the RM. * </p> * @param profile the profile name - * @return the capabilities of the resource profile - * @throws YarnException if resource profiles are not enabled or the profile - * cannot be found + * @return resource profile name with its capabilities + * @throws YARNFeatureNotEnabledException if resource-profile is disabled + * @throws YarnException if any error happens inside YARN * @throws IOException in case of other others */ @Public @@ -891,8 +893,7 @@ public abstract class YarnClient extends AbstractService { * Get available resource types supported by RM. * </p> * @return list of supported resource types with detailed information - * @throws YarnException if resource profiles are not enabled or the profile - * cannot be found + * @throws YarnException if any issue happens inside YARN * @throws IOException in case of other others */ @Public http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 1e99bc7..ffd4fec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -528,10 +528,12 @@ public class DominantResourceCalculator extends ResourceCalculator { long value; if (stepFactorValue != 0) { value = roundUp - ? roundUp((long) Math.ceil(rValue * by), stepFactorValue) + ? roundUp((long) Math.ceil((float) (rValue * by)), stepFactorValue) : roundDown((long) (rValue * by), stepFactorValue); } else { - value = roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by); + value = roundUp + ? (long) Math.ceil((float) (rValue * by)) + : (long) (rValue * by); } tmp.setValue(value); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index f31da11..164880b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -429,6 +429,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { generateByNewInstance(CommitResponse.class); generateByNewInstance(ApplicationTimeout.class); generateByNewInstance(QueueConfigurations.class); + generateByNewInstance(ResourceTypeInfo.class); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java index 931b1c8..7ae23e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java @@ -82,7 +82,7 @@ class AMSProcessingChain implements ApplicationMasterServiceProcessor { public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse resp) throws IOException { + RegisterApplicationMasterResponse resp) throws IOException, YarnException { this.head.registerApplicationMaster(applicationAttemptId, request, resp); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index f657415..46dd216 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -142,7 +142,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; -import org.apache.hadoop.yarn.exceptions.ResourceProfilesNotEnabledException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -182,6 +181,7 @@ import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UTCClock; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @@ -1775,42 +1775,26 @@ public class ClientRMService extends AbstractService implements GetAllResourceProfilesRequest request) throws YarnException, IOException { GetAllResourceProfilesResponse response = GetAllResourceProfilesResponse.newInstance(); - response.setResourceProfiles(getResourceProfiles()); + response.setResourceProfiles(resourceProfilesManager.getResourceProfiles()); return response; } @Override public GetResourceProfileResponse getResourceProfile( GetResourceProfileRequest request) throws YarnException, IOException { - Map<String, Resource> profiles = getResourceProfiles(); - if (!profiles.containsKey(request.getProfileName())) { - throw new YarnException( - "Resource profile '" + request.getProfileName() + "' not found"); - } GetResourceProfileResponse response = GetResourceProfileResponse.newInstance(); - response.setResource(profiles.get(request.getProfileName())); + response.setResource( + resourceProfilesManager.getProfile(request.getProfileName())); return response; } - private Map<String, Resource> getResourceProfiles() throws YarnException { - boolean resourceProfilesEnabled = getConfig() - .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, - YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED); - if (!resourceProfilesEnabled) { - throw new ResourceProfilesNotEnabledException( - "Resource profiles are not enabled"); - } - return resourceProfilesManager.getResourceProfiles(); - } - @Override public GetAllResourceTypeInfoResponse getResourceTypeInfo( GetAllResourceTypeInfoRequest request) throws YarnException, IOException { GetAllResourceTypeInfoResponse response = GetAllResourceTypeInfoResponse.newInstance(); - response.setResourceTypeInfo( - resourceProfilesManager.getAllResourceTypeInfo()); + response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo()); return response; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 3f12f31..3a5ab0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -113,7 +113,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + RegisterApplicationMasterResponse response) + throws IOException, YarnException { RMApp app = getRmContext().getRMApps().get( applicationAttemptId.getApplicationId()); @@ -173,10 +174,11 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { response.setSchedulerResourceTypes(getScheduler() .getSchedulingResourceTypes()); - if (getRmContext().getYarnConfiguration().getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, - YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) { - response - .setResourceProfiles(resourceProfilesManager.getResourceProfiles()); + if (getRmContext().getYarnConfiguration().getBoolean( + YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, + YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) { + response.setResourceProfiles( + resourceProfilesManager.getResourceProfiles()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java index 3c278de..daf3160 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java @@ -127,7 +127,8 @@ public class OpportunisticContainerAllocatorAMService public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + RegisterApplicationMasterResponse response) + throws IOException, YarnException { SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) getScheduler()).getApplicationAttempt(applicationAttemptId); if (appAttempt.getOpportunisticContainerContext() == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index d3e8345..2aae3a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -628,20 +628,6 @@ public class RMServerUtils { } } - public static void convertProfileToResourceCapability( - List<ResourceRequest> asks, Configuration conf, - ResourceProfilesManager resourceProfilesManager) throws YarnException { - boolean profilesEnabled = - conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, - YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED); - if (!profilesEnabled) { - return; - } - for (ResourceRequest req : asks) { - convertProfileToResourceCapability(req, conf, resourceProfilesManager); - } - } - public static void convertProfileToResourceCapability(ResourceRequest ask, Configuration conf, ResourceProfilesManager resourceProfilesManager) throws YarnException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java index 8963121..700f0ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java @@ -18,22 +18,18 @@ package org.apache.hadoop.yarn.server.resourcemanager.resource; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; +import org.apache.hadoop.yarn.exceptions.YarnException; import java.io.IOException; -import java.util.List; import java.util.Map; /** * Interface for the resource profiles manager. Provides an interface to get * the list of available profiles and some helper functions. */ -@Public -@Unstable public interface ResourceProfilesManager { /** @@ -47,14 +43,19 @@ public interface ResourceProfilesManager { * Get the resource capability associated with given profile name. * @param profile name of resource profile * @return resource capability for given profile + * + * @throws YarnException when any invalid profile name or feature is disabled */ - Resource getProfile(String profile); + Resource getProfile(String profile) throws YarnException; /** * Get all supported resource profiles. * @return a map of resource objects associated with each profile + * + * @throws YARNFeatureNotEnabledException when feature is disabled */ - Map<String, Resource> getResourceProfiles(); + Map<String, Resource> getResourceProfiles() throws + YARNFeatureNotEnabledException; /** * Reload profiles based on updated configuration. @@ -65,24 +66,21 @@ public interface ResourceProfilesManager { /** * Get default supported resource profile. * @return resource object which is default + * @throws YarnException when any invalid profile name or feature is disabled */ - Resource getDefaultProfile(); + Resource getDefaultProfile() throws YarnException; /** * Get minimum supported resource profile. * @return resource object which is minimum + * @throws YarnException when any invalid profile name or feature is disabled */ - Resource getMinimumProfile(); + Resource getMinimumProfile() throws YarnException; /** * Get maximum supported resource profile. * @return resource object which is maximum + * @throws YarnException when any invalid profile name or feature is disabled */ - Resource getMaximumProfile(); - - /** - * List of ResourceTypeInfo objects which carry all resources supported by RM. - * @return list of ResourceTypeInfo objects - */ - List<ResourceTypeInfo> getAllResourceTypeInfo(); + Resource getMaximumProfile() throws YarnException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index ff4e3e4..e213a97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.codehaus.jackson.map.ObjectMapper; @@ -52,6 +54,7 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { private List<ResourceTypeInfo> resourceTypeInfo = new ArrayList<ResourceTypeInfo>(); private Configuration conf; + private boolean profileEnabled = false; private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); private static final String VCORES = ResourceInformation.VCORES.getName(); @@ -65,6 +68,11 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { private static final String[] MANDATORY_PROFILES = { DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE }; + private static final String FEATURE_NOT_ENABLED_MSG = + "Resource profile is not enabled, please " + + "enable resource profile feature before using its functions." + + " (by setting " + YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED + + " to true)"; public ResourceProfilesManagerImpl() { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); @@ -98,10 +106,10 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { } private void loadProfiles() throws IOException { - boolean profilesEnabled = + profileEnabled = conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED); - if (!profilesEnabled) { + if (!profileEnabled) { return; } String sourceFile = @@ -131,14 +139,16 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { if (entry.getValue() instanceof Map) { Map profileInfo = (Map) entry.getValue(); // ensure memory and vcores are specified - if (!profileInfo.containsKey(MEMORY) || !profileInfo.containsKey(VCORES)) { + if (!profileInfo.containsKey(MEMORY) + || !profileInfo.containsKey(VCORES)) { throw new IOException( "Illegal resource profile definition; profile '" + profileName + "' must contain '" + MEMORY + "' and '" + VCORES + "'"); } Resource resource = parseResource(profileInfo); profiles.put(profileName, resource); - LOG.info("Added profile '" + profileName + "' with resources " + resource); + LOG.info( + "Added profile '" + profileName + "' with resources: " + resource); } } // check to make sure mandatory profiles are present @@ -149,7 +159,7 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { + Arrays.toString(MANDATORY_PROFILES) + " must be present"); } } - LOG.info("Loaded profiles " + profiles.keySet()); + LOG.info("Loaded profiles: " + profiles.keySet()); } private Resource parseResource(Map profileInfo) throws IOException { @@ -182,13 +192,33 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { return resource; } + private void checkAndThrowExceptionWhenFeatureDisabled() + throws YARNFeatureNotEnabledException { + if (!profileEnabled) { + throw new YARNFeatureNotEnabledException(FEATURE_NOT_ENABLED_MSG); + } + } + @Override - public Resource getProfile(String profile) { - return Resources.clone(profiles.get(profile)); + public Resource getProfile(String profile) throws YarnException{ + checkAndThrowExceptionWhenFeatureDisabled(); + + if (profile == null) { + throw new YarnException("Profile name cannot be null"); + } + + Resource profileRes = profiles.get(profile); + if (profileRes == null) { + throw new YarnException( + "Resource profile '" + profile + "' not found"); + } + return Resources.clone(profileRes); } @Override - public Map<String, Resource> getResourceProfiles() { + public Map<String, Resource> getResourceProfiles() + throws YARNFeatureNotEnabledException { + checkAndThrowExceptionWhenFeatureDisabled(); return Collections.unmodifiableMap(profiles); } @@ -200,17 +230,17 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { } @Override - public Resource getDefaultProfile() { + public Resource getDefaultProfile() throws YarnException { return getProfile(DEFAULT_PROFILE); } @Override - public Resource getMinimumProfile() { + public Resource getMinimumProfile() throws YarnException { return getProfile(MINIMUM_PROFILE); } @Override - public Resource getMaximumProfile() { + public Resource getMaximumProfile() throws YarnException { return getProfile(MAXIMUM_PROFILE); } @@ -220,13 +250,4 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { Long.valueOf(value.substring(0, value.length() - units.length())); return ResourceInformation.newInstance(name, units, resourceValue); } - - public List<ResourceTypeInfo> getAllResourceTypeInfo() { - try { - readLock.lock(); - return Collections.unmodifiableList(resourceTypeInfo); - } finally { - readLock.unlock(); - } - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index b5a1df9..6ecfa78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; @@ -1290,7 +1291,13 @@ public abstract class AbstractYarnScheduler if (!profilesEnabled) { ret = ResourceUtils.getResourceTypesMinimumAllocation(); } else { - ret = rmContext.getResourceProfilesManager().getMinimumProfile(); + try { + ret = rmContext.getResourceProfilesManager().getMinimumProfile(); + } catch (YarnException e) { + LOG.error( + "Exception while getting minimum profile from profile manager:", e); + throw new YarnRuntimeException(e); + } } LOG.info("Minimum allocation = " + ret); return ret; @@ -1312,7 +1319,14 @@ public abstract class AbstractYarnScheduler if (!profilesEnabled) { ret = ResourceUtils.getResourceTypesMaximumAllocation(); } else { - ret = rmContext.getResourceProfilesManager().getMaximumProfile(); + try { + ret = rmContext.getResourceProfilesManager().getMaximumProfile(); + } catch (YarnException e) { + LOG.error( + "Exception while getting maximum profile from ResourceProfileManager:", + e); + throw new YarnRuntimeException(e); + } } LOG.info("Maximum allocation = " + ret); return ret; http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index 4af1e32..2815564 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -103,9 +103,11 @@ public class TestApplicationMasterService { } @Override - public void registerApplicationMaster(ApplicationAttemptId - applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + public void registerApplicationMaster( + ApplicationAttemptId applicationAttemptId, + RegisterApplicationMasterRequest request, + RegisterApplicationMasterResponse response) + throws IOException, YarnException { nextProcessor.registerApplicationMaster( applicationAttemptId, request, response); } @@ -145,7 +147,8 @@ public class TestApplicationMasterService { public void registerApplicationMaster( ApplicationAttemptId applicationAttemptId, RegisterApplicationMasterRequest request, - RegisterApplicationMasterResponse response) throws IOException { + RegisterApplicationMasterResponse response) + throws IOException, YarnException { beforeRegCount.incrementAndGet(); nextProcessor.registerApplicationMaster(applicationAttemptId, request, response); http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d9f92cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java index c542ed8..b53e59c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.resource; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.junit.Assert; import org.junit.Test; @@ -36,8 +37,13 @@ public class TestResourceProfiles { Configuration conf = new Configuration(); // be default resource profiles should not be enabled manager.init(conf); - Map<String, Resource> profiles = manager.getResourceProfiles(); - Assert.assertTrue(profiles.isEmpty()); + try { + manager.getResourceProfiles(); + Assert + .fail("Exception should be thrown as resource profile is not enabled" + + " and getResourceProfiles is invoked."); + } catch (YarnException ie) { + } conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true); try { manager.init(conf); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org