Revert "Merge branch 'trunk' of https://git-wip-us.apache.org/repos/asf/hadoop into trunk"
This reverts commit c163d1797ade0f47d35b4a44381b8ef1dfec5b60, reversing changes made to 0d9804dcef2eab5ebf84667d9ca49bb035d9a731. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39ad9890 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39ad9890 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39ad9890 Branch: refs/heads/HDDS-48 Commit: 39ad98903a5f042573b97a2e5438bc57af7cc7a1 Parents: c163d17 Author: Anu Engineer <aengin...@apache.org> Authored: Thu Jul 5 12:22:18 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Thu Jul 5 12:22:18 2018 -0700 ---------------------------------------------------------------------- .../dev-support/findbugs-exclude.xml | 17 +- .../hadoop/yarn/api/records/Resource.java | 13 - .../api/records/impl/LightWeightResource.java | 23 +- .../scheduler/fair/ConfigurableResource.java | 69 +---- .../fair/FairSchedulerConfiguration.java | 174 ++---------- .../allocation/AllocationFileQueueParser.java | 2 +- .../resourcemanager/webapp/dao/AppInfo.java | 2 +- .../webapp/dao/SchedulerInfo.java | 8 +- .../fair/TestFairSchedulerConfiguration.java | 160 +++-------- .../webapp/TestRMWebServices.java | 31 +-- .../webapp/TestRMWebServicesApps.java | 14 +- ...estRMWebServicesAppsCustomResourceTypes.java | 242 ----------------- .../webapp/TestRMWebServicesCapacitySched.java | 30 +- .../TestRMWebServicesConfigurationMutation.java | 5 - .../webapp/TestRMWebServicesFairScheduler.java | 95 ++++--- .../TestRMWebServicesSchedulerActivities.java | 2 +- ...ustomResourceTypesConfigurationProvider.java | 138 ---------- .../FairSchedulerJsonVerifications.java | 139 ---------- .../FairSchedulerXmlVerifications.java | 153 ----------- ...ervicesFairSchedulerCustomResourceTypes.java | 271 ------------------- .../webapp/helper/AppInfoJsonVerifications.java | 123 --------- .../webapp/helper/AppInfoXmlVerifications.java | 132 --------- .../webapp/helper/BufferedClientResponse.java | 57 ---- .../helper/JsonCustomResourceTypeTestcase.java | 77 ------ .../ResourceRequestsJsonVerifications.java | 252 ----------------- .../ResourceRequestsXmlVerifications.java | 215 --------------- .../helper/XmlCustomResourceTypeTestCase.java | 112 -------- .../src/site/markdown/FairScheduler.md | 6 +- 28 files changed, 157 insertions(+), 2405 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 5cc81e5..5841361 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -67,6 +67,11 @@ </Match> <Match> <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" /> + <Method name="getLocalityStatistics" /> + <Bug pattern="EI_EXPOSE_REP" /> + </Match> + <Match> + <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" /> <Method name="incNumAllocatedContainers"/> <Bug pattern="VO_VOLATILE_INCREMENT" /> </Match> @@ -113,18 +118,6 @@ <Bug pattern="BC_UNCONFIRMED_CAST" /> </Match> - <!-- Ignore exposed internal representations --> - <Match> - <Class name="org.apache.hadoop.yarn.api.records.Resource" /> - <Method name="getResources" /> - <Bug pattern="EI_EXPOSE_REP" /> - </Match> - <Match> - <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" /> - <Method name="getLocalityStatistics" /> - <Bug pattern="EI_EXPOSE_REP" /> - </Match> - <!-- Object cast is based on the event type --> <Match> <Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher" /> http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 173d4c9..71a6b54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.impl.LightWeightResource; @@ -76,18 +75,6 @@ public abstract class Resource implements Comparable<Resource> { @Private public static final int VCORES_INDEX = 1; - /** - * Return a new {@link Resource} instance with all resource values - * initialized to {@code value}. - * @param value the value to use for all resources - * @return a new {@link Resource} instance - */ - @Private - @Unstable - public static Resource newInstance(long value) { - return new LightWeightResource(value); - } - @Public @Stable public static Resource newInstance(int memory, int vCores) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java index 77f77f3..a6e6432 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java @@ -18,8 +18,9 @@ package org.apache.hadoop.yarn.api.records.impl; -import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.util.resource.ResourceUtils; @@ -57,29 +58,13 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.*; * * @see Resource */ -@Private +@InterfaceAudience.Private @Unstable public class LightWeightResource extends Resource { private ResourceInformation memoryResInfo; private ResourceInformation vcoresResInfo; - /** - * Create a new {@link LightWeightResource} instance with all resource values - * initialized to {@code value}. - * @param value the value to use for all resources - */ - public LightWeightResource(long value) { - ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); - initResourceInformations(value, value, types.length); - - for (int i = 2; i < types.length; i++) { - resources[i] = new ResourceInformation(); - ResourceInformation.copy(types[i], resources[i]); - resources[i].setValue(value); - } - } - public LightWeightResource(long memory, int vcores) { int numberOfKnownResourceTypes = ResourceUtils .getNumberOfKnownResourceTypes(); @@ -106,7 +91,7 @@ public class LightWeightResource extends Resource { } } - private void initResourceInformations(long memory, long vcores, + private void initResourceInformations(long memory, int vcores, int numberOfKnownResourceTypes) { this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(), memory); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java index 0c3b0dd..ecdd011 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -18,13 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import java.util.Arrays; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** * A {@code ConfigurableResource} object represents an entity that is used to @@ -37,53 +33,29 @@ public class ConfigurableResource { private final Resource resource; private final double[] percentages; - ConfigurableResource() { - this(getOneHundredPercentArray()); - } - - ConfigurableResource(double[] percentages) { + public ConfigurableResource(double[] percentages) { this.percentages = percentages.clone(); this.resource = null; } - ConfigurableResource(long value) { - this(Resource.newInstance(value)); - } - public ConfigurableResource(Resource resource) { this.percentages = null; this.resource = resource; } - private static double[] getOneHundredPercentArray() { - double[] resourcePercentages = - new double[ResourceUtils.getNumberOfKnownResourceTypes()]; - Arrays.fill(resourcePercentages, 1.0); - - return resourcePercentages; - } - /** * Get resource by multiplying the cluster resource and the percentage of * each resource respectively. Return the absolute resource if either * {@code percentages} or {@code clusterResource} is null. * * @param clusterResource the cluster resource - * @return resource the resulting resource + * @return resource */ public Resource getResource(Resource clusterResource) { if (percentages != null && clusterResource != null) { long memory = (long) (clusterResource.getMemorySize() * percentages[0]); int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]); - Resource res = Resource.newInstance(memory, vcore); - ResourceInformation[] clusterInfo = clusterResource.getResources(); - - for (int i = 2; i < clusterInfo.length; i++) { - res.setResourceValue(i, - (long)(clusterInfo[i].getValue() * percentages[i])); - } - - return res; + return Resource.newInstance(memory, vcore); } else { return resource; } @@ -97,39 +69,4 @@ public class ConfigurableResource { public Resource getResource() { return resource; } - - /** - * Set the value of the wrapped resource if this object isn't setup to use - * percentages. If this object is set to use percentages, this method has - * no effect. - * - * @param name the name of the resource - * @param value the value - */ - void setValue(String name, long value) { - if (resource != null) { - resource.setResourceValue(name, value); - } - } - - /** - * Set the percentage of the resource if this object is setup to use - * percentages. If this object is set to use percentages, this method has - * no effect. - * - * @param name the name of the resource - * @param value the percentage - */ - void setPercentage(String name, double value) { - if (percentages != null) { - Integer index = ResourceUtils.getResourceTypeIndex().get(name); - - if (index != null) { - percentages[index] = value; - } else { - throw new ResourceNotFoundException("The requested resource, \"" - + name + "\", could not be found."); - } - } - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 8c4932b..b50e4bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -33,7 +33,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceUtils; @@ -214,9 +213,6 @@ public class FairSchedulerConfiguration extends Configuration { CONF_PREFIX + "reservable-nodes"; public static final float RESERVABLE_NODES_DEFAULT = 0.05f; - private static final String INVALID_RESOURCE_DEFINITION_PREFIX = - "Error reading resource config--invalid resource definition: "; - public FairSchedulerConfiguration() { super(); } @@ -411,167 +407,54 @@ public class FairSchedulerConfiguration extends Configuration { } /** - * Parses a resource config value in one of three forms: - * <ol> - * <li>Percentage: "50%" or "40% memory, 60% cpu"</li> - * <li>New style resources: "vcores=10, memory-mb=1024" - * or "vcores=60%, memory-mb=40%"</li> - * <li>Old style resources: "1024 mb, 10 vcores"</li> - * </ol> - * In new style resources, any resource that is not specified will be - * set to {@link Long#MAX_VALUE} or 100%, as appropriate. Also, in the new - * style resources, units are not allowed. Units are assumed from the resource - * manager's settings for the resources when the value isn't a percentage. - * - * @param value the resource definition to parse - * @return a {@link ConfigurableResource} that represents the parsed value - * @throws AllocationConfigurationException if the raw value is not a valid - * resource definition + * Parses a resource config value of a form like "1024", "1024 mb", + * or "1024 mb, 3 vcores". If no units are given, megabytes are assumed. + * + * @throws AllocationConfigurationException */ - public static ConfigurableResource parseResourceConfigValue(String value) + public static ConfigurableResource parseResourceConfigValue(String val) throws AllocationConfigurationException { - return parseResourceConfigValue(value, Long.MAX_VALUE); - } - - /** - * Parses a resource config value in one of three forms: - * <ol> - * <li>Percentage: "50%" or "40% memory, 60% cpu"</li> - * <li>New style resources: "vcores=10, memory-mb=1024" - * or "vcores=60%, memory-mb=40%"</li> - * <li>Old style resources: "1024 mb, 10 vcores"</li> - * </ol> - * In new style resources, any resource that is not specified will be - * set to {@code missing} or 0%, as appropriate. Also, in the new style - * resources, units are not allowed. Units are assumed from the resource - * manager's settings for the resources when the value isn't a percentage. - * - * The {@code missing} parameter is only used in the case of new style - * resources without percentages. With new style resources with percentages, - * any missing resources will be assumed to be 100% because percentages are - * only used with maximum resource limits. - * - * @param value the resource definition to parse - * @param missing the value to use for any unspecified resources - * @return a {@link ConfigurableResource} that represents the parsed value - * @throws AllocationConfigurationException if the raw value is not a valid - * resource definition - */ - public static ConfigurableResource parseResourceConfigValue(String value, - long missing) throws AllocationConfigurationException { ConfigurableResource configurableResource; - - if (value.trim().isEmpty()) { - throw new AllocationConfigurationException("Error reading resource " - + "config--the resource string is empty."); - } - try { - if (value.contains("=")) { - configurableResource = parseNewStyleResource(value, missing); - } else if (value.contains("%")) { - configurableResource = parseOldStyleResourceAsPercentage(value); + val = StringUtils.toLowerCase(val); + if (val.contains("%")) { + configurableResource = new ConfigurableResource( + getResourcePercentage(val)); } else { - configurableResource = parseOldStyleResource(value); + int memory = findResource(val, "mb"); + int vcores = findResource(val, "vcores"); + configurableResource = new ConfigurableResource( + BuilderUtils.newResource(memory, vcores)); } - } catch (RuntimeException ex) { + } catch (AllocationConfigurationException ex) { + throw ex; + } catch (Exception ex) { throw new AllocationConfigurationException( "Error reading resource config", ex); } - - return configurableResource; - } - - private static ConfigurableResource parseNewStyleResource(String value, - long missing) throws AllocationConfigurationException { - - final ConfigurableResource configurableResource; - boolean asPercent = value.contains("%"); - if (asPercent) { - configurableResource = new ConfigurableResource(); - } else { - configurableResource = new ConfigurableResource(missing); - } - - String[] resources = value.split(","); - for (String resource : resources) { - String[] parts = resource.split("="); - - if (parts.length != 2) { - throw createConfigException(value, - "Every resource must be of the form: name=value."); - } - - String resourceName = parts[0].trim(); - String resourceValue = parts[1].trim(); - try { - if (asPercent) { - configurableResource.setPercentage(resourceName, - findPercentage(resourceValue, "")); - } else { - configurableResource.setValue(resourceName, - Long.parseLong(resourceValue)); - } - } catch (ResourceNotFoundException ex) { - throw createConfigException(value, "The " - + "resource name, \"" + resourceName + "\" was not " - + "recognized. Please check the value of " - + YarnConfiguration.RESOURCE_TYPES + " in the Resource " - + "Manager's configuration files.", ex); - } catch (NumberFormatException ex) { - // This only comes from Long.parseLong() - throw createConfigException(value, "The " - + "resource values must all be integers. \"" + resourceValue - + "\" is not an integer.", ex); - } catch (AllocationConfigurationException ex) { - // This only comes from findPercentage() - throw createConfigException(value, "The " - + "resource values must all be percentages. \"" - + resourceValue + "\" is either not a number or does not " - + "include the '%' symbol.", ex); - } - } return configurableResource; } - private static ConfigurableResource parseOldStyleResourceAsPercentage( - String value) throws AllocationConfigurationException { - return new ConfigurableResource( - getResourcePercentage(StringUtils.toLowerCase(value))); - } - - private static ConfigurableResource parseOldStyleResource(String value) - throws AllocationConfigurationException { - final String lCaseValue = StringUtils.toLowerCase(value); - int memory = findResource(lCaseValue, "mb"); - int vcores = findResource(lCaseValue, "vcores"); - - return new ConfigurableResource( - BuilderUtils.newResource(memory, vcores)); - } - private static double[] getResourcePercentage( String val) throws AllocationConfigurationException { int numberOfKnownResourceTypes = ResourceUtils .getNumberOfKnownResourceTypes(); double[] resourcePercentage = new double[numberOfKnownResourceTypes]; String[] strings = val.split(","); - if (strings.length == 1) { double percentage = findPercentage(strings[0], ""); for (int i = 0; i < numberOfKnownResourceTypes; i++) { - resourcePercentage[i] = percentage; + resourcePercentage[i] = percentage/100; } } else { - resourcePercentage[0] = findPercentage(val, "memory"); - resourcePercentage[1] = findPercentage(val, "cpu"); + resourcePercentage[0] = findPercentage(val, "memory")/100; + resourcePercentage[1] = findPercentage(val, "cpu")/100; } - return resourcePercentage; } private static double findPercentage(String val, String units) - throws AllocationConfigurationException { + throws AllocationConfigurationException { final Pattern pattern = Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units); Matcher matcher = pattern.matcher(val); @@ -584,22 +467,7 @@ public class FairSchedulerConfiguration extends Configuration { units); } } - return Double.parseDouble(matcher.group(1)) / 100.0; - } - - private static AllocationConfigurationException createConfigException( - String value, String message) { - return createConfigException(value, message, null); - } - - private static AllocationConfigurationException createConfigException( - String value, String message, Throwable t) { - String msg = INVALID_RESOURCE_DEFINITION_PREFIX + value + ". " + message; - if (t != null) { - return new AllocationConfigurationException(msg, t); - } else { - return new AllocationConfigurationException(msg); - } + return Double.parseDouble(matcher.group(1)); } public long getUpdateInterval() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java index 441c34a..d5a436e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java @@ -134,7 +134,7 @@ public class AllocationFileQueueParser { if (MIN_RESOURCES.equals(field.getTagName())) { String text = getTrimmedTextData(field); ConfigurableResource val = - FairSchedulerConfiguration.parseResourceConfigValue(text, 0L); + FairSchedulerConfiguration.parseResourceConfigValue(text); builder.minQueueResources(queueName, val.getResource()); } else if (MAX_RESOURCES.equals(field.getTagName())) { String text = getTrimmedTextData(field); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 9d82bc7..d47f13d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -479,7 +479,7 @@ public class AppInfo { public int getNumNonAMContainersPreempted() { return numNonAMContainerPreempted; } - + public int getNumAMContainersPreempted() { return numAMContainerPreempted; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java index 163f707..81491b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java @@ -41,9 +41,8 @@ public class SchedulerInfo { protected EnumSet<SchedulerResourceTypes> schedulingResourceTypes; protected int maximumClusterPriority; - // JAXB needs this public SchedulerInfo() { - } + } // JAXB needs this public SchedulerInfo(final ResourceManager rm) { ResourceScheduler rs = rm.getResourceScheduler(); @@ -75,10 +74,7 @@ public class SchedulerInfo { } public String getSchedulerResourceTypes() { - if (minAllocResource != null) { - return Arrays.toString(minAllocResource.getResource().getResources()); - } - return null; + return Arrays.toString(minAllocResource.getResource().getResources()); } public int getMaxClusterLevelAppPriority() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 70f83ab..481645b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -48,9 +48,6 @@ import org.apache.log4j.spi.LoggingEvent; import org.junit.Assert; import org.junit.Test; -/** - * Tests fair scheduler configuration. - */ public class TestFairSchedulerConfiguration { private static final String A_CUSTOM_RESOURCE = "a-custom-resource"; @@ -105,152 +102,67 @@ public class TestFairSchedulerConfiguration { @Test public void testParseResourceConfigValue() throws Exception { - Resource expected = BuilderUtils.newResource(5 * 1024, 2); - Resource clusterResource = BuilderUtils.newResource(10 * 1024, 4); - - assertEquals(expected, - parseResourceConfigValue("2 vcores, 5120 mb").getResource()); - assertEquals(expected, - parseResourceConfigValue("5120 mb, 2 vcores").getResource()); - assertEquals(expected, - parseResourceConfigValue("2vcores,5120mb").getResource()); - assertEquals(expected, - parseResourceConfigValue("5120mb,2vcores").getResource()); - assertEquals(expected, - parseResourceConfigValue("5120mb mb, 2 vcores").getResource()); - assertEquals(expected, - parseResourceConfigValue("5120 Mb, 2 vCores").getResource()); - assertEquals(expected, - parseResourceConfigValue(" 5120 mb, 2 vcores ").getResource()); - assertEquals(expected, - parseResourceConfigValue(" 5120.3 mb, 2.35 vcores ").getResource()); - assertEquals(expected, - parseResourceConfigValue(" 5120. mb, 2. vcores ").getResource()); - - assertEquals(expected, + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("2 vcores, 1024 mb").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 mb, 2 vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("2vcores,1024mb").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024mb,2vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 mb, 2 vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 Mb, 2 vCores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource()); + + Resource clusterResource = BuilderUtils.newResource(2048, 4); + assertEquals(BuilderUtils.newResource(1024, 2), parseResourceConfigValue("50% memory, 50% cpu"). getResource(clusterResource)); - assertEquals(expected, + assertEquals(BuilderUtils.newResource(1024, 2), parseResourceConfigValue("50% Memory, 50% CpU"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(5 * 1024, 4), + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("50%").getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 4), parseResourceConfigValue("50% memory, 100% cpu"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(5 * 1024, 4), + assertEquals(BuilderUtils.newResource(1024, 4), parseResourceConfigValue(" 100% cpu, 50% memory"). getResource(clusterResource)); - assertEquals(BuilderUtils.newResource(5 * 1024, 0), + assertEquals(BuilderUtils.newResource(1024, 0), parseResourceConfigValue("50% memory, 0% cpu"). getResource(clusterResource)); - assertEquals(expected, + assertEquals(BuilderUtils.newResource(1024, 2), parseResourceConfigValue("50 % memory, 50 % cpu"). getResource(clusterResource)); - assertEquals(expected, + assertEquals(BuilderUtils.newResource(1024, 2), parseResourceConfigValue("50%memory,50%cpu"). getResource(clusterResource)); - assertEquals(expected, + assertEquals(BuilderUtils.newResource(1024, 2), parseResourceConfigValue(" 50 % memory, 50 % cpu "). getResource(clusterResource)); - assertEquals(expected, + assertEquals(BuilderUtils.newResource(1024, 2), parseResourceConfigValue("50.% memory, 50.% cpu"). getResource(clusterResource)); + + clusterResource = BuilderUtils.newResource(1024 * 10, 4); assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2), parseResourceConfigValue("10.9% memory, 50.6% cpu"). getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue("50%").getResource(clusterResource)); - - Configuration conf = new Configuration(); - - conf.set(YarnConfiguration.RESOURCE_TYPES, "test1"); - ResourceUtils.resetResourceTypes(conf); - - clusterResource = BuilderUtils.newResource(10 * 1024, 4); - expected = BuilderUtils.newResource(5 * 1024, 2); - expected.setResourceValue("test1", Long.MAX_VALUE); - - assertEquals(expected, - parseResourceConfigValue("vcores=2, memory-mb=5120").getResource()); - assertEquals(expected, - parseResourceConfigValue("memory-mb=5120, vcores=2").getResource()); - assertEquals(expected, - parseResourceConfigValue("vcores=2,memory-mb=5120").getResource()); - assertEquals(expected, parseResourceConfigValue(" vcores = 2 , " - + "memory-mb = 5120 ").getResource()); - - expected.setResourceValue("test1", 0L); - - assertEquals(expected, - parseResourceConfigValue("vcores=2, memory-mb=5120", 0L).getResource()); - assertEquals(expected, - parseResourceConfigValue("memory-mb=5120, vcores=2", 0L).getResource()); - assertEquals(expected, - parseResourceConfigValue("vcores=2,memory-mb=5120", 0L).getResource()); - assertEquals(expected, - parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 ", - 0L).getResource()); - - clusterResource.setResourceValue("test1", 8L); - expected.setResourceValue("test1", 4L); - - assertEquals(expected, - parseResourceConfigValue("50%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue("vcores=2, memory-mb=5120, " - + "test1=4").getResource()); - assertEquals(expected, - parseResourceConfigValue("test1=4, vcores=2, " - + "memory-mb=5120").getResource()); - assertEquals(expected, - parseResourceConfigValue("memory-mb=5120, test1=4, " - + "vcores=2").getResource()); - assertEquals(expected, - parseResourceConfigValue("vcores=2,memory-mb=5120," - + "test1=4").getResource()); - assertEquals(expected, - parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 , " - + "test1 = 4 ").getResource()); - - expected = BuilderUtils.newResource(4 * 1024, 3); - expected.setResourceValue("test1", 8L); - - assertEquals(expected, - parseResourceConfigValue("vcores=75%, " - + "memory-mb=40%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue("memory-mb=40%, " - + "vcores=75%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue("vcores=75%," - + "memory-mb=40%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue(" vcores = 75 % , " - + "memory-mb = 40 % ").getResource(clusterResource)); - - expected.setResourceValue("test1", 4L); - - assertEquals(expected, - parseResourceConfigValue("vcores=75%, memory-mb=40%, " - + "test1=50%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue("test1=50%, vcores=75%, " - + "memory-mb=40%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue("memory-mb=40%, test1=50%, " - + "vcores=75%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue("vcores=75%,memory-mb=40%," - + "test1=50%").getResource(clusterResource)); - assertEquals(expected, - parseResourceConfigValue(" vcores = 75 % , memory-mb = 40 % , " - + "test1 = 50 % ").getResource(clusterResource)); } - + @Test(expected = AllocationConfigurationException.class) public void testNoUnits() throws Exception { parseResourceConfigValue("1024"); } - + @Test(expected = AllocationConfigurationException.class) public void testOnlyMemory() throws Exception { parseResourceConfigValue("1024mb"); @@ -260,7 +172,7 @@ public class TestFairSchedulerConfiguration { public void testOnlyCPU() throws Exception { parseResourceConfigValue("1024vcores"); } - + @Test(expected = AllocationConfigurationException.class) public void testGibberish() throws Exception { parseResourceConfigValue("1o24vc0res"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 3902889..0702d65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -53,7 +53,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.*; +import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; +import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -72,12 +76,11 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.eclipse.jetty.server.Response; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -93,8 +96,6 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; public class TestRMWebServices extends JerseyTestBase { - private static final Logger LOG = - LoggerFactory.getLogger(TestRMWebServices.class); private static MockRM rm; @@ -471,19 +472,19 @@ public class TestRMWebServices extends JerseyTestBase { QueueMetrics metrics = rs.getRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); - long totalMBExpect = + long totalMBExpect = metrics.getAvailableMB() + metrics.getAllocatedMB(); - long totalVirtualCoresExpect = + long totalVirtualCoresExpect = metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores(); - assertEquals("appsSubmitted doesn't match", + assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), submittedApps); - assertEquals("appsCompleted doesn't match", + assertEquals("appsCompleted doesn't match", metrics.getAppsCompleted(), completedApps); assertEquals("reservedMB doesn't match", metrics.getReservedMB(), reservedMB); - assertEquals("availableMB doesn't match", + assertEquals("availableMB doesn't match", metrics.getAvailableMB(), availableMB); - assertEquals("allocatedMB doesn't match", + assertEquals("allocatedMB doesn't match", metrics.getAllocatedMB(), allocMB); assertEquals("reservedVirtualCores doesn't match", metrics.getReservedVirtualCores(), reservedVirtualCores); @@ -596,13 +597,11 @@ public class TestRMWebServices extends JerseyTestBase { public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException, Exception { - assertEquals("incorrect number of elements in: " + json, 1, json.length()); + assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); - assertEquals("incorrect number of elements in: " + info, 1, info.length()); + assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - - LOG.debug("schedulerInfo: {}", info); - assertEquals("incorrect number of elements in: " + info, 11, info.length()); + assertEquals("incorrect number of elements", 11, info.length()); verifyClusterSchedulerFifoGeneric(info.getString("type"), info.getString("qstate"), (float) info.getDouble("capacity"), http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 15f94e1..6c6f400 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -79,7 +79,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor; public class TestRMWebServicesApps extends JerseyTestBase { private static MockRM rm; - + private static final int CONTAINER_MB = 1024; private static class WebServletModule extends ServletModule { @@ -324,7 +324,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { assertEquals("incorrect number of elements", 1, apps.length()); array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 2, array.length()); - assertTrue("both app states of ACCEPTED and KILLED are not present", + assertTrue("both app states of ACCEPTED and KILLED are not present", (array.getJSONObject(0).getString("state").equals("ACCEPTED") && array.getJSONObject(1).getString("state").equals("KILLED")) || (array.getJSONObject(0).getString("state").equals("KILLED") && @@ -375,12 +375,12 @@ public class TestRMWebServicesApps extends JerseyTestBase { assertEquals("incorrect number of elements", 1, apps.length()); array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 2, array.length()); - assertTrue("both app states of ACCEPTED and KILLED are not present", + assertTrue("both app states of ACCEPTED and KILLED are not present", (array.getJSONObject(0).getString("state").equals("ACCEPTED") && array.getJSONObject(1).getString("state").equals("KILLED")) || (array.getJSONObject(0).getString("state").equals("KILLED") && array.getJSONObject(1).getString("state").equals("ACCEPTED"))); - + rm.stop(); } @@ -511,8 +511,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("finalStatus", - FinalApplicationStatus.UNDEFINED.toString()) + .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); @@ -1805,8 +1804,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { int numAttempt = 1; while (true) { // fail the AM by sending CONTAINER_FINISHED event without registering. - amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, - ContainerState.COMPLETE); + amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED); if (numAttempt == maxAppAttempts) { rm.waitForState(app1.getApplicationId(), RMAppState.FAILED); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java deleted file mode 100644 index 83e0056..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java +++ /dev/null @@ -1,242 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.webapp; - -import com.google.inject.Guice; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.MockAM; -import org.apache.hadoop.yarn.server.resourcemanager.MockNM; -import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase; -import org.apache.hadoop.yarn.util.resource.ResourceUtils; -import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; -import org.apache.hadoop.yarn.webapp.GuiceServletConfig; -import org.apache.hadoop.yarn.webapp.JerseyTestBase; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.junit.Before; -import org.junit.Test; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import javax.ws.rs.core.MediaType; -import java.util.ArrayList; - -import static org.junit.Assert.assertEquals; - -/** - * This test verifies that custom resource types are correctly serialized to XML - * and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps. - */ -public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase { - - private static MockRM rm; - private static final int CONTAINER_MB = 1024; - - private static class WebServletModule extends ServletModule { - @Override - protected void configureServlets() { - bind(JAXBContextResolver.class); - bind(RMWebServices.class); - bind(GenericExceptionHandler.class); - Configuration conf = new Configuration(); - conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, - ResourceScheduler.class); - initResourceTypes(conf); - rm = new MockRM(conf); - bind(ResourceManager.class).toInstance(rm); - serve("/*").with(GuiceContainer.class); - } - - private void initResourceTypes(Configuration conf) { - conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, - CustomResourceTypesConfigurationProvider.class.getName()); - ResourceUtils.resetResourceTypes(conf); - } - } - - @Before - @Override - public void setUp() throws Exception { - super.setUp(); - createInjectorForWebServletModule(); - } - - private void createInjectorForWebServletModule() { - GuiceServletConfig - .setInjector(Guice.createInjector(new WebServletModule())); - } - - public TestRMWebServicesAppsCustomResourceTypes() { - super(new WebAppDescriptor.Builder( - "org.apache.hadoop.yarn.server.resourcemanager.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); - } - - @Test - public void testRunningAppXml() throws Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); - am1.allocate("*", 2048, 1, new ArrayList<>()); - amNodeManager.nodeHeartbeat(true); - - WebResource r = resource(); - WebResource path = r.path("ws").path("v1").path("cluster").path("apps"); - ClientResponse response = - path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class); - - XmlCustomResourceTypeTestCase testCase = - new XmlCustomResourceTypeTestCase(path, - new BufferedClientResponse(response)); - testCase.verify(document -> { - NodeList apps = document.getElementsByTagName("apps"); - assertEquals("incorrect number of apps elements", 1, apps.getLength()); - - NodeList appArray = ((Element)(apps.item(0))) - .getElementsByTagName("app"); - assertEquals("incorrect number of app elements", 1, appArray.getLength()); - - verifyAppsXML(appArray, app1); - }); - - rm.stop(); - } - - @Test - public void testRunningAppJson() throws Exception { - rm.start(); - MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); - am1.allocate("*", 2048, 1, new ArrayList<>()); - amNodeManager.nodeHeartbeat(true); - - WebResource r = resource(); - WebResource path = r.path("ws").path("v1").path("cluster").path("apps"); - ClientResponse response = - path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); - - JsonCustomResourceTypeTestcase testCase = - new JsonCustomResourceTypeTestcase(path, - new BufferedClientResponse(response)); - testCase.verify(json -> { - try { - assertEquals("incorrect number of apps elements", 1, json.length()); - JSONObject apps = json.getJSONObject("apps"); - assertEquals("incorrect number of app elements", 1, apps.length()); - JSONArray array = apps.getJSONArray("app"); - assertEquals("incorrect count of app", 1, array.length()); - - verifyAppInfoJson(array.getJSONObject(0), app1); - } catch (JSONException e) { - throw new RuntimeException(e); - } - }); - - rm.stop(); - } - - private void verifyAppsXML(NodeList appArray, RMApp app) { - for (int i = 0; i < appArray.getLength(); i++) { - Element element = (Element) appArray.item(i); - AppInfoXmlVerifications.verify(element, app); - - NodeList resourceRequests = - element.getElementsByTagName("resourceRequests"); - assertEquals(1, resourceRequests.getLength()); - Node resourceRequest = resourceRequests.item(0); - ResourceRequest rr = - ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) - .getApplicationAttempt( - app.getCurrentAppAttempt().getAppAttemptId()) - .getAppSchedulingInfo().getAllResourceRequests().get(0); - ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes( - (Element) resourceRequest, rr, - CustomResourceTypesConfigurationProvider.getCustomResourceTypes()); - } - } - - private void verifyAppInfoJson(JSONObject info, RMApp app) throws - JSONException { - int expectedNumberOfElements = getExpectedNumberOfElements(app); - - assertEquals("incorrect number of elements", expectedNumberOfElements, - info.length()); - - AppInfoJsonVerifications.verify(info, app); - - JSONArray resourceRequests = info.getJSONArray("resourceRequests"); - JSONObject requestInfo = resourceRequests.getJSONObject(0); - ResourceRequest rr = - ((AbstractYarnScheduler) rm.getRMContext().getScheduler()) - .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId()) - .getAppSchedulingInfo().getAllResourceRequests().get(0); - - ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes( - requestInfo, rr, - CustomResourceTypesConfigurationProvider.getCustomResourceTypes()); - } - - private int getExpectedNumberOfElements(RMApp app) { - int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests - if (app.getApplicationSubmissionContext() - .getNodeLabelExpression() != null) { - expectedNumberOfElements++; - } - - if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { - expectedNumberOfElements++; - } - - if (AppInfo - .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) { - expectedNumberOfElements++; - } - return expectedNumberOfElements; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 46d0a66..e37f76f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -146,7 +146,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { config.setUserLimitFactor(B2, 100.0f); config.setCapacity(B3, 0.5f); config.setUserLimitFactor(B3, 100.0f); - + config.setQueues(A1, new String[] {"a1a", "a1b"}); final String A1A = A1 + ".a1a"; config.setCapacity(A1A, 85); @@ -254,7 +254,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { } } - public void verifySubQueueXML(Element qElem, String q, + public void verifySubQueueXML(Element qElem, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws Exception { NodeList children = qElem.getChildNodes(); @@ -317,34 +317,30 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { private void verifyClusterScheduler(JSONObject json) throws JSONException, Exception { - assertEquals("incorrect number of elements in: " + json, 1, json.length()); + assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); - assertEquals("incorrect number of elements in: " + info, 1, info.length()); + assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 8, info.length()); + assertEquals("incorrect number of elements", 8, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), (float) info.getDouble("maxCapacity"), info.getString("queueName")); JSONObject health = info.getJSONObject("health"); assertNotNull(health); - assertEquals("incorrect number of elements in: " + health, 3, - health.length()); + assertEquals("incorrect number of elements", 3, health.length()); JSONArray operationsInfo = health.getJSONArray("operationsInfo"); - assertEquals("incorrect number of elements in: " + health, 4, - operationsInfo.length()); + assertEquals("incorrect number of elements", 4, operationsInfo.length()); JSONArray lastRunDetails = health.getJSONArray("lastRunDetails"); - assertEquals("incorrect number of elements in: " + health, 3, - lastRunDetails.length()); + assertEquals("incorrect number of elements", 3, lastRunDetails.length()); JSONArray arr = info.getJSONObject("queues").getJSONArray("queue"); - assertEquals("incorrect number of elements in: " + arr, 2, arr.length()); + assertEquals("incorrect number of elements", 2, arr.length()); // test subqueues for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); - String q = CapacitySchedulerConfiguration.ROOT + "." + - obj.getString("queueName"); + String q = CapacitySchedulerConfiguration.ROOT + "." + obj.getString("queueName"); verifySubQueue(obj, q, 100, 100); } } @@ -359,7 +355,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { assertTrue("queueName doesn't match", "root".matches(queueName)); } - private void verifySubQueue(JSONObject info, String q, + private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { int numExpectedElements = 20; @@ -468,7 +464,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f); } - //Return a child Node of node with the tagname or null if none exists + //Return a child Node of node with the tagname or null if none exists private Node getChildNodeByName(Node node, String tagname) { NodeList nodeList = node.getChildNodes(); for (int i=0; i < nodeList.getLength(); ++i) { @@ -518,7 +514,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { for (int j=0; j<users.getLength(); ++j) { Node user = users.item(j); String username = getChildNodeByName(user, "username") - .getTextContent(); + .getTextContent(); assertTrue(username.equals("user1") || username.equals("user2")); //Should be a parsable integer Integer.parseInt(getChildNodeByName(getChildNodeByName(user, http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 99b5648..3d28f12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; @@ -61,8 +59,6 @@ import static org.junit.Assert.assertNull; * Test scheduler configuration mutation via REST API. */ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { - private static final Logger LOG = LoggerFactory - .getLogger(TestRMWebServicesConfigurationMutation.class); private static final File CONF_FILE = new File(new File("target", "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE); @@ -400,7 +396,6 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); - LOG.debug("Response headers: " + response.getHeaders()); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = cs.getConfiguration(); assertEquals(0.2f, newCSConf http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java index 58c72ee..e77785b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,14 +16,13 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler; +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import javax.ws.rs.core.MediaType; -import com.google.inject.Guice; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -31,9 +30,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; - -import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -42,18 +38,18 @@ import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.Before; import org.junit.Test; -import javax.ws.rs.core.MediaType; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; -/** - * Tests RM Webservices fair scheduler resources. - */ public class TestRMWebServicesFairScheduler extends JerseyTestBase { private static MockRM rm; private static YarnConfiguration conf; - + private static class WebServletModule extends ServletModule { @Override protected void configureServlets() { @@ -62,7 +58,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase { bind(GenericExceptionHandler.class); conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); + ResourceScheduler.class); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); @@ -70,32 +66,32 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase { } static { - GuiceServletConfig - .setInjector(Guice.createInjector(new WebServletModule())); + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule())); } @Before @Override public void setUp() throws Exception { super.setUp(); - GuiceServletConfig - .setInjector(Guice.createInjector(new WebServletModule())); + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule())); } public TestRMWebServicesFairScheduler() { super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.resourcemanager.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); } - + @Test - public void testClusterScheduler() throws JSONException { + public void testClusterScheduler() throws JSONException, Exception { WebResource r = resource(); - ClientResponse response = - r.path("ws").path("v1").path("cluster").path("scheduler") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); @@ -103,51 +99,52 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase { } @Test - public void testClusterSchedulerSlash() throws JSONException { + public void testClusterSchedulerSlash() throws JSONException, Exception { WebResource r = resource(); - ClientResponse response = - r.path("ws").path("v1").path("cluster").path("scheduler/") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler/").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); verifyClusterScheduler(json); } - + @Test - public void testClusterSchedulerWithSubQueues() - throws JSONException { - FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + public void testClusterSchedulerWithSubQueues() throws JSONException, + Exception { + FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler(); QueueManager queueManager = scheduler.getQueueManager(); // create LeafQueue queueManager.getLeafQueue("root.q.subqueue1", true); queueManager.getLeafQueue("root.q.subqueue2", true); WebResource r = resource(); - ClientResponse response = - r.path("ws").path("v1").path("cluster").path("scheduler") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); JSONArray subQueueInfo = json.getJSONObject("scheduler") .getJSONObject("schedulerInfo").getJSONObject("rootQueue") - .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1) - .getJSONObject("childQueues").getJSONArray("queue"); + .getJSONObject("childQueues").getJSONArray("queue") + .getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue"); // subQueueInfo is consist of subqueue1 and subqueue2 info assertEquals(2, subQueueInfo.length()); // Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo. try { subQueueInfo.getJSONObject(1).getJSONObject("childQueues"); - fail("FairSchedulerQueueInfo should omit field 'childQueues'" - + "if child queue is empty."); + fail("FairSchedulerQueueInfo should omit field 'childQueues'" + + "if child queue is empty."); } catch (JSONException je) { assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage()); } } - private void verifyClusterScheduler(JSONObject json) throws JSONException { + private void verifyClusterScheduler(JSONObject json) throws JSONException, + Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 40cf483..1e61186 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -457,7 +457,7 @@ public class TestRMWebServicesSchedulerActivities if (object.getClass() == JSONObject.class) { assertEquals("Number of allocations is wrong", 1, realValue); } else if (object.getClass() == JSONArray.class) { - assertEquals("Number of allocations is wrong in: " + object, + assertEquals("Number of allocations is wrong", ((JSONArray) object).length(), realValue); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java deleted file mode 100644 index bb1fce0..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.LocalConfigurationProvider; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toList; - -/** - * This class can generate an XML configuration file of custom resource types. - * See createInitialResourceTypes for the default values. All custom resource - * type is prefixed with CUSTOM_RESOURCE_PREFIX. Please use the - * getConfigurationInputStream method to get an InputStream of the XML. If you - * want to have different number of resources in your tests, please see usages - * of this class in this test class: - * {@link TestRMWebServicesFairSchedulerCustomResourceTypes} - * - */ -public class CustomResourceTypesConfigurationProvider - extends LocalConfigurationProvider { - - private static class CustomResourceTypes { - private int count; - private String xml; - - CustomResourceTypes(String xml, int count) { - this.xml = xml; - this.count = count; - } - - public int getCount() { - return count; - } - - public String getXml() { - return xml; - } - } - - private static final String CUSTOM_RESOURCE_PREFIX = "customResource-"; - - private static CustomResourceTypes customResourceTypes = - createInitialResourceTypes(); - - private static CustomResourceTypes createInitialResourceTypes() { - return createCustomResourceTypes(2); - } - - private static CustomResourceTypes createCustomResourceTypes(int count) { - List<String> resourceTypeNames = generateResourceTypeNames(count); - - List<String> resourceUnitXmlElements = IntStream.range(0, count) - .boxed() - .map(i -> getResourceUnitsXml(resourceTypeNames.get(i))) - .collect(toList()); - - StringBuilder sb = new StringBuilder("<configuration>\n"); - sb.append(getResourceTypesXml(resourceTypeNames)); - - for (String resourceUnitXml : resourceUnitXmlElements) { - sb.append(resourceUnitXml); - - } - sb.append("</configuration>"); - - return new CustomResourceTypes(sb.toString(), count); - } - - private static List<String> generateResourceTypeNames(int count) { - return IntStream.range(0, count) - .boxed() - .map(i -> CUSTOM_RESOURCE_PREFIX + i) - .collect(toList()); - } - - private static String getResourceUnitsXml(String resource) { - return "<property>\n" + "<name>yarn.resource-types." + resource - + ".units</name>\n" + "<value>k</value>\n" + "</property>\n"; - } - - private static String getResourceTypesXml(List<String> resources) { - final String resourceTypes = makeCommaSeparatedString(resources); - - return "<property>\n" + "<name>yarn.resource-types</name>\n" + "<value>" - + resourceTypes + "</value>\n" + "</property>\n"; - } - - private static String makeCommaSeparatedString(List<String> resources) { - return resources.stream().collect(Collectors.joining(",")); - } - - @Override - public InputStream getConfigurationInputStream(Configuration bootstrapConf, - String name) throws YarnException, IOException { - if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) { - return new ByteArrayInputStream( - customResourceTypes.getXml().getBytes()); - } else { - return super.getConfigurationInputStream(bootstrapConf, name); - } - } - - public static void reset() { - customResourceTypes = createInitialResourceTypes(); - } - - public static void setNumberOfResourceTypes(int count) { - customResourceTypes = createCustomResourceTypes(count); - } - - public static List<String> getCustomResourceTypes() { - return generateResourceTypeNames(customResourceTypes.getCount()); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org