YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev.
(cherry picked from commit 58da54640e37571f82eb340f32641d58160e92b2) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83436f27 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83436f27 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83436f27 Branch: refs/heads/resource-types Commit: 83436f27c38c8294bf54704e878980595e26c6d7 Parents: 457d521 Author: Sunil G <sun...@apache.org> Authored: Tue Apr 25 11:53:11 2017 +0530 Committer: Daniel Templeton <templ...@apache.org> Committed: Tue Oct 17 09:17:10 2017 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/api/records/Resource.java | 25 ++++++++ .../yarn/api/records/ResourceInformation.java | 33 +++++++---- .../hadoop/yarn/util/UnitsConversionUtil.java | 34 ++++++----- .../yarn/conf/TestResourceInformation.java | 4 +- .../yarn/util/TestUnitsConversionUtil.java | 60 ++++++++++---------- .../api/records/impl/pb/ResourcePBImpl.java | 23 ++++---- .../resource/DominantResourceCalculator.java | 54 ++++++++---------- .../hadoop/yarn/util/resource/Resources.java | 18 +++--- .../yarn/util/resource/TestResourceUtils.java | 1 + .../yarn/util/resource/TestResources.java | 12 +++- 10 files changed, 156 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index a25c8ac..c51357c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.api.records; import org.apache.commons.lang.NotImplementedException; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; @@ -101,6 +103,29 @@ public abstract class Resource implements Comparable<Resource> { return new SimpleResource(memory, vCores); } + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static Resource newInstance(Resource resource) { + Resource ret = Resource.newInstance(0, 0); + Resource.copy(resource, ret); + return ret; + } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static void copy(Resource source, Resource dest) { + for (Map.Entry<String, ResourceInformation> entry : source.getResources() + .entrySet()) { + try { + ResourceInformation.copy(entry.getValue(), + dest.getResourceInformation(entry.getKey())); + } catch (YarnException ye) { + dest.setResourceInformation(entry.getKey(), + ResourceInformation.newInstance(entry.getValue())); + } + } + } + /** * This method is DEPRECATED: * Use {@link Resource#getMemorySize()} instead http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index a17e81b..76e754b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -30,7 +30,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> { private String name; private String units; private ResourceTypes resourceType; - private Long value; + private long value; private static final String MEMORY_URI = "memory-mb"; private static final String VCORES_URI = "vcores"; @@ -104,7 +104,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> { * * @return the resource value */ - public Long getValue() { + public long getValue() { return value; } @@ -113,7 +113,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> { * * @param rValue the resource value */ - public void setValue(Long rValue) { + public void setValue(long rValue) { this.value = rValue; } @@ -125,15 +125,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> { */ public static ResourceInformation newInstance(ResourceInformation other) { ResourceInformation ret = new ResourceInformation(); - ret.setName(other.getName()); - ret.setResourceType(other.getResourceType()); - ret.setUnits(other.getUnits()); - ret.setValue(other.getValue()); + copy(other, ret); return ret; } public static ResourceInformation newInstance(String name, String units, - Long value, ResourceTypes type) { + long value, ResourceTypes type) { ResourceInformation ret = new ResourceInformation(); ret.setName(name); ret.setResourceType(type); @@ -143,7 +140,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> { } public static ResourceInformation newInstance(String name, String units, - Long value) { + long value) { return ResourceInformation .newInstance(name, units, value, ResourceTypes.COUNTABLE); } @@ -153,7 +150,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> { .newInstance(name, units, 0L, ResourceTypes.COUNTABLE); } - public static ResourceInformation newInstance(String name, Long value) { + public static ResourceInformation newInstance(String name, long value) { return ResourceInformation .newInstance(name, "", value, ResourceTypes.COUNTABLE); } @@ -162,6 +159,20 @@ public class ResourceInformation implements Comparable<ResourceInformation> { return ResourceInformation.newInstance(name, ""); } + /** + * Copies the content of the source ResourceInformation object to the + * destination object, overwriting all properties of the destination object. + * @param src Source ResourceInformation object + * @param dst Destination ResourceInformation object + */ + + public static void copy(ResourceInformation src, ResourceInformation dst) { + dst.setName(src.getName()); + dst.setResourceType(src.getResourceType()); + dst.setUnits(src.getUnits()); + dst.setValue(src.getValue()); + } + @Override public String toString() { return "name: " + this.name + ", units: " + this.units + ", type: " @@ -197,7 +208,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> { 939769357 + name.hashCode(); // prime * result = 939769357 initially result = prime * result + resourceType.hashCode(); result = prime * result + units.hashCode(); - result = prime * result + value.hashCode(); + result = prime * result + Long.hashCode(value); return result; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java index 47bb3df..c7663de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -128,23 +128,24 @@ public class UnitsConversionUtil { * @param fromValue the value you wish to convert * @return the value in toUnit */ - public static Long convert(String fromUnit, String toUnit, Long fromValue) { - if (toUnit == null || fromUnit == null || fromValue == null) { + public static long convert(String fromUnit, String toUnit, long fromValue) { + if (toUnit == null || fromUnit == null) { throw new IllegalArgumentException("One or more arguments are null"); } - String overflowMsg = - "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit - + "' will result in an overflow of Long"; + if (fromUnit.equals(toUnit)) { return fromValue; } Converter fc = getConverter(fromUnit); Converter tc = getConverter(toUnit); - Long numerator = fc.numerator * tc.denominator; - Long denominator = fc.denominator * tc.numerator; - Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator; + long numerator = fc.numerator * tc.denominator; + long denominator = fc.denominator * tc.numerator; + long numeratorMultiplierLimit = Long.MAX_VALUE / numerator; if (numerator < denominator) { if (numeratorMultiplierLimit < fromValue) { + String overflowMsg = + "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + + "' will result in an overflow of Long"; throw new IllegalArgumentException(overflowMsg); } return (fromValue * numerator) / denominator; @@ -152,8 +153,11 @@ public class UnitsConversionUtil { if (numeratorMultiplierLimit > fromValue) { return (numerator * fromValue) / denominator; } - Long tmp = numerator / denominator; + long tmp = numerator / denominator; if ((Long.MAX_VALUE / tmp) < fromValue) { + String overflowMsg = + "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + + "' will result in an overflow of Long"; throw new IllegalArgumentException(overflowMsg); } return fromValue * tmp; @@ -170,8 +174,8 @@ public class UnitsConversionUtil { * @return +1, 0 or -1 depending on whether the relationship is greater than, * equal to or lesser than */ - public static int compare(String unitA, Long valueA, String unitB, - Long valueB) { + public static int compare(String unitA, long valueA, String unitB, + long valueB) { if (unitA == null || unitB == null || !KNOWN_UNITS.contains(unitA) || !KNOWN_UNITS.contains(unitB)) { throw new IllegalArgumentException("Units cannot be null"); @@ -185,19 +189,19 @@ public class UnitsConversionUtil { Converter unitAC = getConverter(unitA); Converter unitBC = getConverter(unitB); if (unitA.equals(unitB)) { - return valueA.compareTo(valueB); + return Long.valueOf(valueA).compareTo(valueB); } int unitAPos = SORTED_UNITS.indexOf(unitA); int unitBPos = SORTED_UNITS.indexOf(unitB); try { - Long tmpA = valueA; - Long tmpB = valueB; + long tmpA = valueA; + long tmpB = valueB; if (unitAPos < unitBPos) { tmpB = convert(unitB, unitA, valueB); } else { tmpA = convert(unitA, unitB, valueA); } - return tmpA.compareTo(tmpB); + return Long.valueOf(tmpA).compareTo(tmpB); } catch (IllegalArgumentException ie) { BigInteger tmpA = BigInteger.valueOf(valueA); BigInteger tmpB = BigInteger.valueOf(valueB); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java index 28f69c9..2efb990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java @@ -50,7 +50,7 @@ public class TestResourceInformation { @Test public void testValue() { String name = "yarn.io/test"; - Long value = 1l; + long value = 1L; ResourceInformation ri = ResourceInformation.newInstance(name, value); Assert.assertEquals("Resource name incorrect", name, ri.getName()); Assert.assertEquals("Resource value incorrect", value, ri.getValue()); @@ -59,7 +59,7 @@ public class TestResourceInformation { @Test public void testResourceInformation() { String name = "yarn.io/test"; - Long value = 1l; + long value = 1L; String units = "m"; ResourceInformation ri = ResourceInformation.newInstance(name, units, value); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java index 0f999e8..0b512a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java @@ -27,62 +27,62 @@ public class TestUnitsConversionUtil { public void testUnitsConversion() { int value = 5; String fromUnit = ""; - Long test = Long.valueOf(value); + long test = value; Assert.assertEquals("pico test failed", - Long.valueOf(value * 1000l * 1000l * 1000l * 1000l), + value * 1000L * 1000L * 1000L * 1000L, UnitsConversionUtil.convert(fromUnit, "p", test)); Assert.assertEquals("nano test failed", - Long.valueOf(value * 1000l * 1000l * 1000l), + value * 1000L * 1000L * 1000L, UnitsConversionUtil.convert(fromUnit, "n", test)); Assert - .assertEquals("micro test failed", Long.valueOf(value * 1000l * 1000l), + .assertEquals("micro test failed", value * 1000L * 1000L, UnitsConversionUtil.convert(fromUnit, "u", test)); - Assert.assertEquals("milli test failed", Long.valueOf(value * 1000l), + Assert.assertEquals("milli test failed", value * 1000L, UnitsConversionUtil.convert(fromUnit, "m", test)); - test = Long.valueOf(value * 1000l * 1000l * 1000l * 1000l * 1000l); + test = value * 1000L * 1000L * 1000L * 1000L * 1000L; fromUnit = ""; - Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l), + Assert.assertEquals("kilo test failed", test / 1000L, UnitsConversionUtil.convert(fromUnit, "k", test)); Assert - .assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)), + .assertEquals("mega test failed", test / (1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "M", test)); Assert.assertEquals("giga test failed", - Long.valueOf(test / (1000l * 1000l * 1000l)), + test / (1000L * 1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "G", test)); Assert.assertEquals("tera test failed", - Long.valueOf(test / (1000l * 1000l * 1000l * 1000l)), + test / (1000L * 1000L * 1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "T", test)); Assert.assertEquals("peta test failed", - Long.valueOf(test / (1000l * 1000l * 1000l * 1000l * 1000l)), + test / (1000L * 1000L * 1000L * 1000L * 1000L), UnitsConversionUtil.convert(fromUnit, "P", test)); - Assert.assertEquals("nano to pico test failed", Long.valueOf(value * 1000l), - UnitsConversionUtil.convert("n", "p", Long.valueOf(value))); + Assert.assertEquals("nano to pico test failed", value * 1000L, + UnitsConversionUtil.convert("n", "p", value)); - Assert.assertEquals("mega to giga test failed", Long.valueOf(value), - UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l))); + Assert.assertEquals("mega to giga test failed", value, + UnitsConversionUtil.convert("M", "G", value * 1000L)); - Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value), - UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l))); + Assert.assertEquals("Mi to Gi test failed", value, + UnitsConversionUtil.convert("Mi", "Gi", value * 1024L)); - Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024), - UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value))); + Assert.assertEquals("Mi to Ki test failed", value * 1024, + UnitsConversionUtil.convert("Mi", "Ki", value)); - Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024), - UnitsConversionUtil.convert("Ki", "", Long.valueOf(5))); + Assert.assertEquals("Ki to base units test failed", 5 * 1024, + UnitsConversionUtil.convert("Ki", "", 5)); - Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741), - UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024))); + Assert.assertEquals("Mi to k test failed", 1073741, + UnitsConversionUtil.convert("Mi", "k", 1024)); - Assert.assertEquals("M to Mi test failed", Long.valueOf(953), - UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000))); + Assert.assertEquals("M to Mi test failed", 953, + UnitsConversionUtil.convert("M", "Mi", 1000)); } @Test public void testOverflow() { - Long test = Long.valueOf(5 * 1000l * 1000l * 1000l * 1000l * 1000l); + long test = 5 * 1000L * 1000L * 1000L * 1000L * 1000L; try { UnitsConversionUtil.convert("P", "p", test); Assert.fail("this operation should result in an overflow"); @@ -100,9 +100,9 @@ public class TestUnitsConversionUtil { @Test public void testCompare() { String unitA = "P"; - Long valueA = Long.valueOf(1); + long valueA = 1; String unitB = "p"; - Long valueB = Long.valueOf(2); + long valueB = 2; Assert.assertEquals(1, UnitsConversionUtil.compare(unitA, valueA, unitB, valueB)); Assert.assertEquals(-1, @@ -120,7 +120,7 @@ public class TestUnitsConversionUtil { Assert.assertEquals(-1, UnitsConversionUtil.compare(unitB, valueB, unitA, valueA)); Assert.assertEquals(0, - UnitsConversionUtil.compare(unitA, valueA, unitB, 1000l)); + UnitsConversionUtil.compare(unitA, valueA, unitB, 1000L)); unitA = "p"; unitB = "n"; @@ -129,7 +129,7 @@ public class TestUnitsConversionUtil { Assert.assertEquals(1, UnitsConversionUtil.compare(unitB, valueB, unitA, valueA)); Assert.assertEquals(0, - UnitsConversionUtil.compare(unitA, 1000l, unitB, valueA)); + UnitsConversionUtil.compare(unitA, 1000L, unitB, valueA)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 472e645..691e723 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import org.apache.commons.collections.map.UnmodifiableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -46,6 +47,7 @@ public class ResourcePBImpl extends Resource { boolean viaProto = false; private Map<String, ResourceInformation> resources; + private Map<String, ResourceInformation> readOnlyResources; // call via ProtoUtils.convertToProtoFormat(Resource) @@ -68,6 +70,7 @@ public class ResourcePBImpl extends Resource { public ResourcePBImpl(ResourceProto proto) { this.proto = proto; viaProto = true; + this.readOnlyResources = null; this.resources = null; initResources(); } @@ -111,10 +114,9 @@ public class ResourcePBImpl extends Resource { @Override public void setMemorySize(long memory) { - setResourceInformation(ResourceInformation.MEMORY_MB.getName(), - ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(), - ResourceInformation.MEMORY_MB.getUnits(), memory)); - + maybeInitBuilder(); + getResourceInformation(ResourceInformation.MEMORY_MB.getName()) + .setValue(memory); } @Override @@ -127,9 +129,9 @@ public class ResourcePBImpl extends Resource { @Override public void setVirtualCores(int vCores) { - setResourceInformation(ResourceInformation.VCORES.getName(), - ResourceInformation.newInstance(ResourceInformation.VCORES.getName(), - ResourceInformation.VCORES.getUnits(), (long) vCores)); + maybeInitBuilder(); + getResourceInformation(ResourceInformation.VCORES.getName()) + .setValue(vCores); } private void initResources() { @@ -143,7 +145,7 @@ public class ResourcePBImpl extends Resource { entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : ResourceTypes.COUNTABLE; String units = entry.hasUnits() ? entry.getUnits() : ""; - Long value = entry.hasValue() ? entry.getValue() : 0L; + long value = entry.hasValue() ? entry.getValue() : 0L; ResourceInformation ri = ResourceInformation.newInstance(entry.getKey(), units, value, type); if (resources.containsKey(ri.getName())) { @@ -171,7 +173,7 @@ public class ResourcePBImpl extends Resource { } initResources(); if (resources.containsKey(resource)) { - resources.put(resource, resourceInformation); + ResourceInformation.copy(resourceInformation, resources.get(resource)); } } @@ -193,7 +195,7 @@ public class ResourcePBImpl extends Resource { @Override public Map<String, ResourceInformation> getResources() { initResources(); - return Collections.unmodifiableMap(this.resources); + return readOnlyResources; } @Override @@ -226,6 +228,7 @@ public class ResourcePBImpl extends Resource { resources.put(entry.getKey(), ResourceInformation.newInstance(entry.getValue())); } + readOnlyResources = Collections.unmodifiableMap(resources); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 7db1da4..76bc364 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -154,7 +154,7 @@ public class DominantResourceCalculator extends ResourceCalculator { clusterResource.getResourceInformation(rName); ResourceInformation resourceInformation = resource.getResourceInformation(rName); - Long resourceValue = UnitsConversionUtil + long resourceValue = UnitsConversionUtil .convert(resourceInformation.getUnits(), clusterResourceResourceInformation.getUnits(), resourceInformation.getValue()); @@ -180,11 +180,11 @@ public class DominantResourceCalculator extends ResourceCalculator { available.getResourceInformation(resource); ResourceInformation requiredResource = required.getResourceInformation(resource); - Long requiredResourceValue = UnitsConversionUtil + long requiredResourceValue = UnitsConversionUtil .convert(requiredResource.getUnits(), availableResource.getUnits(), requiredResource.getValue()); if (requiredResourceValue != 0) { - Long tmp = availableResource.getValue() / requiredResourceValue; + long tmp = availableResource.getValue() / requiredResourceValue; min = min < tmp ? min : tmp; } } catch (YarnException ye) { @@ -228,7 +228,7 @@ public class DominantResourceCalculator extends ResourceCalculator { a.getResourceInformation(resource); ResourceInformation bResourceInformation = b.getResourceInformation(resource); - Long bResourceValue = UnitsConversionUtil + long bResourceValue = UnitsConversionUtil .convert(bResourceInformation.getUnits(), aResourceInformation.getUnits(), bResourceInformation.getValue()); @@ -249,14 +249,13 @@ public class DominantResourceCalculator extends ResourceCalculator { } public Resource divideAndCeil(Resource numerator, long denominator) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(numerator); for (String resource : resourceNames) { try { - ResourceInformation resourceInformation = ResourceInformation - .newInstance(numerator.getResourceInformation(resource)); + ResourceInformation resourceInformation = + ret.getResourceInformation(resource); resourceInformation.setValue( divideAndCeil(resourceInformation.getValue(), denominator)); - ret.setResourceInformation(resource, resourceInformation); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -276,7 +275,7 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { try { ResourceInformation rResourceInformation = @@ -287,28 +286,26 @@ public class DominantResourceCalculator extends ResourceCalculator { maximumResource.getResourceInformation(resource); ResourceInformation stepFactorResourceInformation = stepFactor.getResourceInformation(resource); - ResourceInformation tmp = - ResourceInformation.newInstance(rResourceInformation); + ResourceInformation tmp = ret.getResourceInformation(resource); - Long rValue = rResourceInformation.getValue(); - Long minimumValue = UnitsConversionUtil + long rValue = rResourceInformation.getValue(); + long minimumValue = UnitsConversionUtil .convert(minimumResourceInformation.getUnits(), rResourceInformation.getUnits(), minimumResourceInformation.getValue()); - Long maximumValue = UnitsConversionUtil + long maximumValue = UnitsConversionUtil .convert(maximumResourceInformation.getUnits(), rResourceInformation.getUnits(), maximumResourceInformation.getValue()); - Long stepFactorValue = UnitsConversionUtil + long stepFactorValue = UnitsConversionUtil .convert(stepFactorResourceInformation.getUnits(), rResourceInformation.getUnits(), stepFactorResourceInformation.getValue()); - Long value = Math.max(rValue, minimumValue); + long value = Math.max(rValue, minimumValue); if (stepFactorValue != 0) { value = roundUp(value, stepFactorValue); } tmp.setValue(Math.min(value, maximumValue)); - ret.setResourceInformation(resource, tmp); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -328,28 +325,27 @@ public class DominantResourceCalculator extends ResourceCalculator { } private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { try { ResourceInformation rResourceInformation = r.getResourceInformation(resource); ResourceInformation stepFactorResourceInformation = stepFactor.getResourceInformation(resource); - ResourceInformation tmp = - ResourceInformation.newInstance(rResourceInformation); - Long rValue = rResourceInformation.getValue(); - Long stepFactorValue = UnitsConversionUtil + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil .convert(stepFactorResourceInformation.getUnits(), rResourceInformation.getUnits(), stepFactorResourceInformation.getValue()); - Long value = rValue; + long value = rValue; if (stepFactorValue != 0) { value = roundUp ? roundUp(rValue, stepFactorValue) : roundDown(rValue, stepFactorValue); } - tmp.setValue(value); - ret.setResourceInformation(resource, tmp); + ResourceInformation + .copy(rResourceInformation, ret.getResourceInformation(resource)); + ret.getResourceInformation(resource).setValue(value); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -372,15 +368,14 @@ public class DominantResourceCalculator extends ResourceCalculator { private Resource multiplyAndNormalize(Resource r, double by, Resource stepFactor, boolean roundUp) { - Resource ret = Resources.createResource(0, 0); + Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { try { ResourceInformation rResourceInformation = r.getResourceInformation(resource); ResourceInformation stepFactorResourceInformation = stepFactor.getResourceInformation(resource); - ResourceInformation tmp = - ResourceInformation.newInstance(rResourceInformation); + ResourceInformation tmp = ret.getResourceInformation(resource); Long rValue = rResourceInformation.getValue(); Long stepFactorValue = UnitsConversionUtil @@ -397,7 +392,6 @@ public class DominantResourceCalculator extends ResourceCalculator { roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by); } tmp.setValue(value); - ret.setResourceInformation(resource, tmp); } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); @@ -414,7 +408,7 @@ public class DominantResourceCalculator extends ResourceCalculator { smaller.getResourceInformation(resource); ResourceInformation bResourceInformation = bigger.getResourceInformation(resource); - Long sResourceValue = UnitsConversionUtil + long sResourceValue = UnitsConversionUtil .convert(sResourceInformation.getUnits(), bResourceInformation.getUnits(), sResourceInformation.getValue()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index a591be9..c499da6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -42,7 +42,7 @@ public class Resources { static class FixedValueResource extends Resource { private Map<String, ResourceInformation> resources; - private Long resourceValue; + private long resourceValue; private String name; /** @@ -50,7 +50,7 @@ public class Resources { * @param rName the name of the resource * @param value the fixed value to be returned for all resource types */ - FixedValueResource(String rName, Long value) { + FixedValueResource(String rName, long value) { this.resourceValue = value; this.name = rName; resources = initResourceMap(); @@ -60,7 +60,7 @@ public class Resources { if(this.resourceValue > Integer.MAX_VALUE) { return Integer.MAX_VALUE; } - return this.resourceValue.intValue(); + return Long.valueOf(this.resourceValue).intValue(); } @Override @@ -209,7 +209,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); @@ -231,7 +231,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs); @@ -294,7 +294,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = (long) (UnitsConversionUtil + long convertedRhs = (long) (UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()) * by); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); @@ -428,7 +428,7 @@ public class Resources { try { ResourceInformation rhsValue = bigger.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); if(lhsValue.getValue() > convertedRhs) { @@ -454,7 +454,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); ResourceInformation outInfo = @@ -475,7 +475,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(name); ResourceInformation lhsValue = entry.getValue(); - Long convertedRhs = UnitsConversionUtil + long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); ResourceInformation outInfo = http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 6f17659..38554b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -283,6 +283,7 @@ public class TestResourceUtils { Map<String, ResourceInformation> actual = ResourceUtils.getNodeResourceInformation(conf); Assert.assertEquals(entry.getValue().getResources(), actual); + dest.delete(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/83436f27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index d81dcdb..1555e55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -42,6 +42,16 @@ import static org.junit.Assert.assertTrue; public class TestResources { + static class ExtendedResources extends Resources { + public static Resource unbounded() { + return new FixedValueResource("UNBOUNDED", Long.MAX_VALUE); + } + + public static Resource none() { + return new FixedValueResource("NONE", 0L); + } + } + private static final String EXTRA_RESOURCE_TYPE = "resource2"; private String resourceTypesFile; @@ -87,7 +97,7 @@ public class TestResources { @Test(timeout = 10000) public void testCompareToWithUnboundedResource() { unsetExtraResourceType(); - Resource unboundedClone = Resources.clone(Resources.unbounded()); + Resource unboundedClone = Resources.clone(ExtendedResources.unbounded()); assertTrue(unboundedClone .compareTo(createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0); assertTrue(unboundedClone.compareTo(createResource(Long.MAX_VALUE, 0)) > 0); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org