YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52ba61a9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52ba61a9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52ba61a9 Branch: refs/heads/YARN-3926 Commit: 52ba61a92239a3f68b1455d7a71e4e3714de5b05 Parents: cf68c1a Author: Arun Suresh <asur...@apache.org> Authored: Thu Mar 17 23:50:22 2016 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Mon Aug 21 16:52:54 2017 -0700 ---------------------------------------------------------------------- .../yarn/api/records/ResourceInformation.java | 2 +- .../hadoop/yarn/util/UnitsConversionUtil.java | 45 ++++++++++++++------ .../yarn/util/TestUnitsConversionUtil.java | 17 +++++++- .../api/records/impl/pb/ResourcePBImpl.java | 2 +- .../yarn/util/resource/TestResourceUtils.java | 2 +- 5 files changed, 52 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/52ba61a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 80e3192..a17e81b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> { private static final String VCORES_URI = "vcores"; public static final ResourceInformation MEMORY_MB = - ResourceInformation.newInstance(MEMORY_URI, "M"); + ResourceInformation.newInstance(MEMORY_URI, "Mi"); public static final ResourceInformation VCORES = ResourceInformation.newInstance(VCORES_URI); http://git-wip-us.apache.org/repos/asf/hadoop/blob/52ba61a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java index 7785263..47bb3df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -46,7 +46,8 @@ public class UnitsConversionUtil { } private static final String[] UNITS = - {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"}; + { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti", + "Pi" }; private static final List<String> SORTED_UNITS = Arrays.asList(UNITS); public static final Set<String> KNOWN_UNITS = createKnownUnitsSet(); private static final Converter PICO = @@ -65,6 +66,15 @@ public class UnitsConversionUtil { private static final Converter PETA = new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L); + private static final Converter KILO_BINARY = new Converter(1024L, 1L); + private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L); + private static final Converter GIGA_BINARY = + new Converter(1024L * 1024L * 1024L, 1L); + private static final Converter TERA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L, 1L); + private static final Converter PETA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L); + private static Set<String> createKnownUnitsSet() { Set<String> ret = new HashSet<>(); ret.addAll(Arrays.asList(UNITS)); @@ -93,6 +103,16 @@ public class UnitsConversionUtil { return TERA; case "P": return PETA; + case "Ki": + return KILO_BINARY; + case "Mi": + return MEGA_BINARY; + case "Gi": + return GIGA_BINARY; + case "Ti": + return TERA_BINARY; + case "Pi": + return PETA_BINARY; default: throw new IllegalArgumentException( "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS); @@ -112,28 +132,29 @@ public class UnitsConversionUtil { if (toUnit == null || fromUnit == null || fromValue == null) { throw new IllegalArgumentException("One or more arguments are null"); } - Long tmp; String overflowMsg = "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + "' will result in an overflow of Long"; + if (fromUnit.equals(toUnit)) { + return fromValue; + } Converter fc = getConverter(fromUnit); Converter tc = getConverter(toUnit); Long numerator = fc.numerator * tc.denominator; Long denominator = fc.denominator * tc.numerator; + Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator; if (numerator < denominator) { - if (!toUnit.equals(fromUnit)) { - tmp = Long.MAX_VALUE / numerator; - if (tmp < fromValue) { - throw new IllegalArgumentException(overflowMsg); - } + if (numeratorMultiplierLimit < fromValue) { + throw new IllegalArgumentException(overflowMsg); } return (fromValue * numerator) / denominator; } - tmp = numerator / denominator; - if (!toUnit.equals(fromUnit)) { - if ((Long.MAX_VALUE / tmp) < fromValue) { - throw new IllegalArgumentException(overflowMsg); - } + if (numeratorMultiplierLimit > fromValue) { + return (numerator * fromValue) / denominator; + } + Long tmp = numerator / denominator; + if ((Long.MAX_VALUE / tmp) < fromValue) { + throw new IllegalArgumentException(overflowMsg); } return fromValue * tmp; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/52ba61a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java index 421768f..0f999e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.util; -import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.junit.Assert; import org.junit.Test; @@ -45,6 +44,7 @@ public class TestUnitsConversionUtil { fromUnit = ""; Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l), UnitsConversionUtil.convert(fromUnit, "k", test)); + Assert .assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)), UnitsConversionUtil.convert(fromUnit, "M", test)); @@ -63,6 +63,21 @@ public class TestUnitsConversionUtil { Assert.assertEquals("mega to giga test failed", Long.valueOf(value), UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l))); + + Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value), + UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l))); + + Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024), + UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value))); + + Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024), + UnitsConversionUtil.convert("Ki", "", Long.valueOf(5))); + + Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741), + UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024))); + + Assert.assertEquals("M to Mi test failed", Long.valueOf(953), + UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000))); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/52ba61a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index fc91ec3..174f3cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -95,7 +95,7 @@ public class ResourcePBImpl extends Resource { initResourcesMap(); ResourceInformation ri = this.getResourceInformation(ResourceInformation.MEMORY_MB.getName()); - return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue()); + return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue()); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/52ba61a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 606e89c..4bb68c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -195,7 +195,7 @@ public class TestResourceUtils { // we must always have memory and vcores with their fixed units Assert.assertTrue(ret.containsKey("memory-mb")); ResourceInformation memInfo = ret.get("memory-mb"); - Assert.assertEquals("M", memInfo.getUnits()); + Assert.assertEquals("Mi", memInfo.getUnits()); Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType()); Assert.assertTrue(ret.containsKey("vcores")); ResourceInformation vcoresInfo = ret.get("vcores"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org