Author: suresh Date: Wed Apr 17 00:23:22 2013 New Revision: 1468698 URL: http://svn.apache.org/r1468698 Log: HDFS-4635. Move BlockManager#computeCapacity to LightWeightGSet. Contributed by Suresh Srinivas.
Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/util/LightWeightGSet.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/util/TestGSet.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1468698&r1=1468697&r2=1468698&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Wed Apr 17 00:23:22 2013 @@ -236,6 +236,9 @@ Release 1.2.0 - unreleased MAPREDUCE-5129. Allow tags to JobHistory for deeper analytics. (billie via acmurthy) + HDFS-4635. Move BlockManager#computeCapacity to LightWeightGSet. + (suresh) + BUG FIXES HADOOP-9467. Metrics2 record filter should check name as well as tags. Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java?rev=1468698&r1=1468697&r2=1468698&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java Wed Apr 17 00:23:22 2013 @@ -315,37 +315,11 @@ class BlocksMap { private GSet<Block, BlockInfo> blocks; BlocksMap(int initialCapacity, float loadFactor) { - this.capacity = computeCapacity(); + // Use 2% of total memory to size the GSet capacity + this.capacity = LightWeightGSet.computeCapacity(2.0, "BlocksMap"); this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity); } - /** - * Let t = 2% of max memory. - * Let e = round(log_2 t). - * Then, we choose capacity = 2^e/(size of reference), - * unless it is outside the close interval [1, 2^30]. - */ - private static int computeCapacity() { - //VM detection - //See http://java.sun.com/docs/hotspot/HotSpotFAQ.html#64bit_detection - final String vmBit = System.getProperty("sun.arch.data.model"); - - //2% of max memory - final double twoPC = Runtime.getRuntime().maxMemory()/50.0; - - //compute capacity - final int e1 = (int)(Math.log(twoPC)/Math.log(2.0) + 0.5); - final int e2 = e1 - ("32".equals(vmBit)? 2: 3); - final int exponent = e2 < 0? 0: e2 > 30? 30: e2; - final int c = 1 << exponent; - - LightWeightGSet.LOG.info("VM type = " + vmBit + "-bit"); - LightWeightGSet.LOG.info("2% max memory = " + twoPC/(1 << 20) + " MB"); - LightWeightGSet.LOG.info("capacity = 2^" + exponent - + " = " + c + " entries"); - return c; - } - void close() { blocks = null; } Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/util/LightWeightGSet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/util/LightWeightGSet.java?rev=1468698&r1=1468697&r2=1468698&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/util/LightWeightGSet.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/util/LightWeightGSet.java Wed Apr 17 00:23:22 2013 @@ -280,4 +280,53 @@ public class LightWeightGSet<K, E extend throw new UnsupportedOperationException("Remove is not supported."); } } + + /** + * Let t = percentage of max memory. + * Let e = round(log_2 t). + * Then, we choose capacity = 2^e/(size of reference), + * unless it is outside the close interval [1, 2^30]. + */ + public static int computeCapacity(double percentage, String mapName) { + return computeCapacity(Runtime.getRuntime().maxMemory(), percentage, + mapName); + } + + /** Visible for testing */ + static int computeCapacity(long maxMemory, double percentage, + String mapName) { + if (percentage > 100.0 || percentage < 0.0) { + throw new IllegalArgumentException("Percentage " + percentage + + " must be greater than or equal to 0 " + + " and less than or equal to 100"); + } + if (maxMemory < 0) { + throw new IllegalArgumentException("Memory " + maxMemory + + " must be greater than or equal to 0"); + } + if (percentage == 0.0 || maxMemory == 0) { + return 0; + } + //VM detection + //See http://java.sun.com/docs/hotspot/HotSpotFAQ.html#64bit_detection + final String vmBit = System.getProperty("sun.arch.data.model"); + + //Percentage of max memory + final double percentDivisor = 100.0/percentage; + final double percentMemory = maxMemory/percentDivisor; + + //compute capacity + final int e1 = (int)(Math.log(percentMemory)/Math.log(2.0) + 0.5); + final int e2 = e1 - ("32".equals(vmBit)? 2: 3); + final int exponent = e2 < 0? 0: e2 > 30? 30: e2; + final int c = 1 << exponent; + + if (LightWeightGSet.LOG.isDebugEnabled()) { + LOG.debug("Computing capacity for map " + mapName); + LOG.debug("VM type = " + vmBit + "-bit"); + LOG.debug(percentage + "% max memory = " + maxMemory); + LOG.debug("capacity = 2^" + exponent + " = " + c + " entries"); + } + return c; + } } Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/util/TestGSet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/util/TestGSet.java?rev=1468698&r1=1468697&r2=1468698&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/util/TestGSet.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/util/TestGSet.java Wed Apr 17 00:23:22 2013 @@ -451,4 +451,81 @@ public class TestGSet { next = e; } } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + * with invalid percent less than 0. + */ + @Test(expected=IllegalArgumentException.class) + public void testComputeCapacityNegativePercent() { + LightWeightGSet.computeCapacity(1024, -1.0, "testMap"); + } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + * with invalid percent greater than 100. + */ + @Test(expected=IllegalArgumentException.class) + public void testComputeCapacityInvalidPercent() { + LightWeightGSet.computeCapacity(1024, 101.0, "testMap"); + } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + * with invalid negative max memory + */ + @Test(expected=IllegalArgumentException.class) + public void testComputeCapacityInvalidMemory() { + LightWeightGSet.computeCapacity(-1, 50.0, "testMap"); + } + + private static boolean isPowerOfTwo(int num) { + return num == 0 || (num > 0 && Integer.bitCount(num) == 1); + } + + /** Return capacity as percentage of total memory */ + private static int getPercent(long total, int capacity) { + // Reference size in bytes + double referenceSize = + System.getProperty("sun.arch.data.model").equals("32") ? 4.0 : 8.0; + return (int)(((capacity * referenceSize)/total) * 100.0); + } + + /** Return capacity as percentage of total memory */ + private static void testCapacity(long maxMemory, double percent) { + int capacity = LightWeightGSet.computeCapacity(maxMemory, percent, "map"); + LightWeightGSet.LOG.info("Validating - total memory " + maxMemory + " percent " + + percent + " returned capacity " + capacity); + // Returned capacity is zero or power of two + Assert.assertTrue(isPowerOfTwo(capacity)); + + // Ensure the capacity returned is the nearest to the asked perecentage + int capacityPercent = getPercent(maxMemory, capacity); + if (capacityPercent == percent) { + return; + } else if (capacityPercent > percent) { + Assert.assertTrue(getPercent(maxMemory, capacity * 2) > percent); + } else { + Assert.assertTrue(getPercent(maxMemory, capacity / 2) < percent); + } + } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + */ + @Test + public void testComputeCapacity() { + // Tests for boundary conditions where percent or memory are zero + testCapacity(0, 0.0); + testCapacity(100, 0.0); + testCapacity(0, 100.0); + + // Compute capacity for some 100 random max memory and percentage + Random r = new Random(); + for (int i = 0; i < 100; i++) { + long maxMemory = r.nextInt(Integer.MAX_VALUE); + double percent = r.nextInt(101); + testCapacity(maxMemory, percent); + } + } }