Author: vinodkv Date: Mon Jul 7 20:14:33 2014 New Revision: 1608596 URL: http://svn.apache.org/r1608596 Log: MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side resource configuration for deciding uber-mode on map-only jobs. Contributed by Siqi Li. svn merge --ignore-ancestry -c 1608595 ../../trunk/
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1608596&r1=1608595&r2=1608596&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Mon Jul 7 20:14:33 2014 @@ -152,6 +152,10 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5868. Fixed an issue with TestPipeApplication that was causing the nightly builds to fail. (Akira Ajisaka via vinodkv) + MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side + resource configuration for deciding uber-mode on map-only jobs. (Siqi Li via + vinodkv) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1608596&r1=1608595&r2=1608596&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Mon Jul 7 20:14:33 2014 @@ -1218,22 +1218,25 @@ public class JobImpl implements org.apac boolean smallNumReduceTasks = (numReduceTasks <= sysMaxReduces); boolean smallInput = (dataInputLength <= sysMaxBytes); // ignoring overhead due to UberAM and statics as negligible here: + long requiredMapMB = conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0); + long requiredReduceMB = conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0); + long requiredMB = Math.max(requiredMapMB, requiredReduceMB); + int requiredMapCores = conf.getInt( + MRJobConfig.MAP_CPU_VCORES, + MRJobConfig.DEFAULT_MAP_CPU_VCORES); + int requiredReduceCores = conf.getInt( + MRJobConfig.REDUCE_CPU_VCORES, + MRJobConfig.DEFAULT_REDUCE_CPU_VCORES); + int requiredCores = Math.max(requiredMapCores, requiredReduceCores); + if (numReduceTasks == 0) { + requiredMB = requiredMapMB; + requiredCores = requiredMapCores; + } boolean smallMemory = - ( (Math.max(conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0), - conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0)) - <= sysMemSizeForUberSlot) - || (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT)); - boolean smallCpu = - ( - Math.max( - conf.getInt( - MRJobConfig.MAP_CPU_VCORES, - MRJobConfig.DEFAULT_MAP_CPU_VCORES), - conf.getInt( - MRJobConfig.REDUCE_CPU_VCORES, - MRJobConfig.DEFAULT_REDUCE_CPU_VCORES)) - <= sysCPUSizeForUberSlot - ); + (requiredMB <= sysMemSizeForUberSlot) + || (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT); + + boolean smallCpu = requiredCores <= sysCPUSizeForUberSlot; boolean notChainJob = !isChainJob(conf); // User has overall veto power over uberization, or user can modify Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java?rev=1608596&r1=1608595&r2=1608596&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java Mon Jul 7 20:14:33 2014 @@ -657,6 +657,15 @@ public class TestJobImpl { conf.setInt(MRJobConfig.JOB_UBERTASK_MAXMAPS, 1); isUber = testUberDecision(conf); Assert.assertFalse(isUber); + + // enable uber mode of 0 reducer no matter how much memory assigned to reducer + conf = new Configuration(); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true); + conf.setInt(MRJobConfig.NUM_REDUCES, 0); + conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 2048); + conf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 10); + isUber = testUberDecision(conf); + Assert.assertTrue(isUber); } private boolean testUberDecision(Configuration conf) {