Author: sandy Date: Wed Sep 18 13:44:33 2013 New Revision: 1524408 URL: http://svn.apache.org/r1524408 Log: MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again in Limits (Sandy Ryza)
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1524408&r1=1524407&r2=1524408&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Sep 18 13:44:33 2013 @@ -164,6 +164,9 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) + MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again + in Limits (Sandy Ryza) + BUG FIXES MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1524408&r1=1524407&r2=1524408&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Wed Sep 18 13:44:33 2013 @@ -41,6 +41,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.counters.Limits; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; @@ -76,6 +77,8 @@ class YarnChild { LOG.debug("Child starting"); final JobConf job = new JobConf(); + // Initing with our JobConf allows us to avoid loading confs twice + Limits.init(job); job.addResource(MRJobConfig.JOB_CONF_FILE); UserGroupInformation.setConfiguration(job); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java?rev=1524408&r1=1524407&r2=1524408&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java Wed Sep 18 13:44:33 2013 @@ -62,8 +62,8 @@ import com.google.common.collect.Iterato public class Counters extends AbstractCounters<Counters.Counter, Counters.Group> { - public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX; - public static int MAX_GROUP_LIMIT = Limits.GROUPS_MAX; + public static int MAX_COUNTER_LIMIT = Limits.getCountersMax(); + public static int MAX_GROUP_LIMIT = Limits.getGroupsMax(); private static HashMap<String, String> depricatedCounterMap = new HashMap<String, String>(); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java?rev=1524408&r1=1524407&r2=1524408&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java Wed Sep 18 13:44:33 2013 @@ -28,37 +28,80 @@ import static org.apache.hadoop.mapreduc public class Limits { static final Configuration conf = new JobConf(); - public static final int GROUP_NAME_MAX = - conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT); - public static final int COUNTER_NAME_MAX = - conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT); - public static final int GROUPS_MAX = - conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT); - public static final int COUNTERS_MAX = - conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT); private int totalCounters; private LimitExceededException firstViolation; + private static boolean isInited; + + private static int GROUP_NAME_MAX; + private static int COUNTER_NAME_MAX; + private static int GROUPS_MAX; + private static int COUNTERS_MAX; + + public synchronized static void init(Configuration conf) { + if (!isInited) { + if (conf == null) { + conf = new JobConf(); + } + GROUP_NAME_MAX = conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, + COUNTER_GROUP_NAME_MAX_DEFAULT); + COUNTER_NAME_MAX = conf.getInt(COUNTER_NAME_MAX_KEY, + COUNTER_NAME_MAX_DEFAULT); + GROUPS_MAX = conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT); + COUNTERS_MAX = conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT); + } + isInited = true; + } + + public static int getGroupNameMax() { + if (!isInited) { + init(null); + } + return GROUP_NAME_MAX; + } + + public static int getCounterNameMax() { + if (!isInited) { + init(null); + } + return COUNTER_NAME_MAX; + } + + public static int getGroupsMax() { + if (!isInited) { + init(null); + } + return GROUPS_MAX; + } + + public static int getCountersMax() { + if (!isInited) { + init(null); + } + return COUNTERS_MAX; + } + public static String filterName(String name, int maxLen) { return name.length() > maxLen ? name.substring(0, maxLen - 1) : name; } public static String filterCounterName(String name) { - return filterName(name, COUNTER_NAME_MAX); + return filterName(name, getCounterNameMax()); } public static String filterGroupName(String name) { - return filterName(name, GROUP_NAME_MAX); + return filterName(name, getGroupNameMax()); } public synchronized void checkCounters(int size) { if (firstViolation != null) { throw new LimitExceededException(firstViolation); } - if (size > COUNTERS_MAX) { + int countersMax = getCountersMax(); + if (size > countersMax) { firstViolation = new LimitExceededException("Too many counters: "+ size + - " max="+ COUNTERS_MAX); + " max="+ countersMax); throw firstViolation; } } @@ -72,9 +115,10 @@ public class Limits { if (firstViolation != null) { throw new LimitExceededException(firstViolation); } - if (size > GROUPS_MAX) { + int groupsMax = getGroupsMax(); + if (size > groupsMax) { firstViolation = new LimitExceededException("Too many counter groups: "+ - size +" max="+ GROUPS_MAX); + size +" max="+ groupsMax); } } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java?rev=1524408&r1=1524407&r2=1524408&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java Wed Sep 18 13:44:33 2013 @@ -101,8 +101,8 @@ public class TestCounters { static final long FS_COUNTER_VALUE = 10; private void testMaxCounters(final Counters counters) { - LOG.info("counters max="+ Limits.COUNTERS_MAX); - for (int i = 0; i < Limits.COUNTERS_MAX; ++i) { + LOG.info("counters max="+ Limits.getCountersMax()); + for (int i = 0; i < Limits.getCountersMax(); ++i) { counters.findCounter("test", "test"+ i); } setExpected(counters); @@ -115,8 +115,8 @@ public class TestCounters { } private void testMaxGroups(final Counters counters) { - LOG.info("counter groups max="+ Limits.GROUPS_MAX); - for (int i = 0; i < Limits.GROUPS_MAX; ++i) { + LOG.info("counter groups max="+ Limits.getGroupsMax()); + for (int i = 0; i < Limits.getGroupsMax(); ++i) { // assuming COUNTERS_MAX > GROUPS_MAX counters.findCounter("test"+ i, "test"); }